diff --git a/.mvn/.gitkeep b/.mvn/.gitkeep new file mode 100644 index 00000000000..e69de29bb2d diff --git a/.mvn/jvm.config b/.mvn/jvm.config new file mode 100644 index 00000000000..3d61a1f27e7 --- /dev/null +++ b/.mvn/jvm.config @@ -0,0 +1,11 @@ +--add-exports jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED +--add-exports jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED +--add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED +--add-exports jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED +--add-exports jdk.compiler/com.sun.tools.javac.model=ALL-UNNAMED +--add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED +--add-exports jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED +--add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED +--add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED +--add-opens jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED +--add-opens jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED diff --git a/examples/storm-hdfs-examples/pom.xml b/examples/storm-hdfs-examples/pom.xml index 40e66a2a582..a307844ef27 100644 --- a/examples/storm-hdfs-examples/pom.xml +++ b/examples/storm-hdfs-examples/pom.xml @@ -93,6 +93,16 @@ + + org.openrewrite.maven + rewrite-maven-plugin + + + + org.codehaus.gmavenplus + gmavenplus-plugin + + org.apache.maven.plugins maven-checkstyle-plugin diff --git a/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/bolt/HdfsFileTopology.java b/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/bolt/HdfsFileTopology.java index 01a446c9353..c4087e72801 100644 --- a/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/bolt/HdfsFileTopology.java +++ b/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/bolt/HdfsFileTopology.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -56,7 +62,8 @@ public static void main(String[] args) throws Exception { SyncPolicy syncPolicy = new CountSyncPolicy(1000); // rotate files when they reach 5MB - FileRotationPolicy rotationPolicy = new TimedRotationPolicy(1.0f, TimedRotationPolicy.TimeUnit.MINUTES); + FileRotationPolicy rotationPolicy = new TimedRotationPolicy(1.0f, + TimedRotationPolicy.TimeUnit.MINUTES); FileNameFormat fileNameFormat = new DefaultFileNameFormat() .withPath("/tmp/foo/") @@ -91,7 +98,9 @@ public static void main(String[] args) throws Exception { if (args.length == 3) { topoName = args[2]; } else if (args.length > 3) { - System.out.println("Usage: HdfsFileTopology [hdfs url] [hdfs yaml config file] "); + System.out + .println("Usage: HdfsFileTopology [hdfs url] [hdfs yaml config file] " + + ""); return; } StormSubmitter.submitTopology(topoName, config, builder.createTopology()); @@ -101,7 +110,7 @@ public static void waitForSeconds(int seconds) { try { Thread.sleep(seconds * 1000); } catch (InterruptedException e) { - //ignore + // ignore } } @@ -145,7 +154,8 @@ public void nextTuple() { total++; if (count > 20000) { count = 0; - System.out.println("Pending count: " + this.pending.size() + ", total: " + this.total); + System.out.println("Pending count: " + this.pending.size() + ", total: " + + this.total); } Thread.yield(); } @@ -168,7 +178,8 @@ public static class MyBolt extends BaseRichBolt { private OutputCollector collector; @Override - public void prepare(Map config, TopologyContext context, OutputCollector collector) { + public void prepare(Map config, TopologyContext context, + OutputCollector collector) { this.counts = new HashMap(); this.collector = collector; } diff --git a/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/bolt/SequenceFileTopology.java b/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/bolt/SequenceFileTopology.java index 90e0aea0d42..111e6a61807 100644 --- a/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/bolt/SequenceFileTopology.java +++ b/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/bolt/SequenceFileTopology.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -25,8 +31,8 @@ import org.apache.storm.hdfs.bolt.format.DefaultSequenceFormat; import org.apache.storm.hdfs.bolt.format.FileNameFormat; import org.apache.storm.hdfs.bolt.rotation.FileRotationPolicy; -import org.apache.storm.hdfs.bolt.rotation.FileSizeRotationPolicy; import org.apache.storm.hdfs.bolt.rotation.FileSizeRotationPolicy.Units; +import org.apache.storm.hdfs.bolt.rotation.FileSizeRotationPolicy; import org.apache.storm.hdfs.bolt.sync.CountSyncPolicy; import org.apache.storm.hdfs.bolt.sync.SyncPolicy; import org.apache.storm.hdfs.common.rotation.MoveFileAction; @@ -94,7 +100,9 @@ public static void main(String[] args) throws Exception { if (args.length == 3) { topoName = args[2]; } else if (args.length > 3) { - System.out.println("Usage: SequenceFileTopology [hdfs url] [hdfs yaml config file] "); + System.out + .println("Usage: SequenceFileTopology [hdfs url] [hdfs yaml config file] " + + ""); return; } StormSubmitter.submitTopology(topoName, config, builder.createTopology()); @@ -104,14 +112,12 @@ public static void waitForSeconds(int seconds) { try { Thread.sleep(seconds * 1000); } catch (InterruptedException e) { - //ignore + // ignore } } - public static class SentenceSpout extends BaseRichSpout { - private ConcurrentHashMap pending; private SpoutOutputCollector collector; private String[] sentences = { @@ -151,7 +157,8 @@ public void nextTuple() { total++; if (count > 20000) { count = 0; - System.out.println("Pending count: " + this.pending.size() + ", total: " + this.total); + System.out.println("Pending count: " + this.pending.size() + ", total: " + + this.total); } Thread.yield(); } @@ -169,14 +176,14 @@ public void fail(Object msgId) { } } - public static class MyBolt extends BaseRichBolt { private HashMap counts = null; private OutputCollector collector; @Override - public void prepare(Map config, TopologyContext context, OutputCollector collector) { + public void prepare(Map config, TopologyContext context, + OutputCollector collector) { this.counts = new HashMap(); this.collector = collector; } @@ -186,7 +193,6 @@ public void execute(Tuple tuple) { collector.ack(tuple); } - @Override public void declareOutputFields(OutputFieldsDeclarer declarer) { // this bolt does not emit anything diff --git a/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/spout/HdfsSpoutTopology.java b/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/spout/HdfsSpoutTopology.java index 01d44832a49..3a59a20cb94 100644 --- a/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/spout/HdfsSpoutTopology.java +++ b/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/spout/HdfsSpoutTopology.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -35,14 +41,14 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; - public class HdfsSpoutTopology { public static final String SPOUT_ID = "hdfsspout"; public static final String BOLT_ID = "constbolt"; /** - * Copies text file content from sourceDir to destinationDir. Moves source files into sourceDir after its done consuming + * Copies text file content from sourceDir to destinationDir. Moves source files into sourceDir + * after its done consuming */ public static void main(String[] args) throws Exception { // 0 - validate args @@ -50,13 +56,21 @@ public static void main(String[] args) throws Exception { System.err.println("Please check command line arguments."); System.err.println("Usage :"); System.err.println( - HdfsSpoutTopology.class.toString() + " topologyName hdfsUri fileFormat sourceDir sourceArchiveDir badDir destinationDir."); + HdfsSpoutTopology.class.toString() + + " topologyName hdfsUri fileFormat sourceDir sourceArchiveDir badDir " + + "destinationDir."); System.err.println(" topologyName - topology name."); System.err.println(" hdfsUri - hdfs name node URI"); - System.err.println(" fileFormat - Set to 'TEXT' for reading text files or 'SEQ' for sequence files."); + System.err + .println(" fileFormat - Set to 'TEXT' for reading text files or 'SEQ' for " + + "sequence files."); System.err.println(" sourceDir - read files from this HDFS dir using HdfsSpout."); - System.err.println(" archiveDir - after a file in sourceDir is read completely, it is moved to this HDFS location."); - System.err.println(" badDir - files that cannot be read properly will be moved to this HDFS location."); + System.err + .println(" archiveDir - after a file in sourceDir is read completely, it is " + + "moved to this HDFS location."); + System.err + .println(" badDir - files that cannot be read properly will be moved to this " + + "HDFS location."); System.err.println(" spoutCount - Num of spout instances."); System.err.println(); System.exit(-1); @@ -153,7 +167,8 @@ public ConstBolt() { } @Override - public void prepare(Map conf, TopologyContext context, OutputCollector collector) { + public void prepare(Map conf, TopologyContext context, + OutputCollector collector) { this.collector = collector; } diff --git a/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/trident/TridentFileTopology.java b/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/trident/TridentFileTopology.java index b722497183c..5d495508222 100644 --- a/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/trident/TridentFileTopology.java +++ b/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/trident/TridentFileTopology.java @@ -41,11 +41,15 @@ public class TridentFileTopology { public static StormTopology buildTopology(String hdfsUrl) { - FixedBatchSpout spout = new FixedBatchSpout(new Fields("sentence", "key"), 1000, new Values("the cow jumped over the moon", 1L), - new Values("the man went to the store and bought some candy", 2L), - new Values("four score and seven years ago", 3L), + FixedBatchSpout spout = new FixedBatchSpout(new Fields("sentence", "key"), 1000, + new Values("the cow jumped over the moon", 1L), + new Values("the man went to the store and " + + "bought some candy", 2L), + new Values("four score and seven years ago", + 3L), new Values("how many apples can you eat", 4L), - new Values("to be or not to be the person", 5L)); + new Values("to be or not to be the person", + 5L)); spout.setCycle(true); TridentTopology topology = new TridentTopology(); @@ -61,7 +65,8 @@ public static StormTopology buildTopology(String hdfsUrl) { RecordFormat recordFormat = new DelimitedRecordFormat() .withFields(hdfsFields); - FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(5.0f, FileSizeRotationPolicy.Units.MB); + FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(5.0f, + FileSizeRotationPolicy.Units.MB); HdfsState.Options options = new HdfsState.HdfsFileOptions() .withFileNameFormat(fileNameFormat) @@ -91,7 +96,9 @@ public static void main(String[] args) throws Exception { if (args.length == 3) { topoName = args[2]; } else if (args.length > 3) { - System.out.println("Usage: TridentFileTopology [hdfs url] [hdfs yaml config file] "); + System.out + .println("Usage: TridentFileTopology [hdfs url] [hdfs yaml config file] " + + ""); return; } conf.setNumWorkers(3); diff --git a/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/trident/TridentSequenceTopology.java b/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/trident/TridentSequenceTopology.java index 74d1d5c8dc5..dddd727e2f5 100644 --- a/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/trident/TridentSequenceTopology.java +++ b/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/trident/TridentSequenceTopology.java @@ -41,11 +41,15 @@ public class TridentSequenceTopology { public static StormTopology buildTopology(String hdfsUrl) { - FixedBatchSpout spout = new FixedBatchSpout(new Fields("sentence", "key"), 1000, new Values("the cow jumped over the moon", 1L), - new Values("the man went to the store and bought some candy", 2L), - new Values("four score and seven years ago", 3L), + FixedBatchSpout spout = new FixedBatchSpout(new Fields("sentence", "key"), 1000, + new Values("the cow jumped over the moon", 1L), + new Values("the man went to the store and " + + "bought some candy", 2L), + new Values("four score and seven years ago", + 3L), new Values("how many apples can you eat", 4L), - new Values("to be or not to be the person", 5L)); + new Values("to be or not to be the person", + 5L)); spout.setCycle(true); TridentTopology topology = new TridentTopology(); @@ -58,7 +62,8 @@ public static StormTopology buildTopology(String hdfsUrl) { .withPrefix("trident") .withExtension(".seq"); - FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(5.0f, FileSizeRotationPolicy.Units.MB); + FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(5.0f, + FileSizeRotationPolicy.Units.MB); HdfsState.Options seqOpts = new HdfsState.SequenceFileOptions() .withFileNameFormat(fileNameFormat) @@ -88,7 +93,8 @@ public static void main(String[] args) throws Exception { if (args.length == 3) { topoName = args[2]; } else if (args.length > 3) { - System.out.println("Usage: TridentSequenceTopology []"); + System.out + .println("Usage: TridentSequenceTopology []"); return; } diff --git a/examples/storm-iceberg-examples/src/main/java/org/apache/storm/iceberg/examples/BoundedTupleSpout.java b/examples/storm-iceberg-examples/src/main/java/org/apache/storm/iceberg/examples/BoundedTupleSpout.java index 95c4596f79b..20c7ceefa0e 100644 --- a/examples/storm-iceberg-examples/src/main/java/org/apache/storm/iceberg/examples/BoundedTupleSpout.java +++ b/examples/storm-iceberg-examples/src/main/java/org/apache/storm/iceberg/examples/BoundedTupleSpout.java @@ -60,7 +60,8 @@ public BoundedTupleSpout(long totalTuples, Fields outputFields, ValuesFactory va } @Override - public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { + public void open(Map conf, TopologyContext context, + SpoutOutputCollector collector) { this.collector = collector; // Each task walks its own stride of the sequence, so the spout can be parallelised without // any two tasks emitting the same row. diff --git a/examples/storm-jdbc-examples/pom.xml b/examples/storm-jdbc-examples/pom.xml index 95525a3cc26..5a3e7ab82e2 100644 --- a/examples/storm-jdbc-examples/pom.xml +++ b/examples/storm-jdbc-examples/pom.xml @@ -85,6 +85,16 @@ + + org.openrewrite.maven + rewrite-maven-plugin + + + + org.codehaus.gmavenplus + gmavenplus-plugin + + org.apache.maven.plugins maven-checkstyle-plugin diff --git a/examples/storm-jdbc-examples/src/main/java/org/apache/storm/jdbc/spout/UserSpout.java b/examples/storm-jdbc-examples/src/main/java/org/apache/storm/jdbc/spout/UserSpout.java index 6854722191d..59c383ca887 100644 --- a/examples/storm-jdbc-examples/src/main/java/org/apache/storm/jdbc/spout/UserSpout.java +++ b/examples/storm-jdbc-examples/src/main/java/org/apache/storm/jdbc/spout/UserSpout.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -19,11 +19,9 @@ package org.apache.storm.jdbc.spout; import com.google.common.collect.Lists; - import java.util.List; import java.util.Map; import java.util.Random; - import org.apache.storm.spout.SpoutOutputCollector; import org.apache.storm.task.TopologyContext; import org.apache.storm.topology.IRichSpout; @@ -52,7 +50,8 @@ public boolean isDistributed() { } @Override - public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { + public void open(Map conf, TopologyContext context, + SpoutOutputCollector collector) { this.collector = collector; } diff --git a/examples/storm-jdbc-examples/src/main/java/org/apache/storm/jdbc/topology/AbstractUserTopology.java b/examples/storm-jdbc-examples/src/main/java/org/apache/storm/jdbc/topology/AbstractUserTopology.java index aa7f68609a2..d741f675b67 100644 --- a/examples/storm-jdbc-examples/src/main/java/org/apache/storm/jdbc/topology/AbstractUserTopology.java +++ b/examples/storm-jdbc-examples/src/main/java/org/apache/storm/jdbc/topology/AbstractUserTopology.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -20,11 +20,9 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; - import java.sql.Types; import java.util.List; import java.util.Map; - import org.apache.storm.Config; import org.apache.storm.StormSubmitter; import org.apache.storm.generated.StormTopology; @@ -47,7 +45,8 @@ public abstract class AbstractUserTopology { "drop table if exists user", "drop table if exists department", "drop table if exists user_department", - "create table if not exists user (user_id integer, user_name varchar(100), dept_name varchar(100), create_date date)", + "create table if not exists user (user_id integer, user_name varchar(100), dept_name " + + "varchar(100), create_date date)", "create table if not exists department (dept_id integer, dept_name varchar(100))", "create table if not exists user_department (user_id integer, dept_id integer)", "insert into department values (1, 'R&D')", @@ -72,22 +71,24 @@ public abstract class AbstractUserTopology { /** * A main method template to extend. + * * @param args main method arguments * @throws Exception any expection occuring durch cluster setup or operation */ public void execute(String[] args) throws Exception { if (args.length != 4 && args.length != 5) { - System.out.println("Usage: " + this.getClass().getSimpleName() + " " + System.out.println("Usage: " + this.getClass().getSimpleName() + + " " + " [topology name]"); System.exit(-1); } Map map = Maps.newHashMap(); - map.put("dataSourceClassName", args[0]); //com.mysql.jdbc.jdbc2.optional.MysqlDataSource - map.put("dataSource.url", args[1]); //jdbc:mysql://localhost/test - map.put("dataSource.user", args[2]); //root + map.put("dataSourceClassName", args[0]); // com.mysql.jdbc.jdbc2.optional.MysqlDataSource + map.put("dataSource.url", args[1]); // jdbc:mysql://localhost/test + map.put("dataSource.user", args[2]); // root if (args.length == 4) { - map.put("dataSource.password", args[3]); //password + map.put("dataSource.password", args[3]); // password } Config config = new Config(); diff --git a/examples/storm-jdbc-examples/src/main/java/org/apache/storm/jdbc/topology/UserPersistenceTopology.java b/examples/storm-jdbc-examples/src/main/java/org/apache/storm/jdbc/topology/UserPersistenceTopology.java index 7d4129341f6..3d4bfbb8530 100644 --- a/examples/storm-jdbc-examples/src/main/java/org/apache/storm/jdbc/topology/UserPersistenceTopology.java +++ b/examples/storm-jdbc-examples/src/main/java/org/apache/storm/jdbc/topology/UserPersistenceTopology.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -19,10 +19,8 @@ package org.apache.storm.jdbc.topology; import com.google.common.collect.Lists; - import java.sql.Types; import java.util.List; - import org.apache.storm.generated.StormTopology; import org.apache.storm.jdbc.bolt.JdbcInsertBolt; import org.apache.storm.jdbc.bolt.JdbcLookupBolt; @@ -31,7 +29,6 @@ import org.apache.storm.jdbc.mapper.SimpleJdbcMapper; import org.apache.storm.topology.TopologyBuilder; - public class UserPersistenceTopology extends AbstractUserTopology { private static final String USER_SPOUT = "USER_SPOUT"; private static final String LOOKUP_BOLT = "LOOKUP_BOLT"; @@ -43,9 +40,10 @@ public static void main(String[] args) throws Exception { @Override public StormTopology getTopology() { - JdbcLookupBolt departmentLookupBolt = new JdbcLookupBolt(connectionProvider, SELECT_QUERY, this.jdbcLookupMapper); + JdbcLookupBolt departmentLookupBolt = new JdbcLookupBolt(connectionProvider, SELECT_QUERY, + this.jdbcLookupMapper); - //must specify column schema when providing custom query. + // must specify column schema when providing custom query. List schemaColumns = Lists.newArrayList(new Column("create_date", Types.DATE), new Column("dept_name", Types.VARCHAR), new Column("user_id", Types.INTEGER), @@ -53,7 +51,8 @@ public StormTopology getTopology() { JdbcMapper mapper = new SimpleJdbcMapper(schemaColumns); JdbcInsertBolt userPersistenceBolt = new JdbcInsertBolt(connectionProvider, mapper) - .withInsertQuery("insert into user (create_date, dept_name, user_id, user_name) values (?,?,?,?)"); + .withInsertQuery("insert into user (create_date, dept_name, user_id, user_name) " + + "values (?,?,?,?)"); // userSpout ==> jdbcBolt TopologyBuilder builder = new TopologyBuilder(); diff --git a/examples/storm-jdbc-examples/src/main/java/org/apache/storm/jdbc/topology/UserPersistenceTridentTopology.java b/examples/storm-jdbc-examples/src/main/java/org/apache/storm/jdbc/topology/UserPersistenceTridentTopology.java index 1c38c351efd..7e3274f5021 100644 --- a/examples/storm-jdbc-examples/src/main/java/org/apache/storm/jdbc/topology/UserPersistenceTridentTopology.java +++ b/examples/storm-jdbc-examples/src/main/java/org/apache/storm/jdbc/topology/UserPersistenceTridentTopology.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -19,9 +19,7 @@ package org.apache.storm.jdbc.topology; import com.google.common.collect.Lists; - import java.sql.Types; - import org.apache.storm.generated.StormTopology; import org.apache.storm.jdbc.common.Column; import org.apache.storm.jdbc.mapper.SimpleJdbcLookupMapper; diff --git a/examples/storm-jms-examples/pom.xml b/examples/storm-jms-examples/pom.xml index 41cf4b9bd71..47f765b7fe2 100644 --- a/examples/storm-jms-examples/pom.xml +++ b/examples/storm-jms-examples/pom.xml @@ -105,6 +105,16 @@ + + org.openrewrite.maven + rewrite-maven-plugin + + + + org.codehaus.gmavenplus + gmavenplus-plugin + + org.apache.maven.plugins maven-checkstyle-plugin diff --git a/examples/storm-jms-examples/src/main/java/org/apache/storm/jms/example/ExampleJmsTopology.java b/examples/storm-jms-examples/src/main/java/org/apache/storm/jms/example/ExampleJmsTopology.java index aea290b752c..c0cd47fcd70 100644 --- a/examples/storm-jms-examples/src/main/java/org/apache/storm/jms/example/ExampleJmsTopology.java +++ b/examples/storm-jms-examples/src/main/java/org/apache/storm/jms/example/ExampleJmsTopology.java @@ -46,6 +46,7 @@ public class ExampleJmsTopology { /** * The main method. + * * @param args takes the topology name as first argument * @throws Exception any expection occuring durch cluster setup or operation */ @@ -72,7 +73,8 @@ public static void main(String[] args) throws Exception { builder.setSpout(JMS_QUEUE_SPOUT, queueSpout, 5); // intermediate bolt, subscribes to jms spout, anchors on tuples, and auto-acks builder.setBolt(INTERMEDIATE_BOLT, - new GenericBolt("INTERMEDIATE_BOLT", true, true, new Fields("json")), 3).shuffleGrouping( + new GenericBolt("INTERMEDIATE_BOLT", true, true, new Fields("json")), 3) + .shuffleGrouping( JMS_QUEUE_SPOUT); // bolt that subscribes to the intermediate bolt, and auto-acks @@ -110,7 +112,8 @@ public Message toMessage(Session session, ITuple input) throws JMSException { builder.setSpout(JMS_TOPIC_SPOUT, topicSpout); - builder.setBolt(ANOTHER_BOLT, new GenericBolt("ANOTHER_BOLT", true, true), 1).shuffleGrouping( + builder.setBolt(ANOTHER_BOLT, new GenericBolt("ANOTHER_BOLT", true, true), 1) + .shuffleGrouping( JMS_TOPIC_SPOUT); Config conf = new Config(); diff --git a/examples/storm-jms-examples/src/main/java/org/apache/storm/jms/example/SpringJmsProvider.java b/examples/storm-jms-examples/src/main/java/org/apache/storm/jms/example/SpringJmsProvider.java index 334a98e3fd7..57f8d41a418 100644 --- a/examples/storm-jms-examples/src/main/java/org/apache/storm/jms/example/SpringJmsProvider.java +++ b/examples/storm-jms-examples/src/main/java/org/apache/storm/jms/example/SpringJmsProvider.java @@ -52,8 +52,10 @@ public class SpringJmsProvider implements JmsProvider { * @param connectionFactoryBean - the JMS connection factory bean name * @param destinationBean - the JMS destination bean name */ - public SpringJmsProvider(String appContextClasspathResource, String connectionFactoryBean, String destinationBean) { - ApplicationContext context = new ClassPathXmlApplicationContext(appContextClasspathResource); + public SpringJmsProvider(String appContextClasspathResource, String connectionFactoryBean, + String destinationBean) { + ApplicationContext context = + new ClassPathXmlApplicationContext(appContextClasspathResource); this.connectionFactory = (ConnectionFactory) context.getBean(connectionFactoryBean); this.destination = (Destination) context.getBean(destinationBean); } diff --git a/examples/storm-kafka-client-examples/pom.xml b/examples/storm-kafka-client-examples/pom.xml index 4700be77148..f29819951e8 100644 --- a/examples/storm-kafka-client-examples/pom.xml +++ b/examples/storm-kafka-client-examples/pom.xml @@ -103,6 +103,16 @@ + + org.openrewrite.maven + rewrite-maven-plugin + + + + org.codehaus.gmavenplus + gmavenplus-plugin + + org.apache.maven.plugins maven-checkstyle-plugin diff --git a/examples/storm-kafka-client-examples/src/main/java/org/apache/storm/kafka/bolt/KafkaProducerTopology.java b/examples/storm-kafka-client-examples/src/main/java/org/apache/storm/kafka/bolt/KafkaProducerTopology.java index f57d98d4d03..f4a1b373659 100644 --- a/examples/storm-kafka-client-examples/src/main/java/org/apache/storm/kafka/bolt/KafkaProducerTopology.java +++ b/examples/storm-kafka-client-examples/src/main/java/org/apache/storm/kafka/bolt/KafkaProducerTopology.java @@ -35,13 +35,14 @@ public class KafkaProducerTopology { * * @param brokerUrl Kafka broker URL * @param topicName Topic to which publish sentences - * @return A Storm topology that produces random UUIDs using a {@link LambdaSpout} and uses a {@link KafkaBolt} to publish the UUIDs to + * @return A Storm topology that produces random UUIDs using a {@link LambdaSpout} and uses a + * {@link KafkaBolt} to publish the UUIDs to * the kafka topic specified */ public static StormTopology newTopology(String brokerUrl, String topicName) { final TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("spout", () -> { - Utils.sleep(1000); //Throttle this spout a bit to avoid maxing out CPU + Utils.sleep(1000); // Throttle this spout a bit to avoid maxing out CPU return UUID.randomUUID().toString(); }); @@ -60,14 +61,18 @@ public static StormTopology newTopology(String brokerUrl, String topicName) { /** * Create the Storm config. - * @return the Storm config for the topology that publishes random UUIDs to Kafka using a Kafka bolt. + * + * @return the Storm config for the topology that publishes random UUIDs to Kafka using a Kafka + * bolt. */ private static Properties newProps(final String brokerUrl, final String topicName) { return new Properties() { { put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, brokerUrl); - put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer"); - put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer"); + put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, + "org.apache.kafka.common.serialization.StringSerializer"); + put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, + "org.apache.kafka.common.serialization.StringSerializer"); put(ProducerConfig.CLIENT_ID_CONFIG, topicName); } }; diff --git a/examples/storm-kafka-client-examples/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutTestBolt.java b/examples/storm-kafka-client-examples/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutTestBolt.java index 2bc348d7b3c..5ef2b9d3b5a 100644 --- a/examples/storm-kafka-client-examples/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutTestBolt.java +++ b/examples/storm-kafka-client-examples/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutTestBolt.java @@ -32,7 +32,8 @@ public class KafkaSpoutTestBolt extends BaseRichBolt { private OutputCollector collector; @Override - public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) { + public void prepare(Map topoConf, TopologyContext context, + OutputCollector collector) { this.collector = collector; } diff --git a/examples/storm-kafka-client-examples/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutTopologyMainNamedTopics.java b/examples/storm-kafka-client-examples/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutTopologyMainNamedTopics.java index beb794871be..e43fbdd79e1 100644 --- a/examples/storm-kafka-client-examples/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutTopologyMainNamedTopics.java +++ b/examples/storm-kafka-client-examples/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutTopologyMainNamedTopics.java @@ -53,13 +53,19 @@ protected void runMain(String[] args) throws Exception { Config tpConf = getConfig(); - // Producers. This is just to get some data in Kafka, normally you would be getting this data from elsewhere - StormSubmitter.submitTopology(TOPIC_0 + "-producer", tpConf, KafkaProducerTopology.newTopology(brokerUrl, TOPIC_0)); - StormSubmitter.submitTopology(TOPIC_1 + "-producer", tpConf, KafkaProducerTopology.newTopology(brokerUrl, TOPIC_1)); - StormSubmitter.submitTopology(TOPIC_2 + "-producer", tpConf, KafkaProducerTopology.newTopology(brokerUrl, TOPIC_2)); + // Producers. This is just to get some data in Kafka, normally you would be getting this + // data from elsewhere + StormSubmitter.submitTopology(TOPIC_0 + "-producer", tpConf, KafkaProducerTopology + .newTopology(brokerUrl, TOPIC_0)); + StormSubmitter.submitTopology(TOPIC_1 + "-producer", tpConf, KafkaProducerTopology + .newTopology(brokerUrl, TOPIC_1)); + StormSubmitter.submitTopology(TOPIC_2 + "-producer", tpConf, KafkaProducerTopology + .newTopology(brokerUrl, TOPIC_2)); - //Consumer. Sets up a topology that reads the given Kafka spouts and logs the received messages - StormSubmitter.submitTopology("storm-kafka-client-spout-test", tpConf, getTopologyKafkaSpout(getKafkaSpoutConfig(brokerUrl))); + // Consumer. Sets up a topology that reads the given Kafka spouts and logs the received + // messages + StormSubmitter.submitTopology("storm-kafka-client-spout-test", tpConf, + getTopologyKafkaSpout(getKafkaSpoutConfig(brokerUrl))); } protected Config getConfig() { @@ -74,7 +80,8 @@ protected StormTopology getTopologyKafkaSpout(KafkaSpoutConfig s tp.setBolt("kafka_bolt", new KafkaSpoutTestBolt()) .shuffleGrouping("kafka_spout", TOPIC_0_1_STREAM) .shuffleGrouping("kafka_spout", TOPIC_2_STREAM); - tp.setBolt("kafka_bolt_1", new KafkaSpoutTestBolt()).shuffleGrouping("kafka_spout", TOPIC_2_STREAM); + tp.setBolt("kafka_bolt_1", new KafkaSpoutTestBolt()).shuffleGrouping("kafka_spout", + TOPIC_2_STREAM); return tp.createTopology(); } diff --git a/examples/storm-kafka-client-examples/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutTopologyMainWildcardTopics.java b/examples/storm-kafka-client-examples/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutTopologyMainWildcardTopics.java index a5740782c94..e6b1b0e9e78 100644 --- a/examples/storm-kafka-client-examples/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutTopologyMainWildcardTopics.java +++ b/examples/storm-kafka-client-examples/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutTopologyMainWildcardTopics.java @@ -28,7 +28,8 @@ import org.apache.storm.tuple.Values; /** - * This example is similar to {@link KafkaSpoutTopologyMainNamedTopics}, but demonstrates subscribing to Kafka topics with a regex. + * This example is similar to {@link KafkaSpoutTopologyMainNamedTopics}, but demonstrates + * subscribing to Kafka topics with a regex. */ public class KafkaSpoutTopologyMainWildcardTopics extends KafkaSpoutTopologyMainNamedTopics { @@ -52,7 +53,8 @@ protected KafkaSpoutConfig getKafkaSpoutConfig(String bootstrapS return KafkaSpoutConfig.builder(bootstrapServers, TOPIC_WILDCARD_PATTERN) .setProp(ConsumerConfig.GROUP_ID_CONFIG, "kafkaSpoutTestGroup") .setRetry(getRetryService()) - .setRecordTranslator((r) -> new Values(r.topic(), r.partition(), r.offset(), r.key(), r.value()), + .setRecordTranslator((r) -> new Values(r.topic(), r.partition(), r.offset(), r.key(), r + .value()), new Fields("topic", "partition", "offset", "key", "value"), STREAM) .setOffsetCommitPeriodMs(10_000) .setFirstPollOffsetStrategy(EARLIEST) diff --git a/examples/storm-kafka-client-examples/src/main/java/org/apache/storm/kafka/trident/TridentKafkaClientTopologyNamedTopics.java b/examples/storm-kafka-client-examples/src/main/java/org/apache/storm/kafka/trident/TridentKafkaClientTopologyNamedTopics.java index 3c92a22d4fe..74624c57133 100644 --- a/examples/storm-kafka-client-examples/src/main/java/org/apache/storm/kafka/trident/TridentKafkaClientTopologyNamedTopics.java +++ b/examples/storm-kafka-client-examples/src/main/java/org/apache/storm/kafka/trident/TridentKafkaClientTopologyNamedTopics.java @@ -40,7 +40,8 @@ /** * This example sets up a few topologies to put random strings in Kafka topics via the KafkaBolt, - * and shows how to set up a Trident topology that reads from some Kafka topics using the KafkaSpout. + * and shows how to set up a Trident topology that reads from some Kafka topics using the + * KafkaSpout. */ public class TridentKafkaClientTopologyNamedTopics { @@ -57,7 +58,8 @@ private KafkaTridentSpoutTransactional newKafkaTridentSpoutTrans return new KafkaTridentSpoutTransactional<>(spoutConfig); } - private static final Func, List> JUST_VALUE_FUNC = new JustValueFunc(); + private static final Func, List> JUST_VALUE_FUNC = + new JustValueFunc(); /** * Needs to be serializable. @@ -93,11 +95,14 @@ protected void run(String[] args) throws AlreadyAliveException, InvalidTopologyE tpConf.setMaxSpoutPending(5); // Producers - StormSubmitter.submitTopology(TOPIC_1 + "-producer", tpConf, KafkaProducerTopology.newTopology(brokerUrl, TOPIC_1)); - StormSubmitter.submitTopology(TOPIC_2 + "-producer", tpConf, KafkaProducerTopology.newTopology(brokerUrl, TOPIC_2)); + StormSubmitter.submitTopology(TOPIC_1 + "-producer", tpConf, KafkaProducerTopology + .newTopology(brokerUrl, TOPIC_1)); + StormSubmitter.submitTopology(TOPIC_2 + "-producer", tpConf, KafkaProducerTopology + .newTopology(brokerUrl, TOPIC_2)); // Consumer KafkaTridentSpoutConfig spoutConfig = newKafkaSpoutConfig(brokerUrl); - ITridentDataSource spout = isOpaque ? newKafkaTridentSpoutOpaque(spoutConfig) : newKafkaTridentSpoutTransactional(spoutConfig); + ITridentDataSource spout = isOpaque + ? newKafkaTridentSpoutOpaque(spoutConfig) : newKafkaTridentSpoutTransactional(spoutConfig); StormSubmitter.submitTopology("topics-consumer", tpConf, TridentKafkaConsumerTopology.newTopology(spout)); } diff --git a/examples/storm-kafka-client-examples/src/main/java/org/apache/storm/kafka/trident/TridentKafkaClientTopologyWildcardTopics.java b/examples/storm-kafka-client-examples/src/main/java/org/apache/storm/kafka/trident/TridentKafkaClientTopologyWildcardTopics.java index f770c75345d..a8ef39b4b88 100644 --- a/examples/storm-kafka-client-examples/src/main/java/org/apache/storm/kafka/trident/TridentKafkaClientTopologyWildcardTopics.java +++ b/examples/storm-kafka-client-examples/src/main/java/org/apache/storm/kafka/trident/TridentKafkaClientTopologyWildcardTopics.java @@ -27,7 +27,8 @@ import org.apache.storm.tuple.Values; /** - * This example is similar to {@link TridentKafkaClientTopologyWildcardTopics}, but demonstrates subscribing to Kafka topics with a regex. + * This example is similar to {@link TridentKafkaClientTopologyWildcardTopics}, but demonstrates + * subscribing to Kafka topics with a regex. */ public class TridentKafkaClientTopologyWildcardTopics extends TridentKafkaClientTopologyNamedTopics { private static final Pattern TOPIC_WILDCARD_PATTERN = Pattern.compile("test-trident(-1)?"); diff --git a/examples/storm-kafka-client-examples/src/main/java/org/apache/storm/kafka/trident/TridentKafkaConsumerTopology.java b/examples/storm-kafka-client-examples/src/main/java/org/apache/storm/kafka/trident/TridentKafkaConsumerTopology.java index 378b7ae56a8..4c33d60d426 100644 --- a/examples/storm-kafka-client-examples/src/main/java/org/apache/storm/kafka/trident/TridentKafkaConsumerTopology.java +++ b/examples/storm-kafka-client-examples/src/main/java/org/apache/storm/kafka/trident/TridentKafkaConsumerTopology.java @@ -32,11 +32,13 @@ public class TridentKafkaConsumerTopology { /** * Creates a new topology that prints inputs to stdout. + * * @param tridentSpout The spout to use */ public static StormTopology newTopology(ITridentDataSource tridentSpout) { final TridentTopology tridentTopology = new TridentTopology(); - final Stream spoutStream = tridentTopology.newStream("spout", tridentSpout).parallelismHint(2); + final Stream spoutStream = tridentTopology.newStream("spout", tridentSpout) + .parallelismHint(2); spoutStream.each(spoutStream.getOutputFields(), new Debug(false)); return tridentTopology.build(); } diff --git a/examples/storm-kafka-client-examples/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutTopologyMainNamedTopicsLocal.java b/examples/storm-kafka-client-examples/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutTopologyMainNamedTopicsLocal.java index 0c82da726b2..7146b47c3a5 100644 --- a/examples/storm-kafka-client-examples/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutTopologyMainNamedTopicsLocal.java +++ b/examples/storm-kafka-client-examples/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutTopologyMainNamedTopicsLocal.java @@ -39,13 +39,19 @@ protected void runExample() throws Exception { Config tpConf = example.getConfig(); LocalCluster localCluster = new LocalCluster(); - // Producers. This is just to get some data in Kafka, normally you would be getting this data from elsewhere - localCluster.submitTopology(TOPIC_0 + "-producer", tpConf, KafkaProducerTopology.newTopology(brokerUrl, TOPIC_0)); - localCluster.submitTopology(TOPIC_1 + "-producer", tpConf, KafkaProducerTopology.newTopology(brokerUrl, TOPIC_1)); - localCluster.submitTopology(TOPIC_2 + "-producer", tpConf, KafkaProducerTopology.newTopology(brokerUrl, TOPIC_2)); + // Producers. This is just to get some data in Kafka, normally you would be getting this + // data from elsewhere + localCluster.submitTopology(TOPIC_0 + "-producer", tpConf, KafkaProducerTopology + .newTopology(brokerUrl, TOPIC_0)); + localCluster.submitTopology(TOPIC_1 + "-producer", tpConf, KafkaProducerTopology + .newTopology(brokerUrl, TOPIC_1)); + localCluster.submitTopology(TOPIC_2 + "-producer", tpConf, KafkaProducerTopology + .newTopology(brokerUrl, TOPIC_2)); - //Consumer. Sets up a topology that reads the given Kafka spouts and logs the received messages - localCluster.submitTopology("storm-kafka-client-spout-test", tpConf, example.getTopologyKafkaSpout(example.getKafkaSpoutConfig(brokerUrl))); + // Consumer. Sets up a topology that reads the given Kafka spouts and logs the received + // messages + localCluster.submitTopology("storm-kafka-client-spout-test", tpConf, example + .getTopologyKafkaSpout(example.getKafkaSpoutConfig(brokerUrl))); stopWaitingForInput(); } diff --git a/examples/storm-loadgen/pom.xml b/examples/storm-loadgen/pom.xml index 0ae6a3b99a3..16080974fd4 100644 --- a/examples/storm-loadgen/pom.xml +++ b/examples/storm-loadgen/pom.xml @@ -122,6 +122,16 @@ + + org.openrewrite.maven + rewrite-maven-plugin + + + + org.codehaus.gmavenplus + gmavenplus-plugin + + org.apache.maven.plugins maven-checkstyle-plugin diff --git a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/CaptureLoad.java b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/CaptureLoad.java index 00a6a32f4e9..9434ed7163b 100644 --- a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/CaptureLoad.java +++ b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/CaptureLoad.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -70,7 +70,8 @@ private static List extractBoltValues(List summaries, if (summaries != null) { for (ExecutorSummary summ : summaries) { if (summ != null && summ.is_set_stats()) { - Map> data = func.apply(summ.get_stats().get_specific().get_bolt()); + Map> data = func.apply(summ.get_stats() + .get_specific().get_bolt()); if (data != null) { List subvalues = data.values().stream() .map((subMap) -> subMap.get(id)) @@ -84,7 +85,8 @@ private static List extractBoltValues(List summaries, return ret; } - static TopologyLoadConf captureTopology(Nimbus.Iface client, TopologySummary topologySummary) throws Exception { + static TopologyLoadConf captureTopology(Nimbus.Iface client, + TopologySummary topologySummary) throws Exception { String topologyName = topologySummary.get_name(); LOG.info("Capturing {}...", topologyName); String topologyId = topologySummary.get_id(); @@ -92,10 +94,11 @@ static TopologyLoadConf captureTopology(Nimbus.Iface client, TopologySummary top TopologyPageInfo tpinfo = client.getTopologyPageInfo(topologyId, ":all-time", false); @SuppressWarnings("checkstyle:VariableDeclarationUsageDistance") StormTopology topo = client.getUserTopology(topologyId); - //Done capturing topology information... + // Done capturing topology information... Map savedTopoConf = new HashMap<>(); - Map topoConf = (Map) JSONValue.parse(client.getTopologyConf(topologyId)); + Map topoConf = (Map) JSONValue.parse(client + .getTopologyConf(topologyId)); for (String key : TopologyLoadConf.IMPORTANT_CONF_KEYS) { Object o = topoConf.get(key); if (o != null) { @@ -103,10 +106,11 @@ static TopologyLoadConf captureTopology(Nimbus.Iface client, TopologySummary top LOG.info("with config {}: {}", key, o); } } - //Lets use the number of actually scheduled workers as a way to bridge RAS and non-RAS + // Lets use the number of actually scheduled workers as a way to bridge RAS and non-RAS int numWorkers = tpinfo.get_num_workers(); if (savedTopoConf.containsKey(Config.TOPOLOGY_WORKERS)) { - numWorkers = Math.max(numWorkers, ((Number) savedTopoConf.get(Config.TOPOLOGY_WORKERS)).intValue()); + numWorkers = Math.max(numWorkers, ((Number) savedTopoConf.get(Config.TOPOLOGY_WORKERS)) + .intValue()); } savedTopoConf.put(Config.TOPOLOGY_WORKERS, numWorkers); @@ -115,7 +119,7 @@ static TopologyLoadConf captureTopology(Nimbus.Iface client, TopologySummary top List inputStreams = new ArrayList<>(); Map outStreams = new HashMap<>(); - //Bolts + // Bolts if (topo.get_bolts() != null) { for (Map.Entry boltSpec : topo.get_bolts().entrySet()) { String boltComp = boltSpec.getKey(); @@ -161,7 +165,8 @@ static TopologyLoadConf captureTopology(Nimbus.Iface client, TopologySummary top if (cpu != null) { bd.withCpuLoad(cpu); } - Double mem = resources.get(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB); + Double mem = resources + .get(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB); if (mem != null) { bd.withMemoryLoad(mem); } @@ -169,7 +174,7 @@ static TopologyLoadConf captureTopology(Nimbus.Iface client, TopologySummary top } } - //Spouts + // Spouts if (topo.get_spouts() != null) { for (Map.Entry spoutSpec : topo.get_spouts().entrySet()) { String spoutComp = spoutSpec.getKey(); @@ -202,7 +207,8 @@ static TopologyLoadConf captureTopology(Nimbus.Iface client, TopologySummary top if (cpu != null) { sd.withCpuLoad(cpu); } - Double mem = resources.get(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB); + Double mem = resources + .get(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB); if (mem != null) { sd.withMemoryLoad(mem); } @@ -210,7 +216,7 @@ static TopologyLoadConf captureTopology(Nimbus.Iface client, TopologySummary top } } - //Stats... + // Stats... Map> byComponent = new HashMap<>(); for (ExecutorSummary executor : info.get_executors()) { String component = executor.get_component_id(); @@ -223,20 +229,21 @@ static TopologyLoadConf captureTopology(Nimbus.Iface client, TopologySummary top } List streams = new ArrayList<>(inputStreams.size()); - //Compute the stats for the different input streams + // Compute the stats for the different input streams for (InputStream.Builder builder : inputStreams) { - GlobalStreamId streamId = new GlobalStreamId(builder.getFromComponent(), builder.getId()); + GlobalStreamId streamId = new GlobalStreamId(builder.getFromComponent(), builder + .getId()); List summaries = byComponent.get(builder.getToComponent()); - //Execute and process latency... + // Execute and process latency... builder.withProcessTime(new NormalDistStats( extractBoltValues(summaries, streamId, BoltStats::get_process_ms_avg))); builder.withExecTime(new NormalDistStats( extractBoltValues(summaries, streamId, BoltStats::get_execute_ms_avg))); - //InputStream is done + // InputStream is done streams.add(builder.build()); } - //There is a bug in some versions that returns 0 for the uptime. + // There is a bug in some versions that returns 0 for the uptime. // To work around it we should get it an alternative (working) way. Map workerToUptime = new HashMap<>(); for (WorkerSummary ws : tpinfo.get_workers()) { @@ -255,18 +262,19 @@ static TopologyLoadConf captureTopology(Nimbus.Iface client, TopologySummary top int uptime = summary.get_uptime_secs(); LOG.debug("UPTIME {}", uptime); if (uptime <= 0) { - //Likely it is because of a bug, so try to get it another way + // Likely it is because of a bug, so try to get it another way String key = summary.get_host() + ":" + summary.get_port(); uptime = workerToUptime.getOrDefault(key, 1); LOG.debug("Getting uptime for worker {}, {}", key, uptime); } - for (Map.Entry> statEntry : summary.get_stats().get_emitted().entrySet()) { + for (Map.Entry> statEntry : summary.get_stats() + .get_emitted().entrySet()) { String timeWindow = statEntry.getKey(); long timeSecs = uptime; try { timeSecs = Long.valueOf(timeWindow); } catch (NumberFormatException e) { - //Ignored... + // Ignored... } timeSecs = Math.min(timeSecs, uptime); Long count = statEntry.getValue().get(id.get_streamId()); @@ -281,7 +289,7 @@ static TopologyLoadConf captureTopology(Nimbus.Iface client, TopologySummary top } builder.withRate(new NormalDistStats(emittedRate)); - //The OutputStream is done + // The OutputStream is done LoadCompConf.Builder comp = boltBuilders.get(id.get_componentId()); if (comp == null) { comp = spoutBuilders.get(id.get_componentId()); @@ -302,6 +310,7 @@ static TopologyLoadConf captureTopology(Nimbus.Iface client, TopologySummary top /** * Main entry point for CaptureLoad command. + * * @param args the arguments to the command * @throws Exception on any error */ @@ -367,13 +376,16 @@ public static void main(String[] args) throws Exception { } } - //ResourceUtils.java is not a available on the classpath to let us parse out the resources we want. + // ResourceUtils.java is not a available on the classpath to let us parse out the resources we + // want. // So we have copied and pasted some of the needed methods here. (with a few changes to logging) - static Map> getBoltsResources(StormTopology topology, Map topologyConf) { + static Map> getBoltsResources(StormTopology topology, Map topologyConf) { Map> boltResources = new HashMap<>(); if (topology.get_bolts() != null) { for (Map.Entry bolt : topology.get_bolts().entrySet()) { - Map topologyResources = parseResources(bolt.getValue().get_common().get_json_conf()); + Map topologyResources = parseResources(bolt.getValue().get_common() + .get_json_conf()); checkInitialization(topologyResources, bolt.getValue().toString(), topologyConf); boltResources.put(bolt.getKey(), topologyResources); } @@ -381,11 +393,13 @@ static Map> getBoltsResources(StormTopology topology return boltResources; } - static Map> getSpoutsResources(StormTopology topology, Map topologyConf) { + static Map> getSpoutsResources(StormTopology topology, Map topologyConf) { Map> spoutResources = new HashMap<>(); if (topology.get_spouts() != null) { for (Map.Entry spout : topology.get_spouts().entrySet()) { - Map topologyResources = parseResources(spout.getValue().get_common().get_json_conf()); + Map topologyResources = parseResources(spout.getValue().get_common() + .get_json_conf()); checkInitialization(topologyResources, spout.getValue().toString(), topologyConf); spoutResources.put(spout.getKey(), topologyResources); } @@ -403,16 +417,21 @@ static Map parseResources(String input) { JSONObject jsonObject = (JSONObject) obj; if (jsonObject.containsKey(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB)) { Double topoMemOnHeap = ObjectReader - .getDouble(jsonObject.get(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB), null); - topologyResources.put(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB, topoMemOnHeap); + .getDouble(jsonObject + .get(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB), null); + topologyResources.put(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB, + topoMemOnHeap); } if (jsonObject.containsKey(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB)) { Double topoMemOffHeap = ObjectReader - .getDouble(jsonObject.get(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB), null); - topologyResources.put(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB, topoMemOffHeap); + .getDouble(jsonObject + .get(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB), null); + topologyResources.put(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB, + topoMemOffHeap); } if (jsonObject.containsKey(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT)) { - Double topoCpu = ObjectReader.getDouble(jsonObject.get(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT), + Double topoCpu = ObjectReader.getDouble(jsonObject + .get(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT), null); topologyResources.put(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT, topoCpu); } @@ -428,12 +447,14 @@ static Map parseResources(String input) { /** * Checks if the topology's resource requirements are initialized. * Will modify topologyResources by adding the appropriate defaults + * * @param topologyResources map of resouces requirements * @param componentId component for which initialization is being conducted * @param topologyConf topology configuration * @throws Exception on any error */ - public static void checkInitialization(Map topologyResources, String componentId, Map topologyConf) { + public static void checkInitialization(Map topologyResources, + String componentId, Map topologyConf) { StringBuilder msgBuilder = new StringBuilder(); for (String resourceName : topologyResources.keySet()) { @@ -448,13 +469,15 @@ public static void checkInitialization(Map topologyResources, St } } - private static String checkInitResource(Map topologyResources, Map topologyConf, String resourceName) { + private static String checkInitResource(Map topologyResources, Map topologyConf, String resourceName) { StringBuilder msgBuilder = new StringBuilder(); if (topologyResources.containsKey(resourceName)) { Double resourceValue = (Double) topologyConf.getOrDefault(resourceName, null); if (resourceValue != null) { topologyResources.put(resourceName, resourceValue); - msgBuilder.append(resourceName.substring(resourceName.lastIndexOf(".")) + " has been set to " + resourceValue); + msgBuilder.append(resourceName.substring(resourceName.lastIndexOf(".")) + + " has been set to " + resourceValue); } } diff --git a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/EstimateThroughput.java b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/EstimateThroughput.java index 2b1570d957e..54386b70e9a 100644 --- a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/EstimateThroughput.java +++ b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/EstimateThroughput.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -44,6 +44,7 @@ public class EstimateThroughput { /** * Main entry point for estimate throughput command. + * * @param args the command line arguments. * @throws Exception on any error. */ @@ -79,7 +80,8 @@ public static void main(String[] args) throws Exception { for (TopologySummary topologySummary : client.getTopologySummaries()) { if (topologyNames.isEmpty() || topologyNames.contains(topologySummary.get_name())) { - TopologyLoadConf capturedConf = CaptureLoad.captureTopology(client, topologySummary); + TopologyLoadConf capturedConf = CaptureLoad.captureTopology(client, + topologySummary); if (capturedConf.looksLikeTrident()) { trident.add(capturedConf); } else { @@ -90,10 +92,12 @@ public static void main(String[] args) throws Exception { System.out.println("TOPOLOGY\tTOTAL MESSAGES/sec\tESTIMATED INPUT MESSAGES/sec"); for (TopologyLoadConf tl : regular) { - System.out.println(tl.name + "\t" + tl.getAllEmittedAggregate() + "\t" + tl.getSpoutEmittedAggregate()); + System.out.println(tl.name + "\t" + tl.getAllEmittedAggregate() + "\t" + tl + .getSpoutEmittedAggregate()); } for (TopologyLoadConf tl : trident) { - System.out.println(tl.name + "\t" + tl.getAllEmittedAggregate() + "\t" + tl.getTridentEstimatedEmittedAggregate()); + System.out.println(tl.name + "\t" + tl.getAllEmittedAggregate() + "\t" + tl + .getTridentEstimatedEmittedAggregate()); } exitStatus = 0; } catch (Exception e) { diff --git a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/ExecAndProcessLatencyEngine.java b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/ExecAndProcessLatencyEngine.java index c2dd81f71a4..6e63ca23f14 100644 --- a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/ExecAndProcessLatencyEngine.java +++ b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/ExecAndProcessLatencyEngine.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -57,19 +57,21 @@ public void prepare() { /** * Sleep for a set number of nano seconds. + * * @param start the start time of the sleep * @param sleepAmount how many nano seconds after start when we should stop. */ public void sleepNano(long start, long sleepAmount) { long endTime = start + sleepAmount; - // A small control algorithm to adjust the amount of time that we sleep to make it more accurate + // A small control algorithm to adjust the amount of time that we sleep to make it more + // accurate long newEnd = endTime - parkOffset.get(); long diff = newEnd - start; - //There are some different levels of accuracy here, and we want to deal with all of them + // There are some different levels of accuracy here, and we want to deal with all of them if (diff <= 1_000) { - //We are done, nothing that short is going to work here + // We are done, nothing that short is going to work here } else if (diff < NANO_IN_MS) { - //Busy wait... + // Busy wait... long sum = 0; while (System.nanoTime() < newEnd) { for (long i = 0; i < 1_000_000; i++) { @@ -77,7 +79,7 @@ public void sleepNano(long start, long sleepAmount) { } } } else { - //More accurate that thread.sleep, but still not great + // More accurate that thread.sleep, but still not great LockSupport.parkNanos(newEnd - System.nanoTime()); } parkOffset.addAndGet((System.nanoTime() - endTime) / 2); @@ -94,16 +96,22 @@ public void sleepUntilNano(long endTime) { /** * Simulate both process and exec times. + * * @param executorIndex the index of this executor. It is used to skew the latencies. * @param startTimeNs when the executor started in nano-seconds. * @param in the metrics for the input stream (or null if you don't want to use them). - * @param r what to run when the process latency is up. Note that this may run on a separate thread after this method call has + * @param r what to run when the process latency is up. Note that this may run on a separate + * thread after this method call has * completed. */ - public void simulateProcessAndExecTime(int executorIndex, long startTimeNs, InputStream in, Runnable r) { - long extraTimeNs = skewedPattern == null ? 0 : toNano(skewedPattern.getExtraSlowness(executorIndex)); - long endExecNs = startTimeNs + extraTimeNs + (in == null ? 0 : ExecAndProcessLatencyEngine.toNano(in.execTime.nextRandom(rand))); - long endProcNs = startTimeNs + extraTimeNs + (in == null ? 0 : ExecAndProcessLatencyEngine.toNano(in.processTime.nextRandom(rand))); + public void simulateProcessAndExecTime(int executorIndex, long startTimeNs, InputStream in, + Runnable r) { + long extraTimeNs = skewedPattern == null ? 0 : toNano(skewedPattern + .getExtraSlowness(executorIndex)); + long endExecNs = startTimeNs + extraTimeNs + (in == null ? 0 : ExecAndProcessLatencyEngine + .toNano(in.execTime.nextRandom(rand))); + long endProcNs = startTimeNs + extraTimeNs + (in == null ? 0 : ExecAndProcessLatencyEngine + .toNano(in.processTime.nextRandom(rand))); if ((endProcNs - 1_000_000) < endExecNs) { sleepUntilNano(endProcNs); diff --git a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/GenLoad.java b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/GenLoad.java index 8ca16eadece..7bfc4ad747f 100644 --- a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/GenLoad.java +++ b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/GenLoad.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -57,6 +57,7 @@ public class GenLoad { /** * Main entry point for GenLoad application. + * * @param args the command line args. * @throws Exception on any error. */ @@ -70,7 +71,8 @@ public static void main(String[] args) throws Exception { .longOpt("test-time") .argName("MINS") .hasArg() - .desc("How long to run the tests for in mins (defaults to " + TEST_EXECUTE_TIME_DEFAULT + ")") + .desc("How long to run the tests for in mins (defaults to " + TEST_EXECUTE_TIME_DEFAULT + + ")") .build()); options.addOption(Option.builder() .longOpt("parallel") @@ -79,8 +81,10 @@ public static void main(String[] args) throws Exception { .desc("How much to scale the topology up or down in parallelism. " + "The new parallelism will round up to the next whole number. " + "If a topology + component is supplied only that component will be scaled. " - + "If topo or component is blank or a '*' all topologies or components matched will be scaled. " - + "Only 1 scaling rule, the most specific, will be applied to a component. Providing a topology name is considered more " + + "If topo or component is blank or a '*' all topologies or components matched " + + "will be scaled. " + + "Only 1 scaling rule, the most specific, will be applied to a component. " + + "Providing a topology name is considered more " + "specific than not providing one." + "(defaults to 1.0 no scaling)") .build()); @@ -90,8 +94,10 @@ public static void main(String[] args) throws Exception { .hasArg() .desc("How much to scale the topology up or down in throughput. " + "If a topology + component is supplied only that component will be scaled. " - + "If topo or component is blank or a '*' all topologies or components matched will be scaled. " - + "Only 1 scaling rule, the most specific, will be applied to a component. Providing a topology name is considered more " + + "If topo or component is blank or a '*' all topologies or components matched " + + "will be scaled. " + + "Only 1 scaling rule, the most specific, will be applied to a component. " + + "Providing a topology name is considered more " + "specific than not providing one." + "(defaults to 1.0 no scaling)") .build()); @@ -103,8 +109,10 @@ public static void main(String[] args) throws Exception { .longOpt("imbalance") .argName("MS(:COUNT)?:TOPO:COMP") .hasArg() - .desc("The number of ms that the first COUNT of TOPO:COMP will wait before processing. This creates an imbalance " - + "that helps test load aware groupings. By default there is no imbalance. If no count is given it defaults to 1") + .desc("The number of ms that the first COUNT of TOPO:COMP will wait before " + + "processing. This creates an imbalance " + + "that helps test load aware groupings. By default there is no imbalance. If no " + + "count is given it defaults to 1") .build()); options.addOption(Option.builder() .longOpt("debug") @@ -129,7 +137,8 @@ public static void main(String[] args) throws Exception { for (String stringParallel : cmd.getOptionValues("parallel")) { Matcher m = MULTI_PATTERN.matcher(stringParallel); if (!m.matches()) { - throw new ParseException("--parallel " + stringParallel + " is not in the format MULTIPLIER(:TOPO:COMP)?"); + throw new ParseException("--parallel " + stringParallel + + " is not in the format MULTIPLIER(:TOPO:COMP)?"); } double parallel = Double.parseDouble(m.group("value")); String topo = m.group("topo"); @@ -151,7 +160,8 @@ public static void main(String[] args) throws Exception { for (String stringThroughput : cmd.getOptionValues("throughput")) { Matcher m = MULTI_PATTERN.matcher(stringThroughput); if (!m.matches()) { - throw new ParseException("--throughput " + stringThroughput + " is not in the format MULTIPLIER(:TOPO:COMP)?"); + throw new ParseException("--throughput " + stringThroughput + + " is not in the format MULTIPLIER(:TOPO:COMP)?"); } double throughput = Double.parseDouble(m.group("value")); String topo = m.group("topo"); @@ -171,13 +181,16 @@ public static void main(String[] args) throws Exception { } if (cmd.hasOption("imbalance")) { for (String stringImbalance : cmd.getOptionValues("imbalance")) { - //We require there to be both a topology and a component in this case, so parse it out as such. + // We require there to be both a topology and a component in this case, so parse + // it out as such. String [] parts = stringImbalance.split(":"); if (parts.length < 3 || parts.length > 4) { - throw new ParseException(stringImbalance + " does not appear to match the expected pattern"); + throw new ParseException(stringImbalance + + " does not appear to match the expected pattern"); } else if (parts.length == 3) { - topoSpecificImbalance.put(parts[1] + ":" + parts[2], SlowExecutorPattern.fromString(parts[0])); - } else { //== 4 + topoSpecificImbalance.put(parts[1] + ":" + parts[2], SlowExecutorPattern + .fromString(parts[0])); + } else { // == 4 topoSpecificImbalance.put(parts[2] + ":" + parts[3], SlowExecutorPattern.fromString(parts[0] + ":" + parts[1])); } @@ -197,11 +210,14 @@ public static void main(String[] args) throws Exception { metrics.put("parallel_adjust", globalParallel); metrics.put("throughput_adjust", globalThroughput); metrics.put("local_or_shuffle", cmd.hasOption("local-or-shuffle")); - metrics.put("topo_parallel", topoSpecificParallel.entrySet().stream().map((entry) -> entry.getValue() + ":" + entry.getKey()) + metrics.put("topo_parallel", topoSpecificParallel.entrySet().stream().map((entry) -> entry + .getValue() + ":" + entry.getKey()) .collect(Collectors.toList())); - metrics.put("topo_throuhgput", topoSpecificThroughput.entrySet().stream().map((entry) -> entry.getValue() + ":" + entry.getKey()) + metrics.put("topo_throuhgput", topoSpecificThroughput.entrySet().stream() + .map((entry) -> entry.getValue() + ":" + entry.getKey()) .collect(Collectors.toList())); - metrics.put("slow_execs", topoSpecificImbalance.entrySet().stream().map((entry) -> entry.getValue() + ":" + entry.getKey()) + metrics.put("slow_execs", topoSpecificImbalance.entrySet().stream().map((entry) -> entry + .getValue() + ":" + entry.getKey()) .collect(Collectors.toList())); Config conf = new Config(); @@ -255,15 +271,16 @@ private static TopologyLoadConf readTopology(String topoFile) throws IOException private static int uniquifier = 0; - private static String parseAndSubmit(TopologyLoadConf tlc, String url) throws IOException, InvalidTopologyException, + private static String parseAndSubmit(TopologyLoadConf tlc, + String url) throws IOException, InvalidTopologyException, AuthorizationException, AlreadyAliveException { - //First we need some configs + // First we need some configs Config conf = new Config(); if (tlc.topoConf != null) { conf.putAll(tlc.topoConf); } - //For some reason on the new code if ackers is null we get 0??? + // For some reason on the new code if ackers is null we get 0??? Object ackers = conf.get(Config.TOPOLOGY_ACKER_EXECUTORS); Object workers = conf.get(Config.TOPOLOGY_WORKERS); if (ackers == null || ((Number) ackers).intValue() <= 0) { @@ -276,17 +293,18 @@ private static String parseAndSubmit(TopologyLoadConf tlc, String url) throws IO conf.registerMetricsConsumer(HttpForwardingMetricsConsumer.class, url, 1); Map workerMetrics = new HashMap<>(); if (!NimbusClient.isLocalOverride()) { - //sigar uses JNI and does not work in local mode + // sigar uses JNI and does not work in local mode workerMetrics.put("CPU", "org.apache.storm.metrics.sigar.CPUMetric"); } conf.put(Config.TOPOLOGY_WORKER_METRICS, workerMetrics); conf.put(Config.TOPOLOGY_BUILTIN_METRICS_BUCKET_SIZE_SECS, 10); - //Lets build a topology. + // Lets build a topology. TopologyBuilder builder = new TopologyBuilder(); for (LoadCompConf spoutConf : tlc.spouts) { System.out.println("ADDING SPOUT " + spoutConf.id); - SpoutDeclarer sd = builder.setSpout(spoutConf.id, new LoadSpout(spoutConf), spoutConf.parallelism); + SpoutDeclarer sd = builder.setSpout(spoutConf.id, new LoadSpout(spoutConf), + spoutConf.parallelism); if (spoutConf.memoryLoad > 0) { sd.setMemoryLoad(spoutConf.memoryLoad); } @@ -317,7 +335,8 @@ private static String parseAndSubmit(TopologyLoadConf tlc, String url) throws IO for (InputStream in : tlc.streams) { BoltDeclarer declarer = boltDeclarers.get(in.toComponent); if (declarer == null) { - throw new IllegalArgumentException("to bolt " + in.toComponent + " does not exist"); + throw new IllegalArgumentException("to bolt " + in.toComponent + + " does not exist"); } LoadBolt lb = bolts.get(in.toComponent); lb.add(in); diff --git a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/GroupingType.java b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/GroupingType.java index a4e0c1af7b5..57a7debc0e9 100644 --- a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/GroupingType.java +++ b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/GroupingType.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -71,6 +71,7 @@ public void assign(BoltDeclarer declarer, InputStream stream) { /** * Parse a String config value and covert it into the enum. + * * @param conf the string config. * @return the parsed grouping type or SHUFFLE if conf is null. * @throws IllegalArgumentException if parsing does not work. diff --git a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/HttpForwardingMetricsConsumer.java b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/HttpForwardingMetricsConsumer.java index f316c76300b..0ca7fbe238e 100644 --- a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/HttpForwardingMetricsConsumer.java +++ b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/HttpForwardingMetricsConsumer.java @@ -37,10 +37,13 @@ * conf.registerMetricsConsumer(HttpForwardingMetricsConsumer.class, "http://example.com:8080/metrics/my-topology/", 1); * ``` * - *

The body of the post is data serialized using {@link org.apache.storm.serialization.KryoValuesSerializer}, with the data passed in - * as a list of `[TaskInfo, Collection<DataPoint>]`. More things may be appended to the end of the list in the future. + *

The body of the post is data serialized using {@link + * org.apache.storm.serialization.KryoValuesSerializer}, with the data passed in + * as a list of `[TaskInfo, Collection<DataPoint>]`. More things may be appended to the end of + * the list in the future. * - *

The values can be deserialized using the org.apache.storm.serialization.KryoValuesDeserializer, and a correct config + classpath. + *

The values can be deserialized using the + * org.apache.storm.serialization.KryoValuesDeserializer, and a correct config + classpath. * *

@see org.apache.storm.serialization.KryoValuesSerializer */ @@ -51,7 +54,8 @@ public class HttpForwardingMetricsConsumer implements IMetricsConsumer { private transient String topologyId; @Override - public void prepare(Map topoConf, Object registrationArgument, TopologyContext context, IErrorReporter errorReporter) { + public void prepare(Map topoConf, Object registrationArgument, + TopologyContext context, IErrorReporter errorReporter) { try { url = new URL((String) registrationArgument); this.errorReporter = errorReporter; @@ -72,7 +76,7 @@ public void handleDataPoints(TaskInfo taskInfo, Collection dataPoints serializer.serializeInto(Arrays.asList(taskInfo, dataPoints, topologyId), out); out.flush(); } - //The connection is not sent unless a response is requested + // The connection is not sent unless a response is requested int response = con.getResponseCode(); } catch (Exception e) { throw new RuntimeException(e); @@ -80,5 +84,5 @@ public void handleDataPoints(TaskInfo taskInfo, Collection dataPoints } @Override - public void cleanup() { } + public void cleanup() {} } diff --git a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/HttpForwardingMetricsServer.java b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/HttpForwardingMetricsServer.java index 74317137fe3..95333ac9da3 100644 --- a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/HttpForwardingMetricsServer.java +++ b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/HttpForwardingMetricsServer.java @@ -55,16 +55,19 @@ protected KryoValuesDeserializer initialValue() { private class MetricsCollectionServlet extends HttpServlet { @Override - protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { + protected void doPost(HttpServletRequest request, + HttpServletResponse response) throws ServletException, IOException { Input in = new Input(request.getInputStream()); List metrics = des.get().deserializeFrom(in); - handle((TaskInfo) metrics.get(0), (Collection) metrics.get(1), (String) metrics.get(2)); + handle((TaskInfo) metrics.get(0), (Collection) metrics.get(1), + (String) metrics.get(2)); response.setStatus(HttpServletResponse.SC_OK); } } /** * Constructor. + * * @param conf the configuration for storm. */ public HttpForwardingMetricsServer(Map conf) { @@ -74,11 +77,13 @@ public HttpForwardingMetricsServer(Map conf) { } } - //This needs to be thread safe - public abstract void handle(TaskInfo taskInfo, Collection dataPoints, String topologyId); + // This needs to be thread safe + public abstract void handle(TaskInfo taskInfo, Collection dataPoints, + String topologyId); /** * Start the server. + * * @param port the port it shuld listen on, or null/<= 0 to pick a free ephemeral port. */ public void serve(Integer port) { @@ -96,7 +101,8 @@ public void serve(Integer port) { this.port = port; url = "http://" + InetAddress.getLocalHost().getHostName() + ":" + this.port + "/"; - ServletContextHandler context = new ServletContextHandler(ServletContextHandler.SESSIONS); + ServletContextHandler context = + new ServletContextHandler(ServletContextHandler.SESSIONS); context.setContextPath("/"); server.setHandler(context); diff --git a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/InputStream.java b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/InputStream.java index 19802d9e0fa..ece22b88dd5 100644 --- a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/InputStream.java +++ b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/InputStream.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -40,19 +40,22 @@ public class InputStream implements Serializable { public final NormalDistStats execTime; public final NormalDistStats processTime; public final GroupingType groupingType; - //Cached GlobalStreamId + // Cached GlobalStreamId private GlobalStreamId gsid = null; /** * Create an output stream from a config. + * * @param conf the config to read from. * @return the read OutputStream. */ public static InputStream fromConf(Map conf) { String component = (String) conf.get("from"); String toComp = (String) conf.get("to"); - NormalDistStats execTime = NormalDistStats.fromConf((Map) conf.get("execTime")); - NormalDistStats processTime = NormalDistStats.fromConf((Map) conf.get("processTime")); + NormalDistStats execTime = NormalDistStats.fromConf((Map) conf + .get("execTime")); + NormalDistStats processTime = NormalDistStats.fromConf((Map) conf + .get("processTime")); Map grouping = (Map) conf.get("grouping"); GroupingType groupingType = GroupingType.fromConf((String) grouping.get("type")); String streamId = (String) grouping.getOrDefault("streamId", "default"); @@ -61,6 +64,7 @@ public static InputStream fromConf(Map conf) { /** * Convert this to a conf. + * * @return the conf. */ public Map toConf() { @@ -142,6 +146,7 @@ public Builder withGroupingType(GroupingType groupingType) { /** * Add the grouping type based off of the thrift Grouping class. + * * @param grouping the Grouping to extract the grouping type from * @return this */ @@ -152,7 +157,7 @@ public Builder withGroupingType(Grouping grouping) { switch (thriftType) { case FIELDS: - //Global Grouping is fields with an empty list + // Global Grouping is fields with an empty list if (grouping.get_fields().isEmpty()) { group = GroupingType.GLOBAL; } else { @@ -172,7 +177,7 @@ public Builder withGroupingType(Grouping grouping) { group = GroupingType.LOCAL_OR_SHUFFLE; break; case CUSTOM_SERIALIZED: - //This might be a partial key grouping.. + // This might be a partial key grouping.. byte[] data = grouping.get_custom_serialized(); try (ByteArrayInputStream bis = new ByteArrayInputStream(data); ObjectInputStream ois = new ObjectInputStream(bis);) { @@ -182,29 +187,33 @@ public Builder withGroupingType(Grouping grouping) { break; } } catch (Exception e) { - //ignored + // ignored } - //Fall through if not supported + // Fall through if not supported default: - LOG.warn("{} is not supported for replay of a topology. Using SHUFFLE", thriftType); + LOG.warn("{} is not supported for replay of a topology. Using SHUFFLE", + thriftType); break; } return withGroupingType(group); } public InputStream build() { - return new InputStream(fromComponent, toComponent, id, execTime, processTime, groupingType); + return new InputStream(fromComponent, toComponent, id, execTime, processTime, + groupingType); } } /** * Create a new input stream to a bolt. + * * @param fromComponent the source component of the stream. * @param id the id of the stream * @param execTime exec time stats * @param processTime process time stats */ - public InputStream(String fromComponent, String toComponent, String id, NormalDistStats execTime, + public InputStream(String fromComponent, String toComponent, String id, + NormalDistStats execTime, NormalDistStats processTime, GroupingType groupingType) { this.fromComponent = fromComponent; this.toComponent = toComponent; @@ -228,6 +237,7 @@ public InputStream(String fromComponent, String toComponent, String id, NormalDi /** * Get the global stream id for this input stream. + * * @return the GlobalStreamId for this input stream. */ public synchronized GlobalStreamId gsid() { @@ -239,25 +249,30 @@ public synchronized GlobalStreamId gsid() { /** * Remap the names of components. + * * @param remappedComponents old name to new name of components. * @param remappedStreams old ID to new ID of streams. * @return a modified version of this with names remapped. */ - public InputStream remap(Map remappedComponents, Map remappedStreams) { + public InputStream remap(Map remappedComponents, Map remappedStreams) { String remapTo = remappedComponents.get(toComponent); String remapFrom = remappedComponents.get(fromComponent); GlobalStreamId remapStreamId = remappedStreams.get(gsid()); - return new InputStream(remapFrom, remapTo, remapStreamId.get_streamId(), execTime, processTime, groupingType); + return new InputStream(remapFrom, remapTo, remapStreamId.get_streamId(), execTime, + processTime, groupingType); } /** * Replace all SHUFFLE groupings with LOCAL_OR_SHUFFLE. + * * @return a modified copy of this */ public InputStream replaceShuffleWithLocalOrShuffle() { if (groupingType != GroupingType.SHUFFLE) { return this; } - return new InputStream(fromComponent, toComponent, id, execTime, processTime, GroupingType.LOCAL_OR_SHUFFLE); + return new InputStream(fromComponent, toComponent, id, execTime, processTime, + GroupingType.LOCAL_OR_SHUFFLE); } } diff --git a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/LoadBolt.java b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/LoadBolt.java index 06f1e5a25ac..388c759ec9e 100644 --- a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/LoadBolt.java +++ b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/LoadBolt.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -58,7 +58,8 @@ public void add(InputStream inputStream) { } @Override - public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) { + public void prepare(Map topoConf, TopologyContext context, + OutputCollector collector) { outputStreams = Collections.unmodifiableList(outputStreamStats.stream() .map((ss) -> new OutputStreamEngine(ss)).collect(Collectors.toList())); this.collector = collector; diff --git a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/LoadCompConf.java b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/LoadCompConf.java index 33460a8d882..f2b490524ae 100644 --- a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/LoadCompConf.java +++ b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/LoadCompConf.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -39,6 +39,7 @@ public class LoadCompConf { /** * Parse the LoadCompConf from a config Map. + * * @param conf the map holding the config for a LoadCompConf. * @return the parsed object. */ @@ -57,13 +58,15 @@ public static LoadCompConf fromConf(Map conf) { SlowExecutorPattern slp = null; if (conf.containsKey("slowExecutorPattern")) { - slp = SlowExecutorPattern.fromConf((Map) conf.get("slowExecutorPattern")); + slp = SlowExecutorPattern.fromConf((Map) conf + .get("slowExecutorPattern")); } return new LoadCompConf(id, parallelism, streams, memoryMb, cpuPercent, slp); } /** * Build a config map for this object. + * * @return the config map. */ public Map toConf() { @@ -92,23 +95,28 @@ public Map toConf() { /** * Chenge the name of components and streams according to the parameters passed in. + * * @param remappedComponents original component name to new component name. * @param remappedStreams original stream id to new stream id. * @return a copy of this with the values remapped. */ - public LoadCompConf remap(Map remappedComponents, Map remappedStreams) { + public LoadCompConf remap(Map remappedComponents, Map remappedStreams) { String remappedId = remappedComponents.get(id); List remappedOutStreams = (streams == null) ? null : streams.stream() .map((orig) -> orig.remap(id, remappedStreams)) .collect(Collectors.toList()); - return new LoadCompConf(remappedId, parallelism, remappedOutStreams, cpuLoad, memoryLoad, slp); + return new LoadCompConf(remappedId, parallelism, remappedOutStreams, cpuLoad, memoryLoad, + slp); } /** * Scale the parallelism of this component by v. The aggregate throughput will be the same. - * The parallelism will be rounded up to the next largest whole number. Parallelism will always be at least 1. + * The parallelism will be rounded up to the next largest whole number. Parallelism will always + * be at least 1. + * * @param v 1.0 is not change 0.5 is drop the parallelism by half. * @return a copy of this with the parallelism adjusted. */ @@ -117,24 +125,29 @@ public LoadCompConf scaleParallel(double v) { } /** - * Set the parallelism of this component, and adjust the throughput so in aggregate it stays the same. + * Set the parallelism of this component, and adjust the throughput so in aggregate it stays the + * same. + * * @param newParallelism the new parallelism to set. * @return a copy of this with the adjustments made. */ public LoadCompConf setParallel(int newParallelism) { - //We need to adjust the throughput accordingly (so that it stays the same in aggregate) + // We need to adjust the throughput accordingly (so that it stays the same in aggregate) double throughputAdjustment = ((double) parallelism) / newParallelism; - return new LoadCompConf(id, newParallelism, streams, cpuLoad, memoryLoad, slp).scaleThroughput(throughputAdjustment); + return new LoadCompConf(id, newParallelism, streams, cpuLoad, memoryLoad, slp) + .scaleThroughput(throughputAdjustment); } /** * Scale the throughput of this component. + * * @param v 1.0 is unchanged 0.5 will cut the throughput in half. * @return a copy of this with the adjustments made. */ public LoadCompConf scaleThroughput(double v) { if (streams != null) { - List newStreams = streams.stream().map((s) -> s.scaleThroughput(v)).collect(Collectors.toList()); + List newStreams = streams.stream().map((s) -> s.scaleThroughput(v)) + .collect(Collectors.toList()); return new LoadCompConf(id, parallelism, newStreams, cpuLoad, memoryLoad, slp); } else { return this; @@ -143,6 +156,7 @@ public LoadCompConf scaleThroughput(double v) { /** * Override the SlowExecutorPattern with a new one. + * * @param slp the new pattern or null if you don't want it to change * @return a copy of this with the adjustments made. */ @@ -156,6 +170,7 @@ public LoadCompConf overrideSlowExecutorPattern(SlowExecutorPattern slp) { /** * Compute the total amount of all messages emitted in all streams per second. + * * @return the sum of all messages emitted per second. */ public double getAllEmittedAggregate() { @@ -202,6 +217,7 @@ public List getStreams() { /** * Add in a single OutputStream to this component. + * * @param stream the stream to add * @return this */ @@ -240,11 +256,13 @@ public LoadCompConf build() { /** * Create a new LoadCompConf with the given values. + * * @param id the id of the component. * @param parallelism tha parallelism of the component. * @param streams the output streams of the component. */ - public LoadCompConf(String id, int parallelism, List streams, double cpuLoad, double memoryLoad, + public LoadCompConf(String id, int parallelism, List streams, double cpuLoad, + double memoryLoad, SlowExecutorPattern slp) { this.id = id; if (id == null) { diff --git a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/LoadMetricsServer.java b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/LoadMetricsServer.java index f02c407093a..b3ccd06ac9a 100644 --- a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/LoadMetricsServer.java +++ b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/LoadMetricsServer.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -113,12 +113,14 @@ public static class Measurements { /** * Constructor. + * * @param histo latency histogram. * @param userMs user CPU in ms. * @param sysMs system CPU in ms. * @param gcMs GC CPU in ms. */ - public Measurements(long uptimeSecs, long acked, long timeWindow, long failed, Histogram histo, + public Measurements(long uptimeSecs, long acked, long timeWindow, long failed, + Histogram histo, double userMs, double sysMs, double gcMs, long memBytes, Set topologyIds, long workers, long executors, long hosts, Map congested, long skippedMaxSpoutMs, double uiCompleteLatency) { @@ -164,6 +166,7 @@ public Measurements() { /** * Add other to this. + * * @param other meaurements to add in. */ public void add(Measurements other) { @@ -319,7 +322,7 @@ private static class NoCloseOutputStream extends FilterOutputStream { @Override public void close() { - //NOOP on purpose + // NOOP on purpose } } @@ -332,7 +335,8 @@ abstract static class FileReporter implements MetricResultsReporter { this(null, Collections.emptyMap(), allExtractors); } - FileReporter(String path, Map query, Map allExtractors) + FileReporter(String path, Map query, Map allExtractors) throws FileNotFoundException { boolean append = Boolean.parseBoolean(query.getOrDefault("append", "false")); boolean tee = Boolean.parseBoolean(query.getOrDefault("tee", "false")); @@ -356,14 +360,14 @@ abstract static class FileReporter implements MetricResultsReporter { includesSysOutOrError = true; } this.out = new PrintStream(out); - //Copy it in case we want to modify it + // Copy it in case we want to modify it this.allExtractors = new LinkedHashMap<>(allExtractors); this.includesSysOutOrError = includesSysOutOrError; } @Override public void start() { - //NOOP + // NOOP } @Override @@ -420,7 +424,7 @@ public void finish(List allTime) throws Exception { private static final Map NAMED_EXTRACTORS; static { - //Perhaps there is a better way to do this??? + // Perhaps there is a better way to do this??? LinkedHashMap tmp = new LinkedHashMap<>(); tmp.put("start_time", new MetricExtractor((m, unit) -> m.startTime(), "s")); tmp.put("end_time", new MetricExtractor((m, unit) -> m.endTime(), "s")); @@ -429,7 +433,8 @@ public void finish(List allTime) throws Exception { tmp.put("99%ile", new MetricExtractor((m, unit) -> m.getLatencyAtPercentile(99.0, unit))); tmp.put("99.9%ile", new MetricExtractor((m, unit) -> m.getLatencyAtPercentile(99.9, unit))); tmp.put("cores", new MetricExtractor( - (m, unit) -> (m.getSysTime(TimeUnit.SECONDS) + m.getUserTime(TimeUnit.SECONDS)) / m.getTimeWindow(), + (m, unit) -> (m.getSysTime(TimeUnit.SECONDS) + m.getUserTime(TimeUnit.SECONDS)) / m + .getTimeWindow(), "")); tmp.put("mem", new MetricExtractor((m, unit) -> m.getMemMb(), "MB")); tmp.put("failed", new MetricExtractor((m, unit) -> m.getFailed(), "")); @@ -457,9 +462,12 @@ public void finish(List allTime) throws Exception { + " " + System.getProperty("java.version"), "")); tmp.put("os_arch", new MetricExtractor((m, unit) -> System.getProperty("os.arch"), "")); tmp.put("os_name", new MetricExtractor((m, unit) -> System.getProperty("os.name"), "")); - tmp.put("os_version", new MetricExtractor((m, unit) -> System.getProperty("os.version"), "")); - tmp.put("config_override", new MetricExtractor((m, unit) -> Utils.readCommandLineOpts(), "")); - tmp.put("ui_complete_latency", new MetricExtractor((m, unit) -> m.getUiCompleteLatency(unit))); + tmp.put("os_version", new MetricExtractor((m, unit) -> System.getProperty("os.version"), + "")); + tmp.put("config_override", new MetricExtractor((m, unit) -> Utils.readCommandLineOpts(), + "")); + tmp.put("ui_complete_latency", new MetricExtractor((m, unit) -> m + .getUiCompleteLatency(unit))); NAMED_EXTRACTORS = Collections.unmodifiableMap(tmp); } @@ -504,20 +512,24 @@ abstract static class ColumnsFileReporter extends FileReporter { protected final int precision; protected String doubleFormat; - ColumnsFileReporter(String path, Map query, Map extractorsMap) + ColumnsFileReporter(String path, Map query, Map extractorsMap) throws FileNotFoundException { this(path, query, extractorsMap, null); } - ColumnsFileReporter(String path, Map query, Map extractorsMap, + ColumnsFileReporter(String path, Map query, Map extractorsMap, String defaultPreceision) throws FileNotFoundException { super(path, query, extractorsMap); targetUnit = UNIT_MAP.get(query.getOrDefault("time", "MILLISECONDS").toUpperCase()); if (targetUnit == null) { - throw new IllegalArgumentException(query.get("time") + " is not a supported time unit"); + throw new IllegalArgumentException(query.get("time") + + " is not a supported time unit"); } if (query.containsKey("columns")) { - List extractors = handleExtractorCleanup(Arrays.asList(query.get("columns").split("\\s*,\\s*"))); + List extractors = handleExtractorCleanup(Arrays.asList(query.get("columns") + .split("\\s*,\\s*"))); HashSet notFound = new HashSet<>(extractors); notFound.removeAll(allExtractors.keySet()); @@ -526,17 +538,19 @@ abstract static class ColumnsFileReporter extends FileReporter { } this.extractors = extractors; } else { - //Wrapping it makes it mutable + // Wrapping it makes it mutable extractors = new ArrayList<>(Arrays.asList("start_time", "end_time", "rate", "mean", "99%ile", "99.9%ile", "cores", "mem", "failed", "ids", "congested")); } if (query.containsKey("extraColumns")) { List moreExtractors = - handleExtractorCleanup(Arrays.asList(query.get("extraColumns").split("\\s*,\\s*"))); + handleExtractorCleanup(Arrays.asList(query.get("extraColumns") + .split("\\s*,\\s*"))); for (String extractor : moreExtractors) { if (!allExtractors.containsKey(extractor)) { - throw new IllegalArgumentException(extractor + " is not a supported column"); + throw new IllegalArgumentException(extractor + + " is not a supported column"); } if (!extractors.contains(extractor)) { extractors.add(extractor); @@ -564,8 +578,10 @@ protected List handleExtractorCleanup(List orig) { allExtractors.put(extractor, new MetricExtractor((m, t) -> confValue, "")); ret.add(extractor); } else if (extractor.endsWith("%ile")) { - double number = Double.valueOf(extractor.substring(0, extractor.length() - "%ile".length())); - allExtractors.put(extractor, new MetricExtractor((m, t) -> m.getLatencyAtPercentile(number, t))); + double number = Double.valueOf(extractor.substring(0, extractor + .length() - "%ile".length())); + allExtractors.put(extractor, new MetricExtractor((m, t) -> m + .getLatencyAtPercentile(number, t))); ret.add(extractor); } else if ("*".equals(extractor)) { ret.addAll(allExtractors.keySet()); @@ -585,21 +601,23 @@ protected String format(Object o) { } } - static class FixedWidthReporter extends ColumnsFileReporter { public final String longFormat; public final String stringFormat; - FixedWidthReporter(String path, Map query, Map extractorsMap) + FixedWidthReporter(String path, Map query, Map extractorsMap) throws FileNotFoundException { super(path, query, extractorsMap, "3"); - int columnWidth = Integer.parseInt(query.getOrDefault("columnWidth", "15")) - 1; //Always have a space in between + int columnWidth = Integer.parseInt(query.getOrDefault("columnWidth", + "15")) - 1; // Always have a space in between doubleFormat = "%," + columnWidth + "." + precision + "f"; longFormat = "%," + columnWidth + "d"; stringFormat = "%" + columnWidth + "s"; } - FixedWidthReporter(Map allExtractors) throws FileNotFoundException { + FixedWidthReporter(Map allExtractors) throws FileNotFoundException { this(null, Collections.emptyMap(), allExtractors); } @@ -652,7 +670,8 @@ public void reportWindow(Measurements m, List allTime) { static class SepValReporter extends ColumnsFileReporter { private final String separator; - SepValReporter(String separator, String path, Map query, Map extractorsMap) + SepValReporter(String separator, String path, Map query, Map extractorsMap) throws FileNotFoundException { super(path, query, extractorsMap); this.separator = separator; @@ -707,13 +726,15 @@ static class LegacyReporter extends FileReporter { targetUnitOverride = null; } - LegacyReporter(String path, Map query, Map allExtractors) + LegacyReporter(String path, Map query, Map allExtractors) throws FileNotFoundException { super(path, query, allExtractors); if (query.containsKey("time")) { targetUnitOverride = UNIT_MAP.get(query.get("time").toUpperCase()); if (targetUnitOverride == null) { - throw new IllegalArgumentException(query.get("time") + " is not a supported time unit"); + throw new IllegalArgumentException(query.get("time") + + " is not a supported time unit"); } } else { targetUnitOverride = null; @@ -749,10 +770,11 @@ public void reportWindow(Measurements m, List allTime) { /** * Add Command line options for configuring the output of this. + * * @param options command line options to update */ public static void addCommandLineOptions(Options options) { - //We want to be able to select the measurement interval + // We want to be able to select the measurement interval // reporting window (We don't need 3 different reports) // We want to be able to specify format (and configs specific to the format) // With perhaps defaults overall @@ -760,7 +782,8 @@ public static void addCommandLineOptions(Options options) { .longOpt("report-interval") .hasArg() .argName("SECS") - .desc("How long in between reported metrics. Will be rounded up to the next 10 sec boundary.\n" + .desc("How long in between reported metrics. Will be rounded up to the next 10 sec " + + "boundary.\n" + "default " + DEFAULT_REPORT_INTERVAL) .build()); @@ -768,7 +791,8 @@ public static void addCommandLineOptions(Options options) { .longOpt("report-window") .hasArg() .argName("SECS") - .desc("How long of a rolling window should be in each report. Will be rounded up to the next report interval boundary.\n" + .desc("How long of a rolling window should be in each report. Will be rounded up to " + + "the next report interval boundary.\n" + "default " + DEFAULT_WINDOW_INTERVAL) .build()); @@ -781,7 +805,8 @@ public static void addCommandLineOptions(Options options) { + "LEGACY - (write things out in the legacy format)\n" + "TSV - tab separated values\n" + "CSV - comma separated values\n" - + "PATH and OPTIONS are each optional but must be marked with a ':' or '?' separator respectively.") + + "PATH and OPTIONS are each optional but must be marked with a ':' or '?' " + + "separator respectively.") .build()); } @@ -798,7 +823,8 @@ public static void addCommandLineOptions(Options options) { private final AtomicLong gcMs = new AtomicLong(0); private final AtomicLong skippedMaxSpoutMs = new AtomicLong(0); private final ConcurrentHashMap memoryBytes = new ConcurrentHashMap<>(); - private final AtomicReference> congested = new AtomicReference<>(new ConcurrentHashMap<>()); + private final AtomicReference> congested = + new AtomicReference<>(new ConcurrentHashMap<>()); private final List reporters; private long prevAcked = 0; private long prevFailed = 0; @@ -808,7 +834,8 @@ public static void addCommandLineOptions(Options options) { private final LinkedList allCombined = new LinkedList<>(); - LoadMetricsServer(Map conf, CommandLine commandLine, Map parameterMetrics) throws URISyntaxException, + LoadMetricsServer(Map conf, CommandLine commandLine, Map parameterMetrics) throws URISyntaxException, FileNotFoundException { super(conf); Map allExtractors = new LinkedHashMap<>(NAMED_EXTRACTORS); @@ -829,7 +856,8 @@ public static void addCommandLineOptions(Options options) { for (String reporterString : commandLine.getOptionValues("reporter")) { Matcher m = REPORTER_PATTERN.matcher(reporterString); if (!m.matches()) { - throw new IllegalArgumentException(reporterString + " does not look like it is a reporter"); + throw new IllegalArgumentException(reporterString + + " does not look like it is a reporter"); } String type = m.group("type"); String path = m.group("path"); @@ -898,12 +926,14 @@ private void finishMetricsOutput() throws Exception { /** * Monitor the list of topologies for the given time frame. + * * @param execTimeMins how long to monitor for * @param client the client to use when monitoring * @param topoNames the names of the topologies to monitor * @throws Exception on any error */ - public void monitorFor(double execTimeMins, Nimbus.Iface client, Collection topoNames) throws Exception { + public void monitorFor(double execTimeMins, Nimbus.Iface client, + Collection topoNames) throws Exception { startMetricsOutput(); long iterations = (long) ((execTimeMins * 60) / reportIntervalSecs); for (int i = 0; i < iterations; i++) { @@ -952,7 +982,8 @@ private void outputMetrics(Nimbus.Iface client, Collection names) throws } } } - Double latency = tpi.get_topology_stats().get_window_to_complete_latencies_ms().get(":all-time"); + Double latency = tpi.get_topology_stats().get_window_to_complete_latencies_ms() + .get(":all-time"); Long latAcked = tpi.get_topology_stats().get_window_to_acked().get(":all-time"); if (latency != null && latAcked != null) { totalLatCount += latAcked; @@ -980,8 +1011,10 @@ private void outputMetrics(Nimbus.Iface client, Collection names) throws long skippedMaxSpout = skippedMaxSpoutMs.getAndSet(0); long memBytes = readMemory(); - allCombined.add(new Measurements(uptime, ackedThisTime, thisTime, failedThisTime, copy, user, sys, gc, memBytes, - ids, workers.size(), executors, hosts.size(), congested.getAndSet(new ConcurrentHashMap<>()), skippedMaxSpout, + allCombined.add(new Measurements(uptime, ackedThisTime, thisTime, failedThisTime, copy, + user, sys, gc, memBytes, + ids, workers.size(), executors, hosts.size(), congested + .getAndSet(new ConcurrentHashMap<>()), skippedMaxSpout, totalLatMs / totalLatCount)); Measurements inWindow = Measurements.combine(allCombined, null, windowLength); for (MetricResultsReporter reporter : reporters) { @@ -991,7 +1024,8 @@ private void outputMetrics(Nimbus.Iface client, Collection names) throws @Override @SuppressWarnings("unchecked") - public void handle(IMetricsConsumer.TaskInfo taskInfo, Collection dataPoints, String topologyId) { + public void handle(IMetricsConsumer.TaskInfo taskInfo, + Collection dataPoints, String topologyId) { String worker = taskInfo.srcWorkerHost + ":" + taskInfo.srcWorkerPort; for (IMetricsConsumer.DataPoint dp : dataPoints) { if (dp.name.startsWith("comp-lat-histo") && dp.value instanceof Histogram) { @@ -1045,7 +1079,8 @@ public void handle(IMetricsConsumer.TaskInfo taskInfo, Collection= 0.8) { congested.get().put( topologyId + ":" + taskInfo.srcComponentId + ":" + taskInfo.srcTaskId, diff --git a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/LoadSpout.java b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/LoadSpout.java index 1fc006eb800..6db5e9e6b37 100644 --- a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/LoadSpout.java +++ b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/LoadSpout.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -44,7 +44,7 @@ private static class OutputStreamEngineWithHisto extends OutputStreamEngine { OutputStreamEngineWithHisto(OutputStream stats, TopologyContext context) { super(stats); histogram = new HistogramMetric(3600000000000L, 3); - //TODO: perhaps we can adjust the frequency later... + // TODO: perhaps we can adjust the frequency later... context.registerMetric("comp-lat-histo-" + stats.id, histogram, 10); } } @@ -70,13 +70,14 @@ public void done() { private final List streamStats; private List streams; private SpoutOutputCollector collector; - //This is an attempt to give all of the streams an equal opportunity to emit something. + // This is an attempt to give all of the streams an equal opportunity to emit something. private long nextStreamCounter = 0; private final int numStreams; private final Queue replays = new ArrayDeque<>(); /** * Create a simple load spout with just a set rate per second on the default stream. + * * @param ratePerSecond the rate to send messages at. */ public LoadSpout(double ratePerSecond) { @@ -94,9 +95,11 @@ public LoadSpout(LoadCompConf conf) { } @Override - public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { + public void open(Map conf, TopologyContext context, + SpoutOutputCollector collector) { streams = Collections.unmodifiableList(streamStats.stream() - .map((ss) -> new OutputStreamEngineWithHisto(ss, context)).collect(Collectors.toList())); + .map((ss) -> new OutputStreamEngineWithHisto(ss, context)).collect(Collectors + .toList())); this.collector = collector; } diff --git a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/NormalDistStats.java b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/NormalDistStats.java index 12dc6c5efc2..1987e3ae84f 100644 --- a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/NormalDistStats.java +++ b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/NormalDistStats.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -40,6 +40,7 @@ public class NormalDistStats implements Serializable { /** * Read the stats from a config. + * * @param conf the config. * @return the corresponding stats. */ @@ -49,6 +50,7 @@ public static NormalDistStats fromConf(Map conf) { /** * Read the stats from a config. + * * @param conf the config. * @param def the default mean. * @return the corresponding stats. @@ -66,6 +68,7 @@ public static NormalDistStats fromConf(Map conf, Double def) { /** * Return this as a config. + * * @return the config version of this. */ public Map toConf() { @@ -78,11 +81,13 @@ public Map toConf() { } /** - * Create an instance of this from a list of values. The metrics will be computed from the values. + * Create an instance of this from a list of values. The metrics will be computed from the + * values. + * * @param values the values to compute metrics from. */ public NormalDistStats(List values) { - //Compute the stats for these and save them + // Compute the stats for these and save them double min = values.isEmpty() ? 0.0 : values.get(0); double max = values.isEmpty() ? 0.0 : values.get(0); double sum = 0.0; @@ -110,6 +115,7 @@ public NormalDistStats(List values) { /** * A Constructor for the pre computed stats. + * * @param mean the mean of the values. * @param stddev the standard deviation of the values. * @param min the min of the values. @@ -124,6 +130,7 @@ public NormalDistStats(double mean, double stddev, double min, double max) { /** * Generate a random number that follows the statistical distribution. + * * @param rand the random number generator to use * @return the next number that should follow the statistical distribution. */ @@ -137,8 +144,10 @@ public String toString() { } /** - * Scale the stats by v. This is not scaling everything proportionally. We don't want the stddev to increase + * Scale the stats by v. This is not scaling everything proportionally. We don't want the stddev + * to increase * so instead we scale the mean and shift everything up or down by the same amount. + * * @param v the amount to scale by 1.0 is nothing 0.5 is half. * @return a copy of this with the needed adjustments. */ diff --git a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/OutputStream.java b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/OutputStream.java index 33f894f1200..7acea95d098 100644 --- a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/OutputStream.java +++ b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/OutputStream.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -27,13 +27,14 @@ * A set of measurements about a stream so we can statistically reproduce it. */ public class OutputStream implements Serializable { - //The global stream id is this + the from component it must be a part of. + // The global stream id is this + the from component it must be a part of. public final String id; public final NormalDistStats rate; public final boolean areKeysSkewed; /** * Create an output stream from a config. + * * @param conf the config to read from. * @return the read OutputStream. */ @@ -46,6 +47,7 @@ public static OutputStream fromConf(Map conf) { /** * Convert this to a conf. + * * @return the conf. */ public Map toConf() { @@ -104,6 +106,7 @@ public OutputStream build() { /** * Create a new stream with stats. + * * @param id the id of the stream * @param rate the rate of tuples being emitted on this stream * @param areKeysSkewed true if keys are skewed else false. For skewed keys diff --git a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/OutputStreamEngine.java b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/OutputStreamEngine.java index d8b52034ec2..5830ece1d4c 100644 --- a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/OutputStreamEngine.java +++ b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/OutputStreamEngine.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -23,8 +23,8 @@ /** * Provides an API to simulate the output of a stream. - *

- * Right now it is just rate, but in the future we expect to do data skew as well... + * + *

Right now it is just rate, but in the future we expect to do data skew as well... *

*/ public class OutputStreamEngine { @@ -33,7 +33,8 @@ public class OutputStreamEngine { private static final String[] KEYS = new String[2048]; static { - //We get a new random number and seed it to make sure that runs are consistent where possible. + // We get a new random number and seed it to make sure that runs are consistent where + // possible. Random r = new Random(KEYS.length); for (int i = 0; i < KEYS.length; i++) { KEYS[i] = String.valueOf(r.nextDouble()); @@ -51,13 +52,14 @@ public class OutputStreamEngine { /** * Create an engine that can simulate the given stats. + * * @param stats the stats to follow */ public OutputStreamEngine(OutputStream stats) { this.stats = stats; rand = ThreadLocalRandom.current(); selectNewRate(); - //Start emitting right now + // Start emitting right now nextEmitTime = System.nanoTime(); nextRateRandomizeTime = nextEmitTime + UPDATE_RATE_PERIOD_NS; emitsLeft = emitAmount; @@ -70,7 +72,7 @@ private void selectNewRate() { periodNano = Math.max(1, (long) (NANO_PER_SEC / ratePerSecond)); emitAmount = Math.max(1, (long) ((ratePerSecond / NANO_PER_SEC) * periodNano)); } else { - //if it is is 0 or less it really is 1 per 10 seconds. + // if it is is 0 or less it really is 1 per 10 seconds. periodNano = (long) NANO_PER_SEC * 10; emitAmount = 1; } @@ -78,6 +80,7 @@ private void selectNewRate() { /** * Should we emit or not. + * * @return the start time of the message, or null of nothing should be emitted. */ public Long shouldEmit() { @@ -88,7 +91,7 @@ public Long shouldEmit() { } if (nextRateRandomizeTime <= time) { - //Once every UPDATE_RATE_PERIOD_NS + // Once every UPDATE_RATE_PERIOD_NS selectNewRate(); nextRateRandomizeTime = nextEmitTime + UPDATE_RATE_PERIOD_NS; } @@ -102,14 +105,17 @@ public Long shouldEmit() { /** * Get the next key to emit. + * * @return the key that should be emitted. */ public String nextKey() { int keyIndex; if (stats.areKeysSkewed) { - //We set the stddev of the skewed keys to be 1/5 of the length, but then we use the absolute value + // We set the stddev of the skewed keys to be 1/5 of the length, but then we use the + // absolute value // of that so everything is skewed towards 0 - keyIndex = Math.min(KEYS.length - 1, Math.abs((int) (rand.nextGaussian() * KEYS.length / 5))); + keyIndex = Math.min(KEYS.length - 1, Math.abs((int) (rand + .nextGaussian() * KEYS.length / 5))); } else { keyIndex = rand.nextInt(KEYS.length); } diff --git a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/ScopedTopologySet.java b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/ScopedTopologySet.java index f7e79121453..20cde608aee 100644 --- a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/ScopedTopologySet.java +++ b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/ScopedTopologySet.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -45,6 +45,7 @@ public class ScopedTopologySet extends HashSet implements AutoCloseable /** * Constructor. + * * @param client the client used to kill the topologies when this exist. */ public ScopedTopologySet(Nimbus.Iface client) { diff --git a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/SlowExecutorPattern.java b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/SlowExecutorPattern.java index d2c3ac502c3..cd9d7a966a5 100644 --- a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/SlowExecutorPattern.java +++ b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/SlowExecutorPattern.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -26,22 +26,26 @@ import org.apache.storm.utils.ObjectReader; /** - * A repeating pattern of skewedness in processing times. This is used to simulate an executor that slows down. + * A repeating pattern of skewedness in processing times. This is used to simulate an executor that + * slows down. */ public class SlowExecutorPattern implements Serializable { - private static final Pattern PARSER = Pattern.compile("\\s*(?[^:]+)\\s*(?::\\s*(?[0-9]+))?\\s*"); + private static final Pattern PARSER = Pattern + .compile("\\s*(?[^:]+)\\s*(?::\\s*(?[0-9]+))?\\s*"); public final double maxSlownessMs; public final int count; /** * Parses a string (command line) representation of "<SLOWNESS>(:<COUNT>)?". + * * @param strRepresentation the string representation to parse * @return the corresponding SlowExecutorPattern. */ public static SlowExecutorPattern fromString(String strRepresentation) { Matcher m = PARSER.matcher(strRepresentation); if (!m.matches()) { - throw new IllegalArgumentException(strRepresentation + " is not in the form (:)?"); + throw new IllegalArgumentException(strRepresentation + + " is not in the form (:)?"); } double slownessMs = Double.valueOf(m.group("slowness")); String c = m.group("count"); @@ -51,6 +55,7 @@ public static SlowExecutorPattern fromString(String strRepresentation) { /** * Creates a SlowExecutorPattern from a Map config. + * * @param conf the conf to parse. * @return the corresponding SlowExecutorPattern. */ @@ -62,6 +67,7 @@ public static SlowExecutorPattern fromConf(Map conf) { /** * Convert this to a Config map. + * * @return the corresponding Config map to this. */ public Map toConf() { diff --git a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/ThroughputVsLatency.java b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/ThroughputVsLatency.java index 17aa8cf9441..b730ce6f395 100644 --- a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/ThroughputVsLatency.java +++ b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/ThroughputVsLatency.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -69,6 +69,7 @@ public static class FastRandomSentenceSpout extends LoadSpout { /** * Constructor. + * * @param ratePerSecond the rate to emite tuples at. */ public FastRandomSentenceSpout(long ratePerSecond) { @@ -142,6 +143,7 @@ public void declareOutputFields(OutputFieldsDeclarer declarer) { /** * The main entry point for ThroughputVsLatency. + * * @param args the command line args * @throws Exception on any error. */ @@ -155,13 +157,15 @@ public static void main(String[] args) throws Exception { .longOpt("test-time") .argName("MINS") .hasArg() - .desc("How long to run the tests for in mins (defaults to " + TEST_EXECUTE_TIME_DEFAULT + ")") + .desc("How long to run the tests for in mins (defaults to " + TEST_EXECUTE_TIME_DEFAULT + + ")") .build()); options.addOption(Option.builder() .longOpt("rate") .argName("SENTENCES/SEC") .hasArg() - .desc("How many sentences per second to run. (defaults to " + DEFAULT_RATE_PER_SECOND + ")") + .desc("How many sentences per second to run. (defaults to " + DEFAULT_RATE_PER_SECOND + + ")") .build()); options.addOption(Option.builder() .longOpt("name") @@ -185,7 +189,8 @@ public static void main(String[] args) throws Exception { .longOpt("splitter-imbalance") .argName("MS(:COUNT)?") .hasArg() - .desc("The number of ms that the first COUNT splitters will wait before processing. This creates an imbalance " + .desc("The number of ms that the first COUNT splitters will wait before processing. " + + "This creates an imbalance " + "that helps test load aware groupings (defaults to 0:1)") .build()); options.addOption(Option.builder() @@ -256,7 +261,7 @@ public static void main(String[] args) throws Exception { conf.registerMetricsConsumer(HttpForwardingMetricsConsumer.class, url, 1); Map workerMetrics = new HashMap<>(); if (!NimbusClient.isLocalOverride()) { - //sigar uses JNI and does not work in local mode + // sigar uses JNI and does not work in local mode workerMetrics.put("CPU", "org.apache.storm.metrics.sigar.CPUMetric"); } conf.put(Config.TOPOLOGY_WORKER_METRICS, workerMetrics); @@ -264,10 +269,12 @@ public static void main(String[] args) throws Exception { TopologyBuilder builder = new TopologyBuilder(); - builder.setSpout("spout", new FastRandomSentenceSpout((long) ratePerSecond / numSpouts), numSpouts); + builder.setSpout("spout", new FastRandomSentenceSpout((long) ratePerSecond / numSpouts), + numSpouts); builder.setBolt("split", new SplitSentence(slowness), numSplits) .shuffleGrouping("spout"); - builder.setBolt("count", new WordCount(), numCounts).fieldsGrouping("split", new Fields("word")); + builder.setBolt("count", new WordCount(), numCounts).fieldsGrouping("split", + new Fields("word")); int exitStatus = -1; try (ScopedTopologySet topologyNames = new ScopedTopologySet(client.getClient())) { diff --git a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/TopologyLoadConf.java b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/TopologyLoadConf.java index 3765f21d13d..a425c314e54 100644 --- a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/TopologyLoadConf.java +++ b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/TopologyLoadConf.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -47,7 +47,8 @@ */ public class TopologyLoadConf { private static final Logger LOG = LoggerFactory.getLogger(TopologyLoadConf.class); - static final Set IMPORTANT_CONF_KEYS = Collections.unmodifiableSet(new HashSet(Arrays.asList( + static final Set IMPORTANT_CONF_KEYS = Collections.unmodifiableSet(new HashSet(Arrays + .asList( Config.TOPOLOGY_WORKERS, Config.TOPOLOGY_ACKER_EXECUTORS, Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT, @@ -83,6 +84,7 @@ public class TopologyLoadConf { /** * Parse the TopologyLoadConf from a file in YAML format. + * * @param file the file to read from * @return the parsed conf * @throws IOException if there is an issue reading the file. @@ -95,6 +97,7 @@ public static TopologyLoadConf fromConf(File file) throws IOException { /** * Parse the TopologyLoadConf from a config map. + * * @param conf the config with the TopologyLoadConf in it * @return the parsed instance. */ @@ -130,6 +133,7 @@ public static TopologyLoadConf fromConf(Map conf) { /** * Write this out to a file in YAML format. + * * @param file the file to write to. * @throws IOException if there is an error writing to the file. */ @@ -142,6 +146,7 @@ public void writeTo(File file) throws IOException { /** * Convert this into a YAML String. + * * @return this as a YAML String. */ public String toYamlString() { @@ -153,6 +158,7 @@ public String toYamlString() { /** * Covert this into a Map config. + * * @return this as a Map config. */ public Map toConf() { @@ -182,6 +188,7 @@ public Map toConf() { /** * Constructor. + * * @param name the name of the topology. * @param topoConf the config for the topology * @param spouts the spouts for the topology @@ -230,11 +237,12 @@ public TopologyLoadConf withName(String baseName) { /** * The first one that is not null. + * * @param rest all the other somethings * @param whatever type you want. * @return the first one that is not null */ - static V or(V...rest) { + static V or(V... rest) { for (V i : rest) { if (i != null) { return i; @@ -243,7 +251,8 @@ static V or(V...rest) { return null; } - LoadCompConf scaleCompParallel(LoadCompConf comp, double v, Map topoSpecificParallel) { + LoadCompConf scaleCompParallel(LoadCompConf comp, double v, Map topoSpecificParallel) { LoadCompConf ret = comp; double scale = or(topoSpecificParallel.get(name + ":" + comp.id), topoSpecificParallel.get(name + ":*"), @@ -255,7 +264,8 @@ LoadCompConf scaleCompParallel(LoadCompConf comp, double v, Map return ret; } - LoadCompConf scaleCompThroughput(LoadCompConf comp, double v, Map topoSpecificParallel) { + LoadCompConf scaleCompThroughput(LoadCompConf comp, double v, Map topoSpecificParallel) { LoadCompConf ret = comp; double scale = or(topoSpecificParallel.get(name + ":" + comp.id), topoSpecificParallel.get(name + ":*"), @@ -267,7 +277,8 @@ LoadCompConf scaleCompThroughput(LoadCompConf comp, double v, Map topoSpecific) { + private LoadCompConf overrideCompSlowExec(LoadCompConf comp, Map topoSpecific) { LoadCompConf ret = comp; SlowExecutorPattern slp = topoSpecific.get(name + ":" + comp.id); if (slp != null) { @@ -277,23 +288,29 @@ private LoadCompConf overrideCompSlowExec(LoadCompConf comp, Map topoSpecific) { if (v == 1.0 && (topoSpecific == null || topoSpecific.isEmpty())) { return this; } - List scaledSpouts = spouts.stream().map((s) -> scaleCompParallel(s, v, topoSpecific)) + List scaledSpouts = spouts.stream().map((s) -> scaleCompParallel(s, v, + topoSpecific)) .collect(Collectors.toList()); - List scaledBolts = bolts.stream().map((s) -> scaleCompParallel(s, v, topoSpecific)) + List scaledBolts = bolts.stream().map((s) -> scaleCompParallel(s, v, + topoSpecific)) .collect(Collectors.toList()); return new TopologyLoadConf(name, topoConf, scaledSpouts, scaledBolts, streams); } /** * Scale the throughput of the entire topology by a percentage. + * * @param v the amount to scale it by 1.0 is nothing 0.5 cuts it in half and 2.0 doubles it. * @return a copy of this with the needed adjustments made. */ @@ -301,15 +318,18 @@ public TopologyLoadConf scaleThroughput(double v, Map topoSpecif if (v == 1.0 && (topoSpecific == null || topoSpecific.isEmpty())) { return this; } - List scaledSpouts = spouts.stream().map((s) -> scaleCompThroughput(s, v, topoSpecific)) + List scaledSpouts = spouts.stream().map((s) -> scaleCompThroughput(s, v, + topoSpecific)) .collect(Collectors.toList()); - List scaledBolts = bolts.stream().map((s) -> scaleCompThroughput(s, v, topoSpecific)) + List scaledBolts = bolts.stream().map((s) -> scaleCompThroughput(s, v, + topoSpecific)) .collect(Collectors.toList()); return new TopologyLoadConf(name, topoConf, scaledSpouts, scaledBolts, streams); } /** * Override the SlowExecutorPattern for given components. + * * @param topoSpecific what we are going to use to override. * @return a copy of this with the needed adjustments made. */ @@ -317,15 +337,18 @@ public TopologyLoadConf overrideSlowExecs(Map topoS if (topoSpecific == null || topoSpecific.isEmpty()) { return this; } - List modedSpouts = spouts.stream().map((s) -> overrideCompSlowExec(s, topoSpecific)) + List modedSpouts = spouts.stream().map((s) -> overrideCompSlowExec(s, + topoSpecific)) .collect(Collectors.toList()); - List modedBolts = bolts.stream().map((b) -> overrideCompSlowExec(b, topoSpecific)) + List modedBolts = bolts.stream().map((b) -> overrideCompSlowExec(b, + topoSpecific)) .collect(Collectors.toList()); return new TopologyLoadConf(name, topoConf, modedSpouts, modedBolts, streams); } /** * Create a new version of this topology with identifiable information removed. + * * @return the anonymized version of the TopologyLoadConf. */ public TopologyLoadConf anonymize() { @@ -362,16 +385,17 @@ public TopologyLoadConf anonymize() { } GlobalStreamId orig = in.gsid(); if (!remappedStreams.containsKey(orig)) { - //Even if the topology is not valid we still need to remap it all - String remappedComp = remappedComponents.computeIfAbsent(in.fromComponent, (key) -> { - LOG.warn("stream's {} from is not defined {}", in.id, in.fromComponent); - return getUniqueBoltName(); - }); + // Even if the topology is not valid we still need to remap it all + String remappedComp = remappedComponents.computeIfAbsent(in.fromComponent, + (key) -> { + LOG.warn("stream's {} from is not defined {}", in.id, in.fromComponent); + return getUniqueBoltName(); + }); remappedStreams.put(orig, new GlobalStreamId(remappedComp, getUniqueStreamName())); } } - //Now we need to map them all back again + // Now we need to map them all back again List remappedSpouts = spouts.stream() .map((orig) -> orig.remap(remappedComponents, remappedStreams)) .collect(Collectors.toList()); @@ -381,11 +405,12 @@ public TopologyLoadConf anonymize() { List remappedInputStreams = streams.stream() .map((orig) -> orig.remap(remappedComponents, remappedStreams)) .collect(Collectors.toList()); - return new TopologyLoadConf(getUniqueTopoName(), anonymizeTopoConf(topoConf), remappedSpouts, remappedBolts, remappedInputStreams); + return new TopologyLoadConf(getUniqueTopoName(), anonymizeTopoConf(topoConf), + remappedSpouts, remappedBolts, remappedInputStreams); } private static Map anonymizeTopoConf(Map topoConf) { - //Only keep important conf keys + // Only keep important conf keys Map ret = new HashMap<>(); for (Map.Entry entry : topoConf.entrySet()) { String key = entry.getKey(); @@ -416,13 +441,15 @@ private static Object cleanupChildOpts(Object value) { for (String subValue : (Collection) value) { ret.add((String) cleanupChildOpts(subValue)); } - return ret.stream().filter((item) -> item != null && !item.isEmpty()).collect(Collectors.toList()); + return ret.stream().filter((item) -> item != null && !item.isEmpty()).collect(Collectors + .toList()); } } /** * Try to see if this looks like a trident topology. * NOTE: this will not work for anonymized configs + * * @return true if it does else false. */ public boolean looksLikeTrident() { @@ -448,6 +475,7 @@ public boolean looksLikeTrident() { /** * Get the messages emitted per second in aggregate across all streams in the topology. + * * @return messages per second. */ public double getAllEmittedAggregate() { @@ -460,6 +488,7 @@ public double getAllEmittedAggregate() { /** * Get the messages emitted per second in aggregate for all of the spouts in the topology. + * * @return messages per second. */ public double getSpoutEmittedAggregate() { @@ -471,12 +500,14 @@ public double getSpoutEmittedAggregate() { } /** - * Try and guess at the actual number of messages emitted per second by a trident topology, not the number of batches. + * Try and guess at the actual number of messages emitted per second by a trident topology, not + * the number of batches. * This does not work on an anonymized conf. + * * @return messages per second or 0 if this does not look like a trident topology. */ public double getTridentEstimatedEmittedAggregate() { - //In this case we are ignoring the coord stuff, and only looking at + // In this case we are ignoring the coord stuff, and only looking at double ret = 0; if (looksLikeTrident()) { List all = new ArrayList<>(bolts); @@ -499,7 +530,8 @@ public double getTridentEstimatedEmittedAggregate() { } public TopologyLoadConf replaceShuffleWithLocalOrShuffle() { - List modified = streams.stream().map((in) -> in.replaceShuffleWithLocalOrShuffle()).collect(Collectors.toList()); + List modified = streams.stream().map((in) -> in + .replaceShuffleWithLocalOrShuffle()).collect(Collectors.toList()); return new TopologyLoadConf(name, topoConf, spouts, bolts, modified); } } diff --git a/examples/storm-loadgen/src/test/java/org/apache/storm/loadgen/LoadCompConfTest.java b/examples/storm-loadgen/src/test/java/org/apache/storm/loadgen/LoadCompConfTest.java index ed10692cdc4..14b6f22c71a 100644 --- a/examples/storm-loadgen/src/test/java/org/apache/storm/loadgen/LoadCompConfTest.java +++ b/examples/storm-loadgen/src/test/java/org/apache/storm/loadgen/LoadCompConfTest.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -18,24 +18,25 @@ package org.apache.storm.loadgen; -import org.junit.jupiter.api.Test; - import static org.junit.jupiter.api.Assertions.assertEquals; +import org.junit.jupiter.api.Test; + public class LoadCompConfTest { @Test public void scaleParallel() { LoadCompConf orig = new LoadCompConf.Builder() .withId("SOME_SPOUT") .withParallelism(1) - .withStream(new OutputStream("default", new NormalDistStats(500.0, 100.0, 300.0, 600.0), false)) + .withStream(new OutputStream("default", new NormalDistStats(500.0, 100.0, 300.0, 600.0), + false)) .build(); assertEquals(500.0, orig.getAllEmittedAggregate(), 0.001); LoadCompConf scaled = orig.scaleParallel(2); - //Parallelism is double + // Parallelism is double assertEquals(2, scaled.parallelism); assertEquals("SOME_SPOUT", scaled.id); - //But throughput is the same + // But throughput is the same assertEquals(500.0, scaled.getAllEmittedAggregate(), 0.001); } @@ -44,14 +45,15 @@ public void scaleThroughput() { LoadCompConf orig = new LoadCompConf.Builder() .withId("SOME_SPOUT") .withParallelism(1) - .withStream(new OutputStream("default", new NormalDistStats(500.0, 100.0, 300.0, 600.0), false)) + .withStream(new OutputStream("default", new NormalDistStats(500.0, 100.0, 300.0, 600.0), + false)) .build(); assertEquals(500.0, orig.getAllEmittedAggregate(), 0.001); LoadCompConf scaled = orig.scaleThroughput(2.0); - //Parallelism is same + // Parallelism is same assertEquals(1, scaled.parallelism); assertEquals("SOME_SPOUT", scaled.id); - //But throughput is the same + // But throughput is the same assertEquals(1000.0, scaled.getAllEmittedAggregate(), 0.001); } -} \ No newline at end of file +} diff --git a/examples/storm-loadgen/src/test/java/org/apache/storm/loadgen/LoadMetricsServerTest.java b/examples/storm-loadgen/src/test/java/org/apache/storm/loadgen/LoadMetricsServerTest.java index 7f549be1c0c..6ef5c832ed1 100644 --- a/examples/storm-loadgen/src/test/java/org/apache/storm/loadgen/LoadMetricsServerTest.java +++ b/examples/storm-loadgen/src/test/java/org/apache/storm/loadgen/LoadMetricsServerTest.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -18,12 +18,12 @@ package org.apache.storm.loadgen; +import static org.apache.storm.loadgen.LoadMetricsServer.convert; +import static org.junit.jupiter.api.Assertions.assertEquals; + import java.util.concurrent.TimeUnit; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.apache.storm.loadgen.LoadMetricsServer.convert; - public class LoadMetricsServerTest { @Test public void convertTest() { @@ -34,4 +34,4 @@ public void convertTest() { } } } -} \ No newline at end of file +} diff --git a/examples/storm-loadgen/src/test/java/org/apache/storm/loadgen/NormalDistStatsTest.java b/examples/storm-loadgen/src/test/java/org/apache/storm/loadgen/NormalDistStatsTest.java index 3512bbae13d..2387456b9c3 100644 --- a/examples/storm-loadgen/src/test/java/org/apache/storm/loadgen/NormalDistStatsTest.java +++ b/examples/storm-loadgen/src/test/java/org/apache/storm/loadgen/NormalDistStatsTest.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -18,10 +18,10 @@ package org.apache.storm.loadgen; -import org.junit.jupiter.api.Test; - import static org.junit.jupiter.api.Assertions.assertEquals; +import org.junit.jupiter.api.Test; + public class NormalDistStatsTest { public static void assertNDSEquals(NormalDistStats a, NormalDistStats b) { assertEquals(a.mean, b.mean, 0.0001, "mean"); @@ -40,4 +40,4 @@ public void scaleBy() { assertNDSEquals(expectedHalf, orig.scaleBy(0.5)); } -} \ No newline at end of file +} diff --git a/examples/storm-loadgen/src/test/java/org/apache/storm/loadgen/OutputStreamTest.java b/examples/storm-loadgen/src/test/java/org/apache/storm/loadgen/OutputStreamTest.java index 3980e2a7ab6..15f58e0ca74 100644 --- a/examples/storm-loadgen/src/test/java/org/apache/storm/loadgen/OutputStreamTest.java +++ b/examples/storm-loadgen/src/test/java/org/apache/storm/loadgen/OutputStreamTest.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -18,14 +18,15 @@ package org.apache.storm.loadgen; -import org.junit.jupiter.api.Test; - import static org.junit.jupiter.api.Assertions.assertEquals; +import org.junit.jupiter.api.Test; + public class OutputStreamTest { @Test public void scaleThroughput() { - OutputStream orig = new OutputStream("ID", new NormalDistStats(100.0, 1.0, 99.0, 101.0), false); + OutputStream orig = new OutputStream("ID", new NormalDistStats(100.0, 1.0, 99.0, 101.0), + false); OutputStream scaled = orig.scaleThroughput(2.0); assertEquals(orig.id, scaled.id); assertEquals(orig.areKeysSkewed, scaled.areKeysSkewed); @@ -34,4 +35,4 @@ public void scaleThroughput() { assertEquals(scaled.rate.min, 199.0, 0.0001); assertEquals(scaled.rate.max, 201.0, 0.0001); } -} \ No newline at end of file +} diff --git a/examples/storm-perf/pom.xml b/examples/storm-perf/pom.xml index 7e9377cbf10..69c435b9e2e 100644 --- a/examples/storm-perf/pom.xml +++ b/examples/storm-perf/pom.xml @@ -86,6 +86,16 @@ ${storm.topology} + + org.openrewrite.maven + rewrite-maven-plugin + + + + org.codehaus.gmavenplus + gmavenplus-plugin + + org.apache.maven.plugins maven-checkstyle-plugin diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/BackPressureTopo.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/BackPressureTopo.java index 5fb70220eb9..a3b0da5ac94 100644 --- a/examples/storm-perf/src/main/java/org/apache/storm/perf/BackPressureTopo.java +++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/BackPressureTopo.java @@ -34,7 +34,6 @@ import org.apache.storm.utils.Utils; import org.slf4j.LoggerFactory; - public class BackPressureTopo { private static final String SPOUT_ID = "ConstSpout"; @@ -80,7 +79,8 @@ public static void main(String[] args) throws Exception { return; } // Submit topology to storm cluster - Helper.runOnClusterAndPrintMetrics(runTime, "BackPressureTopo", topoConf, getTopology(topoConf)); + Helper.runOnClusterAndPrintMetrics(runTime, "BackPressureTopo", topoConf, + getTopology(topoConf)); } private static class ThrottledBolt extends BaseRichBolt { @@ -93,7 +93,8 @@ private static class ThrottledBolt extends BaseRichBolt { } @Override - public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) { + public void prepare(Map topoConf, TopologyContext context, + OutputCollector collector) { this.collector = collector; } @@ -104,7 +105,7 @@ public void execute(Tuple tuple) { try { Thread.sleep(sleepMs); } catch (InterruptedException e) { - //.. ignore + // .. ignore } } diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/ConstSpoutIdBoltNullBoltTopo.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/ConstSpoutIdBoltNullBoltTopo.java index c83763d6e26..b30f689377f 100644 --- a/examples/storm-perf/src/main/java/org/apache/storm/perf/ConstSpoutIdBoltNullBoltTopo.java +++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/ConstSpoutIdBoltNullBoltTopo.java @@ -29,8 +29,10 @@ import org.apache.storm.utils.Utils; /** - * ConstSpout -> IdBolt -> DevNullBolt This topology measures speed of messaging between spouts->bolt and bolt->bolt ConstSpout : - * Continuously emits a constant string IdBolt : clones and emits input tuples DevNullBolt : discards incoming tuples. + * ConstSpout -> IdBolt -> DevNullBolt This topology measures speed of messaging between + * spouts->bolt and bolt->bolt ConstSpout : + * Continuously emits a constant string IdBolt : clones and emits input tuples DevNullBolt : + * discards incoming tuples. */ public class ConstSpoutIdBoltNullBoltTopo { @@ -67,11 +69,11 @@ static StormTopology getTopology(Map conf) { int numBolt2 = Helper.getInt(conf, BOLT2_COUNT, 1); builder.setBolt(BOLT2_ID, bolt2, numBolt2) .localOrShuffleGrouping(BOLT1_ID); - System.err.printf("====> Using : numSpouts = %d , numBolt1 = %d, numBolt2=%d\n", numSpouts, numBolt1, numBolt2); + System.err.printf("====> Using : numSpouts = %d , numBolt1 = %d, numBolt2=%d\n", numSpouts, + numBolt1, numBolt2); return builder.createTopology(); } - public static void main(String[] args) throws Exception { int runTime = -1; Config topoConf = new Config(); diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/ConstSpoutNullBoltTopo.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/ConstSpoutNullBoltTopo.java index 63ef51b45c7..34b4704ee23 100755 --- a/examples/storm-perf/src/main/java/org/apache/storm/perf/ConstSpoutNullBoltTopo.java +++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/ConstSpoutNullBoltTopo.java @@ -87,11 +87,15 @@ public static void main(String[] args) throws Exception { Config topoConf = new Config(); // Configured for achieving max throughput in single worker mode (empirically found). // For reference : numbers taken on MacBook Pro mid 2015 - // -- ACKer=0: ~8 mill/sec (batchSz=2k & recvQsize=50k). 6.7 mill/sec (batchSz=1 & recvQsize=1k) - // -- ACKer=1: ~1 mill/sec, lat= ~1 microsec (batchSz=1 & bolt.wait.strategy=Park bolt.wait.park.micros=0) - // -- ACKer=1: ~1.3 mill/sec, lat= ~11 micros (batchSz=1 & receive.buffer.size=1k, bolt.wait & bp.wait = + // -- ACKer=0: ~8 mill/sec (batchSz=2k & recvQsize=50k). 6.7 mill/sec (batchSz=1 & + // recvQsize=1k) + // -- ACKer=1: ~1 mill/sec, lat= ~1 microsec (batchSz=1 & bolt.wait.strategy=Park + // bolt.wait.park.micros=0) + // -- ACKer=1: ~1.3 mill/sec, lat= ~11 micros (batchSz=1 & receive.buffer.size=1k, bolt.wait + // & bp.wait = // Progressive[defaults]) - // -- ACKer=1: ~1.6 mill/sec, lat= ~300 micros (batchSz=500 & bolt.wait.strategy=Park bolt.wait.park.micros=0) + // -- ACKer=1: ~1.6 mill/sec, lat= ~300 micros (batchSz=500 & bolt.wait.strategy=Park + // bolt.wait.park.micros=0) topoConf.put(Config.TOPOLOGY_SPOUT_RECVQ_SKIPS, 8); topoConf.put(Config.TOPOLOGY_PRODUCER_BATCH_SIZE, 500); topoConf.put(Config.TOPOLOGY_EXECUTOR_RECEIVE_BUFFER_SIZE, 50_000); diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/ConstSpoutOnlyTopo.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/ConstSpoutOnlyTopo.java index 1b012fe2fb8..44eb2cb37fd 100755 --- a/examples/storm-perf/src/main/java/org/apache/storm/perf/ConstSpoutOnlyTopo.java +++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/ConstSpoutOnlyTopo.java @@ -35,7 +35,6 @@ public class ConstSpoutOnlyTopo { public static final String TOPOLOGY_NAME = "ConstSpoutOnlyTopo"; public static final String SPOUT_ID = "constSpout"; - static StormTopology getTopology() { // 1 - Setup Const Spout -------- diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/FileReadWordCountSpoutCompressionTopo.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/FileReadWordCountSpoutCompressionTopo.java index 51c43057ef4..a15aa0f7f78 100644 --- a/examples/storm-perf/src/main/java/org/apache/storm/perf/FileReadWordCountSpoutCompressionTopo.java +++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/FileReadWordCountSpoutCompressionTopo.java @@ -50,7 +50,6 @@ public class FileReadWordCountSpoutCompressionTopo { public static final int DEFAULT_SPLIT_BOLT_NUM = 2; public static final int DEFAULT_COUNT_BOLT_NUM = 2; - static StormTopology getTopology(Map config) { final int spoutNum = Helper.getInt(config, SPOUT_NUM, DEFAULT_SPOUT_NUM); @@ -62,8 +61,10 @@ static StormTopology getTopology(Map config) { // sampledata/longrandomwords.txt contains sentences with at least 1500 bytes builder.setSpout(SPOUT_ID, new FileReadSpout(inputFile), spoutNum) .addConfiguration(Config.TOPOLOGY_TUPLE_COMPRESSION_ENABLE, true); - builder.setBolt(SPLIT_ID, new SplitSentenceBolt(), spBoltNum).localOrShuffleGrouping(SPOUT_ID); - builder.setBolt(COUNT_ID, new CountBolt(), cntBoltNum).fieldsGrouping(SPLIT_ID, new Fields(SplitSentenceBolt.FIELDS)); + builder.setBolt(SPLIT_ID, new SplitSentenceBolt(), spBoltNum) + .localOrShuffleGrouping(SPOUT_ID); + builder.setBolt(COUNT_ID, new CountBolt(), cntBoltNum).fieldsGrouping(SPLIT_ID, + new Fields(SplitSentenceBolt.FIELDS)); return builder.createTopology(); } @@ -78,7 +79,8 @@ public static void main(String[] args) throws Exception { topoConf.putAll(Utils.findAndReadConfigFile(args[1])); } topoConf.put(Config.TOPOLOGY_PRODUCER_BATCH_SIZE, 1000); - topoConf.put(Config.TOPOLOGY_BOLT_WAIT_STRATEGY, "org.apache.storm.policy.WaitStrategyPark"); + topoConf.put(Config.TOPOLOGY_BOLT_WAIT_STRATEGY, + "org.apache.storm.policy.WaitStrategyPark"); topoConf.put(Config.TOPOLOGY_BOLT_WAIT_PARK_MICROSEC, 0); topoConf.put(Config.TOPOLOGY_DISABLE_LOADAWARE_MESSAGING, true); topoConf.put(Config.TOPOLOGY_STATS_SAMPLE_RATE, 0.0005); diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/FileReadWordCountTopo.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/FileReadWordCountTopo.java index 78eaab7eff7..a0f9c5f8e54 100644 --- a/examples/storm-perf/src/main/java/org/apache/storm/perf/FileReadWordCountTopo.java +++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/FileReadWordCountTopo.java @@ -50,7 +50,6 @@ public class FileReadWordCountTopo { public static final int DEFAULT_SPLIT_BOLT_NUM = 2; public static final int DEFAULT_COUNT_BOLT_NUM = 2; - static StormTopology getTopology(Map config) { final int spoutNum = Helper.getInt(config, SPOUT_NUM, DEFAULT_SPOUT_NUM); @@ -60,8 +59,10 @@ static StormTopology getTopology(Map config) { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout(SPOUT_ID, new FileReadSpout(inputFile), spoutNum); - builder.setBolt(SPLIT_ID, new SplitSentenceBolt(), spBoltNum).localOrShuffleGrouping(SPOUT_ID); - builder.setBolt(COUNT_ID, new CountBolt(), cntBoltNum).fieldsGrouping(SPLIT_ID, new Fields(SplitSentenceBolt.FIELDS)); + builder.setBolt(SPLIT_ID, new SplitSentenceBolt(), spBoltNum) + .localOrShuffleGrouping(SPOUT_ID); + builder.setBolt(COUNT_ID, new CountBolt(), cntBoltNum).fieldsGrouping(SPLIT_ID, + new Fields(SplitSentenceBolt.FIELDS)); return builder.createTopology(); } @@ -76,7 +77,8 @@ public static void main(String[] args) throws Exception { topoConf.putAll(Utils.findAndReadConfigFile(args[1])); } topoConf.put(Config.TOPOLOGY_PRODUCER_BATCH_SIZE, 1000); - topoConf.put(Config.TOPOLOGY_BOLT_WAIT_STRATEGY, "org.apache.storm.policy.WaitStrategyPark"); + topoConf.put(Config.TOPOLOGY_BOLT_WAIT_STRATEGY, + "org.apache.storm.policy.WaitStrategyPark"); topoConf.put(Config.TOPOLOGY_BOLT_WAIT_PARK_MICROSEC, 0); topoConf.put(Config.TOPOLOGY_DISABLE_LOADAWARE_MESSAGING, true); topoConf.put(Config.TOPOLOGY_STATS_SAMPLE_RATE, 0.0005); diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/HdfsSpoutNullBoltTopo.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/HdfsSpoutNullBoltTopo.java index 9876cac9f3f..bb55ee2bdf0 100644 --- a/examples/storm-perf/src/main/java/org/apache/storm/perf/HdfsSpoutNullBoltTopo.java +++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/HdfsSpoutNullBoltTopo.java @@ -92,13 +92,15 @@ public static void main(String[] args) throws Exception { Config topoConf = new Config(); topoConf.putAll(Utils.findAndReadConfigFile(args[1])); topoConf.put(Config.TOPOLOGY_PRODUCER_BATCH_SIZE, 1000); - topoConf.put(Config.TOPOLOGY_BOLT_WAIT_STRATEGY, "org.apache.storm.policy.WaitStrategyPark"); + topoConf.put(Config.TOPOLOGY_BOLT_WAIT_STRATEGY, + "org.apache.storm.policy.WaitStrategyPark"); topoConf.put(Config.TOPOLOGY_BOLT_WAIT_PARK_MICROSEC, 0); topoConf.put(Config.TOPOLOGY_DISABLE_LOADAWARE_MESSAGING, true); topoConf.put(Config.TOPOLOGY_STATS_SAMPLE_RATE, 0.0005); topoConf.putAll(Utils.readCommandLineOpts()); // Submit to Storm cluster - Helper.runOnClusterAndPrintMetrics(durationSec, TOPOLOGY_NAME, topoConf, getTopology(topoConf)); + Helper.runOnClusterAndPrintMetrics(durationSec, TOPOLOGY_NAME, topoConf, + getTopology(topoConf)); } } diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/JitterAwareGroupingTopology.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/JitterAwareGroupingTopology.java index 7bc53476f3c..c12eec83023 100644 --- a/examples/storm-perf/src/main/java/org/apache/storm/perf/JitterAwareGroupingTopology.java +++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/JitterAwareGroupingTopology.java @@ -52,14 +52,17 @@ * *

{@code JitteryWorkerBolt} tasks have task-index-dependent latency jitter: task 0 is * perfectly steady and each higher index is progressively jitterier, because the per-tuple noise - * width grows with the task index over a constant floor. This deliberately exercises the signal + * width grows with the task index over a constant floor. This deliberately exercises the + * signal * the grouping ranks on — RFC-1889 jitter (the EWMA of {@code |Δlatency|}), which measures dispersion, * not level. A per-task mean offset would not work: it cancels in the estimator's * consecutive-difference, leaving every task with identical jitter. With upstream feedback enabled, - * {@link JitterAwareStreamGrouping} steers more tuples toward the steadiest (lowest-jitter) tasks; run it + * {@link JitterAwareStreamGrouping} steers more tuples toward the steadiest (lowest-jitter) tasks; + * run it * against the {@code loadaware} baseline mode to measure the effect. * - *

Run the baseline and the jitter-aware run back-to-back to compare. Select the grouping with the + *

Run the baseline and the jitter-aware run back-to-back to compare. Select the grouping with + * the * {@code grouping.mode} flag: *

  *   # Baseline: plain LoadAwareShuffleGrouping (no upstream feedback needed).
@@ -139,8 +142,10 @@ static StormTopology getTopology(Map conf) {
     /**
      * Picks the {@code splitter -> worker} grouping from {@link #GROUPING_MODE}. Defaults to the
      * feedback-driven {@link JitterAwareStreamGrouping}; {@code grouping.mode=loadaware} selects the plain
-     * {@link LoadAwareShuffleGrouping} baseline so the two can be benchmarked back-to-back. Both implement
-     * {@link org.apache.storm.grouping.LoadAwareCustomStreamGrouping} and require the locality-aware
+     * {@link LoadAwareShuffleGrouping} baseline so the two can be benchmarked back-to-back. Both
+     * implement
+     * {@link org.apache.storm.grouping.LoadAwareCustomStreamGrouping} and require the
+     * locality-aware
      * configuration set in {@link #main}.
      */
     static CustomStreamGrouping selectGrouping(Map conf) {
@@ -166,17 +171,24 @@ public static void main(String[] args) throws Exception {
         }
 
         topoConf.put(Config.TOPOLOGY_STATS_EWMA_ENABLE, true);
-        // max.spout.pending counts SPOUT tuples (sentences), but each fans out to ~7 word-tuples at the
-        // SplitterBolt, so the in-flight backlog at the slow workers is ~7x this number. Keep it low enough
-        // that steady-state complete latency stays well under topology.message.timeout.secs with ~0 fails;
-        // 4000 oversubscribed the synthetic workers and pinned latency at the timeout. Tune per cluster:
+        // max.spout.pending counts SPOUT tuples (sentences), but each fans out to ~7 word-tuples at
+        // the
+        // SplitterBolt, so the in-flight backlog at the slow workers is ~7x this number. Keep it
+        // low enough
+        // that steady-state complete latency stays well under topology.message.timeout.secs with ~0
+        // fails;
+        // 4000 oversubscribed the synthetic workers and pinned latency at the timeout. Tune per
+        // cluster:
         // complete_latency ~= (pending * fanout) / aggregate_worker_ack_rate (Little's law).
         topoConf.putIfAbsent(Config.TOPOLOGY_MAX_SPOUT_PENDING, 500);
         topoConf.putIfAbsent(Config.TOPOLOGY_UPSTREAM_FEEDBACK_FREQ_SECS, 10);
 
-        // Both grouping modes resolve to a LoadAwareShuffleGrouping (directly, or as the jitter grouping's
-        // fallback), whose prepare() requires these locality-aware keys. Normally supplied by defaults.yaml;
-        // set defensively so the topology is self-contained. Placed before the CLI merge so -c can override.
+        // Both grouping modes resolve to a LoadAwareShuffleGrouping (directly, or as the jitter
+        // grouping's
+        // fallback), whose prepare() requires these locality-aware keys. Normally supplied by
+        // defaults.yaml;
+        // set defensively so the topology is self-contained. Placed before the CLI merge so -c can
+        // override.
         topoConf.putIfAbsent(Config.STORM_NETWORK_TOPOGRAPHY_PLUGIN,
             "org.apache.storm.networktopography.DefaultRackDNSToSwitchMapping");
         topoConf.putIfAbsent(Config.TOPOLOGY_LOCALITYAWARE_HIGHER_BOUND, 0.8);
@@ -205,7 +217,8 @@ private static class GenSpout extends BaseRichSpout {
         }
 
         @Override
-        public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
+        public void open(Map conf, TopologyContext context,
+            SpoutOutputCollector collector) {
             this.collector = collector;
             try {
                 this.lines = FileReadSpout.readLines(new FileInputStream(filePath));
@@ -237,7 +250,8 @@ private static class SplitterBolt extends BaseRichBolt {
         private OutputCollector collector;
 
         @Override
-        public void prepare(Map conf, TopologyContext context, OutputCollector collector) {
+        public void prepare(Map conf, TopologyContext context,
+            OutputCollector collector) {
             this.collector = collector;
         }
 
@@ -257,12 +271,14 @@ public void declareOutputFields(OutputFieldsDeclarer declarer) {
     }
 
     /**
-     * Counts words and parks for a constant floor plus task-index-proportional random noise, so each
+     * Counts words and parks for a constant floor plus task-index-proportional random noise, so
+     * each
      * task's RFC-1889 execute-jitter (the EWMA of {@code |Δ execute-latency|}) differs by construction.
      *
      * 

Task {@code i} parks for {@code baseDelayUs} µs plus uniform noise in {@code [0, i * baseDelayUs]} * µs per tuple. Crucially, the constant floor cancels in the consecutive-difference the jitter - * estimator takes, so only the noise width drives jitter — not a per-task mean offset (which + * estimator takes, so only the noise width drives jitter — not a per-task mean offset + * (which * would cancel and leave every task with identical jitter). Steady-state jitter is {@code width / 3}. * For a 4-task setup with the default {@code baseDelayUs = 2000}: *

    @@ -271,11 +287,14 @@ public void declareOutputFields(OutputFieldsDeclarer declarer) { *
  • Task 2: ~2–6 ms — jitter ≈ 1.33 ms
  • *
  • Task 3: ~2–8 ms — jitter ≈ 2 ms (jitteriest)
  • *
- * Widths are millisecond-scale so the EWMA jitter gauge (millisecond resolution) records distinct - * values per task. {@link JitterAwareStreamGrouping} then steers tuples toward the lowest-jitter + * Widths are millisecond-scale so the EWMA jitter gauge (millisecond resolution) records + * distinct + * values per task. {@link JitterAwareStreamGrouping} then steers tuples toward the + * lowest-jitter * task (task 0) when feedback is enabled. * - *

{@link LockSupport#parkNanos} is used instead of a spin loop so jittery tasks yield the CPU + *

{@link LockSupport#parkNanos} is used instead of a spin loop so jittery tasks yield the + * CPU * and do not starve the steady task's executor thread. */ private static class JitteryWorkerBolt extends BaseRichBolt { @@ -290,14 +309,18 @@ private static class JitteryWorkerBolt extends BaseRichBolt { } @Override - public void prepare(Map conf, TopologyContext context, OutputCollector collector) { + public void prepare(Map conf, TopologyContext context, + OutputCollector collector) { this.collector = collector; long baseDelayNs = baseDelayUs * 1_000L; - // Constant floor parked by every task: keeps each task doing real work, but cancels in the + // Constant floor parked by every task: keeps each task doing real work, but cancels in + // the // consecutive-difference the RFC-1889 jitter estimator takes, so it adds no jitter. this.baseFloorNs = baseDelayNs; - // Noise WIDTH grows with task index, so execute-jitter (EWMA of |Δlatency|) genuinely differs - // per task: task 0 is perfectly steady (zero jitter), higher indices are progressively jitterier. + // Noise WIDTH grows with task index, so execute-jitter (EWMA of |Δlatency|) genuinely + // differs + // per task: task 0 is perfectly steady (zero jitter), higher indices are progressively + // jitterier. this.jitterWidthNs = baseDelayNs * context.getThisTaskIndex(); } @@ -307,7 +330,8 @@ public void execute(Tuple tuple) { counts.merge(word, 1, Integer::sum); int count = counts.get(word); - long noiseNs = jitterWidthNs == 0L ? 0L : (long) (ThreadLocalRandom.current().nextDouble() * jitterWidthNs); + long noiseNs = jitterWidthNs == 0L ? 0L : (long) (ThreadLocalRandom.current() + .nextDouble() * jitterWidthNs); long sleepNs = baseFloorNs + noiseNs; if (sleepNs > 0) { LockSupport.parkNanos(sleepNs); @@ -332,7 +356,8 @@ private static class SinkBolt extends BaseRichBolt { private OutputCollector collector; @Override - public void prepare(Map conf, TopologyContext context, OutputCollector collector) { + public void prepare(Map conf, TopologyContext context, + OutputCollector collector) { this.collector = collector; } diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/KafkaClientHdfsTopo.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/KafkaClientHdfsTopo.java index 5eb61b2e4e3..defaef39ef5 100755 --- a/examples/storm-perf/src/main/java/org/apache/storm/perf/KafkaClientHdfsTopo.java +++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/KafkaClientHdfsTopo.java @@ -58,7 +58,6 @@ public class KafkaClientHdfsTopo { public static final String HDFS_PATH = "hdfs.dir"; public static final String HDFS_BATCH = "hdfs.batch"; - public static final int DEFAULT_SPOUT_NUM = 1; public static final int DEFAULT_BOLT_NUM = 1; public static final int DEFAULT_HDFS_BATCH = 1000; @@ -68,7 +67,6 @@ public class KafkaClientHdfsTopo { public static final String SPOUT_ID = "kafkaSpout"; public static final String BOLT_ID = "hdfsBolt"; - static StormTopology getTopology(Map config) { final int spoutNum = getInt(config, SPOUT_NUM, DEFAULT_SPOUT_NUM); @@ -80,7 +78,8 @@ static StormTopology getTopology(Map config) { String bootstrapHosts = getStr(config, KAFKA_BOOTSTRAP_HOSTS); String topicName = getStr(config, KAFKA_TOPIC); - KafkaSpoutConfig spoutConfig = KafkaSpoutConfig.builder(bootstrapHosts, topicName) + KafkaSpoutConfig spoutConfig = KafkaSpoutConfig.builder(bootstrapHosts, + topicName) .setFirstPollOffsetStrategy( FirstPollOffsetStrategy.EARLIEST) .build(); @@ -91,9 +90,11 @@ static StormTopology getTopology(Map config) { String hdfsUrls = getStr(config, HDFS_URI); RecordFormat format = new LineWriter("value"); SyncPolicy syncPolicy = new CountSyncPolicy(hdfsBatch); - FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(1.0f, FileSizeRotationPolicy.Units.GB); + FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(1.0f, + FileSizeRotationPolicy.Units.GB); - FileNameFormat fileNameFormat = new DefaultFileNameFormat().withPath(getStr(config, HDFS_PATH)); + FileNameFormat fileNameFormat = new DefaultFileNameFormat().withPath(getStr(config, + HDFS_PATH)); // Instantiate the HdfsBolt HdfsBolt bolt = new HdfsBolt() @@ -113,7 +114,6 @@ static StormTopology getTopology(Map config) { return builder.createTopology(); } - public static int getInt(Map map, Object key, int def) { return ObjectReader.getInt(Utils.get(map, key, def)); } @@ -122,9 +122,9 @@ public static String getStr(Map map, Object key) { return (String) map.get(key); } - /** - * Copies text file content from sourceDir to destinationDir. Moves source files into sourceDir after its done consuming. + * Copies text file content from sourceDir to destinationDir. Moves source files into sourceDir + * after its done consuming. */ public static void main(String[] args) throws Exception { @@ -138,13 +138,15 @@ public static void main(String[] args) throws Exception { topoConf.put(Config.TOPOLOGY_PRODUCER_BATCH_SIZE, 1000); topoConf.put(Config.TOPOLOGY_DISABLE_LOADAWARE_MESSAGING, true); topoConf.put(Config.TOPOLOGY_STATS_SAMPLE_RATE, 0.0005); - topoConf.put(Config.TOPOLOGY_BOLT_WAIT_STRATEGY, "org.apache.storm.policy.WaitStrategyPark"); + topoConf.put(Config.TOPOLOGY_BOLT_WAIT_STRATEGY, + "org.apache.storm.policy.WaitStrategyPark"); topoConf.put(Config.TOPOLOGY_BOLT_WAIT_PARK_MICROSEC, 0); topoConf.putAll(Utils.readCommandLineOpts()); // Submit topology to Storm cluster Integer durationSec = Integer.parseInt(args[0]); - Helper.runOnClusterAndPrintMetrics(durationSec, TOPOLOGY_NAME, topoConf, getTopology(topoConf)); + Helper.runOnClusterAndPrintMetrics(durationSec, TOPOLOGY_NAME, topoConf, + getTopology(topoConf)); } public static class LineWriter implements RecordFormat { diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/KafkaClientSpoutNullBoltTopo.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/KafkaClientSpoutNullBoltTopo.java index 4ac1ed8e86e..ec346cce869 100644 --- a/examples/storm-perf/src/main/java/org/apache/storm/perf/KafkaClientSpoutNullBoltTopo.java +++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/KafkaClientSpoutNullBoltTopo.java @@ -22,15 +22,16 @@ import org.apache.storm.generated.StormTopology; import org.apache.storm.kafka.spout.FirstPollOffsetStrategy; import org.apache.storm.kafka.spout.KafkaSpout; -import org.apache.storm.kafka.spout.KafkaSpoutConfig; import org.apache.storm.kafka.spout.KafkaSpoutConfig.ProcessingGuarantee; +import org.apache.storm.kafka.spout.KafkaSpoutConfig; import org.apache.storm.perf.bolt.DevNullBolt; import org.apache.storm.perf.utils.Helper; import org.apache.storm.topology.TopologyBuilder; import org.apache.storm.utils.Utils; /** - * Benchmark topology for measuring spout read/emit/ack performance. The spout reads and emits tuples. The bolt acks and discards received + * Benchmark topology for measuring spout read/emit/ack performance. The spout reads and emits + * tuples. The bolt acks and discards received * tuples. */ public class KafkaClientSpoutNullBoltTopo { @@ -62,14 +63,17 @@ public static StormTopology getTopology(Map config) { final int boltNum = Helper.getInt(config, BOLT_NUM, DEFAULT_BOLT_NUM); // 1 - Setup Kafka Spout -------- - String bootstrapServers = Optional.ofNullable(Helper.getStr(config, BOOTSTRAP_SERVERS)).orElse("127.0.0.1:9092"); - String kafkaTopic = Optional.ofNullable(Helper.getStr(config, KAFKA_TOPIC)).orElse("storm-perf-null-bolt-topic"); + String bootstrapServers = Optional.ofNullable(Helper.getStr(config, BOOTSTRAP_SERVERS)) + .orElse("127.0.0.1:9092"); + String kafkaTopic = Optional.ofNullable(Helper.getStr(config, KAFKA_TOPIC)) + .orElse("storm-perf-null-bolt-topic"); ProcessingGuarantee processingGuarantee = ProcessingGuarantee.valueOf( Optional.ofNullable(Helper.getStr(config, PROCESSING_GUARANTEE)) .orElse(ProcessingGuarantee.AT_LEAST_ONCE.name())); int offsetCommitPeriodMs = Helper.getInt(config, OFFSET_COMMIT_PERIOD_MS, 30_000); - KafkaSpoutConfig kafkaSpoutConfig = KafkaSpoutConfig.builder(bootstrapServers, kafkaTopic) + KafkaSpoutConfig kafkaSpoutConfig = KafkaSpoutConfig + .builder(bootstrapServers, kafkaTopic) .setProcessingGuarantee(processingGuarantee) .setOffsetCommitPeriodMs(offsetCommitPeriodMs) .setFirstPollOffsetStrategy( @@ -109,7 +113,8 @@ public static void main(String[] args) throws Exception { } // Submit to Storm cluster - Helper.runOnClusterAndPrintMetrics(durationSec, TOPOLOGY_NAME, topoConf, getTopology(topoConf)); + Helper.runOnClusterAndPrintMetrics(durationSec, TOPOLOGY_NAME, topoConf, + getTopology(topoConf)); } } diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/LowThroughputTopo.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/LowThroughputTopo.java index 71ffd7caeae..de3fb0cdf7e 100644 --- a/examples/storm-perf/src/main/java/org/apache/storm/perf/LowThroughputTopo.java +++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/LowThroughputTopo.java @@ -49,7 +49,8 @@ static StormTopology getTopology(Map conf) { Long sleepMs = ObjectReader.getLong(conf.get(SLEEP_MS)); // 1 - Setup Spout -------- - ThrottledSpout spout = new ThrottledSpout(sleepMs).withOutputFields(ThrottledSpout.DEFAULT_FIELD_NAME); + ThrottledSpout spout = new ThrottledSpout(sleepMs) + .withOutputFields(ThrottledSpout.DEFAULT_FIELD_NAME); // 2 - Setup DevNull Bolt -------- LatencyPrintBolt bolt = new LatencyPrintBolt(); @@ -83,7 +84,8 @@ public static void main(String[] args) throws Exception { } topoConf.putAll(Utils.readCommandLineOpts()); // Submit topology to storm cluster - Helper.runOnClusterAndPrintMetrics(runTime, "LowThroughputTopo", topoConf, getTopology(topoConf)); + Helper.runOnClusterAndPrintMetrics(runTime, "LowThroughputTopo", topoConf, + getTopology(topoConf)); } private static class ThrottledSpout extends BaseRichSpout { diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/SimplifiedWordCountTopo.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/SimplifiedWordCountTopo.java index 0a9148c593d..24d6175c57b 100644 --- a/examples/storm-perf/src/main/java/org/apache/storm/perf/SimplifiedWordCountTopo.java +++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/SimplifiedWordCountTopo.java @@ -42,7 +42,6 @@ public class SimplifiedWordCountTopo { public static final int DEFAULT_SPOUT_NUM = 1; public static final int DEFAULT_COUNT_BOLT_NUM = 1; - static StormTopology getTopology(Map config) { final int spoutNum = Helper.getInt(config, SPOUT_NUM, DEFAULT_SPOUT_NUM); @@ -51,7 +50,8 @@ static StormTopology getTopology(Map config) { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout(SPOUT_ID, new WordGenSpout(inputFile), spoutNum); - builder.setBolt(COUNT_ID, new CountBolt(), cntBoltNum).fieldsGrouping(SPOUT_ID, new Fields(WordGenSpout.FIELDS)); + builder.setBolt(COUNT_ID, new CountBolt(), cntBoltNum).fieldsGrouping(SPOUT_ID, + new Fields(WordGenSpout.FIELDS)); return builder.createTopology(); } @@ -74,7 +74,8 @@ public static void main(String[] args) throws Exception { topoConf.put(Config.TOPOLOGY_PRODUCER_BATCH_SIZE, 1000); topoConf.put(Config.TOPOLOGY_DISABLE_LOADAWARE_MESSAGING, true); topoConf.put(Config.TOPOLOGY_STATS_SAMPLE_RATE, 0.0005); - topoConf.put(Config.TOPOLOGY_BOLT_WAIT_STRATEGY, "org.apache.storm.policy.WaitStrategyPark"); + topoConf.put(Config.TOPOLOGY_BOLT_WAIT_STRATEGY, + "org.apache.storm.policy.WaitStrategyPark"); topoConf.put(Config.TOPOLOGY_BOLT_WAIT_PARK_MICROSEC, 0); topoConf.putAll(Utils.readCommandLineOpts()); diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/StrGenSpoutHdfsBoltTopo.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/StrGenSpoutHdfsBoltTopo.java index 0c53db97c6f..41bde2f37f5 100755 --- a/examples/storm-perf/src/main/java/org/apache/storm/perf/StrGenSpoutHdfsBoltTopo.java +++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/StrGenSpoutHdfsBoltTopo.java @@ -16,7 +16,6 @@ * limitations under the License */ - package org.apache.storm.perf; import java.util.Map; @@ -63,7 +62,6 @@ public class StrGenSpoutHdfsBoltTopo { public static final String SPOUT_ID = "GenSpout"; public static final String BOLT_ID = "hdfsBolt"; - static StormTopology getTopology(Map topoConf) { final int hdfsBatch = Helper.getInt(topoConf, HDFS_BATCH, DEFAULT_HDFS_BATCH); @@ -75,12 +73,14 @@ static StormTopology getTopology(Map topoConf) { String hdfsUrl = Helper.getStr(topoConf, HDFS_URI); RecordFormat format = new LineWriter("str"); SyncPolicy syncPolicy = new CountSyncPolicy(hdfsBatch); - FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(1.0f, FileSizeRotationPolicy.Units.GB); + FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(1.0f, + FileSizeRotationPolicy.Units.GB); final int spoutNum = Helper.getInt(topoConf, SPOUT_NUM, DEFAULT_SPOUT_NUM); final int boltNum = Helper.getInt(topoConf, BOLT_NUM, DEFAULT_BOLT_NUM); // Use default, Storm-generated file names - FileNameFormat fileNameFormat = new DefaultFileNameFormat().withPath(Helper.getStr(topoConf, HDFS_PATH)); + FileNameFormat fileNameFormat = new DefaultFileNameFormat().withPath(Helper.getStr(topoConf, + HDFS_PATH)); // Instantiate the HdfsBolt HdfsBolt bolt = new HdfsBolt() @@ -101,13 +101,12 @@ static StormTopology getTopology(Map topoConf) { return builder.createTopology(); } - /** * Spout generates random strings and HDFS bolt writes them to a text file. */ public static void main(String[] args) throws Exception { String confFile = "conf/HdfsSpoutTopo.yaml"; - int runTime = -1; //Run until Ctrl-C + int runTime = -1; // Run until Ctrl-C if (args.length > 0) { runTime = Integer.parseInt(args[0]); } @@ -124,7 +123,8 @@ public static void main(String[] args) throws Exception { Map topoConf = Utils.findAndReadConfigFile(confFile); topoConf.put(Config.TOPOLOGY_PRODUCER_BATCH_SIZE, 1000); - topoConf.put(Config.TOPOLOGY_BOLT_WAIT_STRATEGY, "org.apache.storm.policy.WaitStrategyPark"); + topoConf.put(Config.TOPOLOGY_BOLT_WAIT_STRATEGY, + "org.apache.storm.policy.WaitStrategyPark"); topoConf.put(Config.TOPOLOGY_BOLT_WAIT_PARK_MICROSEC, 0); topoConf.put(Config.TOPOLOGY_DISABLE_LOADAWARE_MESSAGING, true); topoConf.put(Config.TOPOLOGY_STATS_SAMPLE_RATE, 0.0005); @@ -133,7 +133,6 @@ public static void main(String[] args) throws Exception { Helper.runOnClusterAndPrintMetrics(runTime, TOPOLOGY_NAME, topoConf, getTopology(topoConf)); } - public static class LineWriter implements RecordFormat { private static final long serialVersionUID = 7524288317405514146L; private String lineDelimiter = System.lineSeparator(); diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/ThroughputMeter.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/ThroughputMeter.java index 3d8e736224e..8a7b439b712 100644 --- a/examples/storm-perf/src/main/java/org/apache/storm/perf/ThroughputMeter.java +++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/ThroughputMeter.java @@ -32,6 +32,7 @@ public ThroughputMeter(String name) { /** * Calculate throughput. + * * @return events/sec */ private static double calcThroughput(long count, long startTime, long endTime) { diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/bolt/CountBolt.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/bolt/CountBolt.java index ee1bf7ee1f7..0a4f0e82382 100644 --- a/examples/storm-perf/src/main/java/org/apache/storm/perf/bolt/CountBolt.java +++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/bolt/CountBolt.java @@ -28,7 +28,6 @@ import org.apache.storm.tuple.Tuple; import org.apache.storm.tuple.Values; - public class CountBolt extends BaseBasicBolt { public static final String FIELDS_WORD = "word"; public static final String FIELDS_COUNT = "count"; diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/bolt/DevNullBolt.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/bolt/DevNullBolt.java index cc181c04c47..6c0f6faf70c 100755 --- a/examples/storm-perf/src/main/java/org/apache/storm/perf/bolt/DevNullBolt.java +++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/bolt/DevNullBolt.java @@ -28,7 +28,6 @@ import org.apache.storm.utils.ObjectReader; import org.slf4j.LoggerFactory; - public class DevNullBolt extends BaseRichBolt { private static final org.slf4j.Logger LOG = LoggerFactory.getLogger(DevNullBolt.class); private OutputCollector collector; @@ -36,7 +35,8 @@ public class DevNullBolt extends BaseRichBolt { private int count = 0; @Override - public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) { + public void prepare(Map topoConf, TopologyContext context, + OutputCollector collector) { this.collector = collector; this.sleepNanos = ObjectReader.getLong(topoConf.get("nullbolt.sleep.micros"), 0L) * 1_000; } diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/bolt/IdBolt.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/bolt/IdBolt.java index 5387499bb8f..957ed7b1a75 100644 --- a/examples/storm-perf/src/main/java/org/apache/storm/perf/bolt/IdBolt.java +++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/bolt/IdBolt.java @@ -31,7 +31,8 @@ public class IdBolt extends BaseRichBolt { private OutputCollector collector; @Override - public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) { + public void prepare(Map topoConf, TopologyContext context, + OutputCollector collector) { this.collector = collector; } diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/bolt/SplitSentenceBolt.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/bolt/SplitSentenceBolt.java index 85a3f5aa9d4..64e002b48c6 100644 --- a/examples/storm-perf/src/main/java/org/apache/storm/perf/bolt/SplitSentenceBolt.java +++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/bolt/SplitSentenceBolt.java @@ -27,7 +27,6 @@ import org.apache.storm.tuple.Tuple; import org.apache.storm.tuple.Values; - public class SplitSentenceBolt extends BaseBasicBolt { public static final String FIELDS = "word"; diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/queuetest/Acker.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/queuetest/Acker.java index adcc3a3d6bb..a5635bffe8b 100644 --- a/examples/storm-perf/src/main/java/org/apache/storm/perf/queuetest/Acker.java +++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/queuetest/Acker.java @@ -33,7 +33,6 @@ class Acker extends MyThread { this.spoutInQ = spoutInQ; } - @Override public void run() { long start = System.currentTimeMillis(); diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/queuetest/JCQueuePerfTest.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/queuetest/JCQueuePerfTest.java index 2cbee19e340..883dc87cd7f 100644 --- a/examples/storm-perf/src/main/java/org/apache/storm/perf/queuetest/JCQueuePerfTest.java +++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/queuetest/JCQueuePerfTest.java @@ -47,8 +47,10 @@ public static void main(String[] args) throws Exception { private static void ackingProducerSimulation() { WaitStrategyPark ws = new WaitStrategyPark(100); StormMetricRegistry registry = new StormMetricRegistry(); - JCQueue spoutQ = new JCQueue("spoutQ", "spoutQ", 1024, 0, 100, ws, "test", "test", Collections.singletonList(1000), 1000, registry); - JCQueue ackQ = new JCQueue("ackQ", "ackQ", 1024, 0, 100, ws, "test", "test", Collections.singletonList(1000), 1000, registry); + JCQueue spoutQ = new JCQueue("spoutQ", "spoutQ", 1024, 0, 100, ws, "test", "test", + Collections.singletonList(1000), 1000, registry); + JCQueue ackQ = new JCQueue("ackQ", "ackQ", 1024, 0, 100, ws, "test", "test", Collections + .singletonList(1000), 1000, registry); final AckingProducer ackingProducer = new AckingProducer(spoutQ, ackQ); final Acker acker = new Acker(ackQ, spoutQ); @@ -61,7 +63,8 @@ private static void producerFwdConsumer(int prodBatchSz) { StormMetricRegistry registry = new StormMetricRegistry(); JCQueue q1 = new JCQueue("q1", "q1", 1024, 0, prodBatchSz, ws, "test", "test", Collections.singletonList(1000), 1000, registry); - JCQueue q2 = new JCQueue("q2", "q2", 1024, 0, prodBatchSz, ws, "test", "test", Collections.singletonList(1000), 1000, registry); + JCQueue q2 = new JCQueue("q2", "q2", 1024, 0, prodBatchSz, ws, "test", "test", Collections + .singletonList(1000), 1000, registry); final Producer prod = new Producer(q1); final Forwarder fwd = new Forwarder(q1, q2); @@ -70,9 +73,9 @@ private static void producerFwdConsumer(int prodBatchSz) { runAllThds(prod, fwd, cons); } - private static void oneProducer1Consumer(int prodBatchSz) { - JCQueue q1 = new JCQueue("q1", "q1", 50_000, 0, prodBatchSz, new WaitStrategyPark(100), "test", "test", + JCQueue q1 = new JCQueue("q1", "q1", 50_000, 0, prodBatchSz, new WaitStrategyPark(100), + "test", "test", Collections.singletonList(1000), 1000, new StormMetricRegistry()); final Producer prod1 = new Producer(q1); @@ -82,7 +85,8 @@ private static void oneProducer1Consumer(int prodBatchSz) { } private static void twoProducer1Consumer(int prodBatchSz) { - JCQueue q1 = new JCQueue("q1", "q1", 50_000, 0, prodBatchSz, new WaitStrategyPark(100), "test", "test", + JCQueue q1 = new JCQueue("q1", "q1", 50_000, 0, prodBatchSz, new WaitStrategyPark(100), + "test", "test", Collections.singletonList(1000), 1000, new StormMetricRegistry()); final Producer prod1 = new Producer(q1); @@ -93,7 +97,8 @@ private static void twoProducer1Consumer(int prodBatchSz) { } private static void threeProducer1Consumer(int prodBatchSz) { - JCQueue q1 = new JCQueue("q1", "q1", 50_000, 0, prodBatchSz, new WaitStrategyPark(100), "test", "test", + JCQueue q1 = new JCQueue("q1", "q1", 50_000, 0, prodBatchSz, new WaitStrategyPark(100), + "test", "test", Collections.singletonList(1000), 1000, new StormMetricRegistry()); final Producer prod1 = new Producer(q1); @@ -104,12 +109,13 @@ private static void threeProducer1Consumer(int prodBatchSz) { runAllThds(prod1, prod2, prod3, cons1); } - private static void oneProducer2Consumers(int prodBatchSz) { WaitStrategyPark ws = new WaitStrategyPark(100); StormMetricRegistry registry = new StormMetricRegistry(); - JCQueue q1 = new JCQueue("q1", "q1", 1024, 0, prodBatchSz, ws, "test", "test", Collections.singletonList(1000), 1000, registry); - JCQueue q2 = new JCQueue("q2", "q2", 1024, 0, prodBatchSz, ws, "test", "test", Collections.singletonList(1000), 1000, registry); + JCQueue q1 = new JCQueue("q1", "q1", 1024, 0, prodBatchSz, ws, "test", "test", Collections + .singletonList(1000), 1000, registry); + JCQueue q2 = new JCQueue("q2", "q2", 1024, 0, prodBatchSz, ws, "test", "test", Collections + .singletonList(1000), 1000, registry); final Producer2 prod1 = new Producer2(q1, q2); final Consumer cons1 = new Consumer(q1); @@ -140,7 +146,8 @@ public static void addShutdownHooks(MyThread... threads) { } for (MyThread thread : threads) { - System.err.printf("%s : %d, Throughput: %,d \n", thread.getName(), thread.count, thread.throughput()); + System.err.printf("%s : %d, Throughput: %,d \n", thread.getName(), + thread.count, thread.throughput()); } } catch (InterruptedException e) { return; diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/spout/ConstSpout.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/spout/ConstSpout.java index 656f992eff0..ed7ec079f2f 100755 --- a/examples/storm-perf/src/main/java/org/apache/storm/perf/spout/ConstSpout.java +++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/spout/ConstSpout.java @@ -52,7 +52,8 @@ public void declareOutputFields(OutputFieldsDeclarer declarer) { } @Override - public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { + public void open(Map conf, TopologyContext context, + SpoutOutputCollector collector) { this.collector = collector; this.sleep = ObjectReader.getLong(conf.get("spout.sleep"), 0L); } diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/spout/FileReadSpout.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/spout/FileReadSpout.java index a819fa25e18..bdfb27fa316 100644 --- a/examples/storm-perf/src/main/java/org/apache/storm/perf/spout/FileReadSpout.java +++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/spout/FileReadSpout.java @@ -44,7 +44,6 @@ public class FileReadSpout extends BaseRichSpout { private long count = 0; - public FileReadSpout(String file) { this.file = file; } diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/spout/StringGenSpout.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/spout/StringGenSpout.java index 1d0470c3eb6..d73e8613aec 100755 --- a/examples/storm-perf/src/main/java/org/apache/storm/perf/spout/StringGenSpout.java +++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/spout/StringGenSpout.java @@ -30,7 +30,8 @@ import org.apache.storm.tuple.Fields; /** - * Spout pre-computes a list with 30k fixed length random strings. Emits sequentially from this list, over and over again. + * Spout pre-computes a list with 30k fixed length random strings. Emits sequentially from this + * list, over and over again. */ public class StringGenSpout extends BaseRichSpout { @@ -67,7 +68,8 @@ public void declareOutputFields(OutputFieldsDeclarer declarer) { } @Override - public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { + public void open(Map conf, TopologyContext context, + SpoutOutputCollector collector) { this.records = genStringList(strLen, strCount); this.collector = collector; @@ -83,7 +85,6 @@ public void nextTuple() { } } - @Override public void ack(Object msgId) { super.ack(msgId); diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/spout/WordGenSpout.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/spout/WordGenSpout.java index a6a1fc9b391..c7ea29ac989 100644 --- a/examples/storm-perf/src/main/java/org/apache/storm/perf/spout/WordGenSpout.java +++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/spout/WordGenSpout.java @@ -45,7 +45,6 @@ public class WordGenSpout extends BaseRichSpout { private ThroughputMeter emitMeter; private ArrayList words; - public WordGenSpout(String file) { this.file = file; } diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/toolstest/JCToolsPerfTest.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/toolstest/JCToolsPerfTest.java index b5afe28b6a6..492a94549b4 100644 --- a/examples/storm-perf/src/main/java/org/apache/storm/perf/toolstest/JCToolsPerfTest.java +++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/toolstest/JCToolsPerfTest.java @@ -73,7 +73,6 @@ private static void threeProducer1Consumer() { runAllThds(prod1, prod2, prod3, cons1); } - private static void oneProducer2Consumers() { MpscArrayQueue q1 = new MpscArrayQueue(50_000); MpscArrayQueue q2 = new MpscArrayQueue(50_000); @@ -107,7 +106,8 @@ public static void addShutdownHooks(MyThd... threads) { } for (MyThd thread : threads) { - System.err.printf("%s : %d, Throughput: %,d \n", thread.getName(), thread.count, thread.throughput()); + System.err.printf("%s : %d, Throughput: %,d \n", thread.getName(), + thread.count, thread.throughput()); } } catch (InterruptedException e) { return; diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/utils/BasicMetricsCollector.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/utils/BasicMetricsCollector.java index 735abe01a86..e94ff7b94f8 100755 --- a/examples/storm-perf/src/main/java/org/apache/storm/perf/utils/BasicMetricsCollector.java +++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/utils/BasicMetricsCollector.java @@ -132,10 +132,12 @@ void updateTopologyStats() { void updateExecutorStats() { long timeDiff = this.curSample.getSampleTime() - this.lastSample.getSampleTime(); - long transferredDiff = this.curSample.getTotalTransferred() - this.lastSample.getTotalTransferred(); + long transferredDiff = this.curSample.getTotalTransferred() - this.lastSample + .getTotalTransferred(); long throughput = transferredDiff / (timeDiff / 1000); - long spoutDiff = this.curSample.getSpoutTransferred() - this.lastSample.getSpoutTransferred(); + long spoutDiff = this.curSample.getSpoutTransferred() - this.lastSample + .getSpoutTransferred(); long spoutAckedDiff = this.curSample.getTotalAcked() - this.lastSample.getTotalAcked(); long spoutThroughput = spoutDiff / (timeDiff / 1000); @@ -192,11 +194,13 @@ void writeHeader(PrintWriter writer) { } writer.println( - "\n------------------------------------------------------------------------------------------------------------------"); + "\n-----------------------------------------------------------------------------------" + + "-------------------------------"); String str = Utils.join(header, ","); writer.println(str); writer - .println("------------------------------------------------------------------------------------------------------------------"); + .println("----------------------------------------------------------------------------" + + "--------------------------------------"); writer.flush(); } diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/utils/Helper.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/utils/Helper.java index a573308b607..8df90cf6e30 100755 --- a/examples/storm-perf/src/main/java/org/apache/storm/perf/utils/Helper.java +++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/utils/Helper.java @@ -27,7 +27,6 @@ import org.apache.storm.utils.ObjectReader; import org.apache.storm.utils.Utils; - public class Helper { public static void kill(Nimbus.Iface client, String topoName) throws Exception { @@ -44,10 +43,12 @@ public static String getStr(Map map, Object key) { return (String) map.get(key); } - public static void collectMetricsAndKill(String topologyName, Integer pollInterval, int duration) throws Exception { + public static void collectMetricsAndKill(String topologyName, Integer pollInterval, + int duration) throws Exception { Map clusterConf = Utils.readStormConfig(); Nimbus.Iface client = NimbusClient.Builder.withConf(clusterConf).build().getClient(); - try (BasicMetricsCollector metricsCollector = new BasicMetricsCollector(topologyName, clusterConf)) { + try (BasicMetricsCollector metricsCollector = new BasicMetricsCollector(topologyName, + clusterConf)) { if (duration > 0) { int times = duration / pollInterval; @@ -57,7 +58,7 @@ public static void collectMetricsAndKill(String topologyName, Integer pollInterv metricsCollector.collect(client); } } else { - while (true) { //until Ctrl-C + while (true) { // until Ctrl-C metricsCollector.collect(client); Thread.sleep(pollInterval * 1000); } @@ -87,7 +88,8 @@ public void run() { }); } - public static void runOnClusterAndPrintMetrics(int durationSec, String topoName, Map topoConf, StormTopology topology) + public static void runOnClusterAndPrintMetrics(int durationSec, String topoName, Map topoConf, StormTopology topology) throws Exception { // submit topology StormSubmitter.submitTopologyWithProgressBar(topoName, topoConf, topology); diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/utils/IdentityBolt.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/utils/IdentityBolt.java index df27263b150..5c6d9d010bf 100755 --- a/examples/storm-perf/src/main/java/org/apache/storm/perf/utils/IdentityBolt.java +++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/utils/IdentityBolt.java @@ -29,7 +29,8 @@ public class IdentityBolt extends BaseRichBolt { private OutputCollector collector; @Override - public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) { + public void prepare(Map topoConf, TopologyContext context, + OutputCollector collector) { this.collector = collector; } diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/utils/MetricsSample.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/utils/MetricsSample.java index c411844a34b..3aac2c5222e 100755 --- a/examples/storm-perf/src/main/java/org/apache/storm/perf/utils/MetricsSample.java +++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/utils/MetricsSample.java @@ -131,7 +131,7 @@ private static MetricsSample getMetricsSample(TopologyInfo topInfo) { } } - Double total = 0d; + Double total = 0D; Map vals = spoutStats.get_complete_ms_avg().get(":all-time"); if (vals != null) { for (String key : vals.keySet()) { diff --git a/examples/storm-redis-examples/pom.xml b/examples/storm-redis-examples/pom.xml index 68db938f648..662f5e2a36c 100644 --- a/examples/storm-redis-examples/pom.xml +++ b/examples/storm-redis-examples/pom.xml @@ -89,6 +89,16 @@ + + org.openrewrite.maven + rewrite-maven-plugin + + + + org.codehaus.gmavenplus + gmavenplus-plugin + + org.apache.maven.plugins maven-checkstyle-plugin diff --git a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/tools/Base64ToBinaryStateMigrationUtil.java b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/tools/Base64ToBinaryStateMigrationUtil.java index 9d0d4c113d4..e3cafc3a2cc 100644 --- a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/tools/Base64ToBinaryStateMigrationUtil.java +++ b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/tools/Base64ToBinaryStateMigrationUtil.java @@ -21,7 +21,6 @@ import java.io.IOException; import java.util.HashMap; import java.util.Map; - import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; import org.apache.commons.cli.DefaultParser; @@ -38,7 +37,8 @@ import redis.clients.jedis.util.SafeEncoder; public class Base64ToBinaryStateMigrationUtil { - private static final Logger LOG = LoggerFactory.getLogger(Base64ToBinaryStateMigrationUtil.class); + private static final Logger LOG = LoggerFactory + .getLogger(Base64ToBinaryStateMigrationUtil.class); private static final String OPTION_REDIS_HOST_SHORT = "h"; private static final String OPTION_REDIS_HOST_LONG = "host"; private static final String OPTION_REDIS_PORT_SHORT = "p"; @@ -139,7 +139,8 @@ public static void main(String[] args) throws IOException, ParseException { .setTimeout(2000) .build(); - Base64ToBinaryStateMigrationUtil migrationUtil = new Base64ToBinaryStateMigrationUtil(jedisPoolConfig); + Base64ToBinaryStateMigrationUtil migrationUtil = + new Base64ToBinaryStateMigrationUtil(jedisPoolConfig); for (String namespace : namespaces) { migrationUtil.migrate(namespace); @@ -150,11 +151,16 @@ public static void main(String[] args) throws IOException, ParseException { private static Options buildOptions() { Options options = new Options(); - options.addOption(OPTION_NAMESPACE_SHORT, OPTION_NAMESPACE_LONG, true, "REQUIRED the list of namespace to migrate."); - options.addOption(OPTION_REDIS_HOST_SHORT, OPTION_REDIS_HOST_LONG, true, "Redis hostname (default: localhost)"); - options.addOption(OPTION_REDIS_PORT_SHORT, OPTION_REDIS_PORT_LONG, true, "Redis port (default: 6379)"); - options.addOption(null, OPTION_REDIS_PASSWORD_LONG, true, "Redis password (default: no password)"); - options.addOption(OPTION_REDIS_DB_NUM_SHORT, OPTION_REDIS_DB_NUM_LONG, true, "Redis DB number (default: 0)"); + options.addOption(OPTION_NAMESPACE_SHORT, OPTION_NAMESPACE_LONG, true, + "REQUIRED the list of namespace to migrate."); + options.addOption(OPTION_REDIS_HOST_SHORT, OPTION_REDIS_HOST_LONG, true, + "Redis hostname (default: localhost)"); + options.addOption(OPTION_REDIS_PORT_SHORT, OPTION_REDIS_PORT_LONG, true, + "Redis port (default: 6379)"); + options.addOption(null, OPTION_REDIS_PASSWORD_LONG, true, + "Redis password (default: no password)"); + options.addOption(OPTION_REDIS_DB_NUM_SHORT, OPTION_REDIS_DB_NUM_LONG, true, + "Redis DB number (default: 0)"); return options; } diff --git a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/topology/LookupWordCount.java b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/topology/LookupWordCount.java index 243f1bffc46..e7630dc06e3 100644 --- a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/topology/LookupWordCount.java +++ b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/topology/LookupWordCount.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -19,11 +19,9 @@ package org.apache.storm.redis.topology; import com.google.common.collect.Lists; - import java.util.List; import java.util.Map; import java.util.Random; - import org.apache.storm.Config; import org.apache.storm.StormSubmitter; import org.apache.storm.redis.bolt.RedisLookupBolt; @@ -39,7 +37,6 @@ import org.apache.storm.tuple.ITuple; import org.apache.storm.tuple.Tuple; import org.apache.storm.tuple.Values; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -57,7 +54,8 @@ public static class PrintWordTotalCountBolt extends BaseRichBolt { private OutputCollector collector; @Override - public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) { + public void prepare(Map topoConf, TopologyContext context, + OutputCollector collector) { this.collector = collector; } @@ -101,7 +99,7 @@ public static void main(String[] args) throws Exception { PrintWordTotalCountBolt printBolt = new PrintWordTotalCountBolt(); - //wordspout -> lookupbolt + // wordspout -> lookupbolt TopologyBuilder builder = new TopologyBuilder(); builder.setSpout(WORD_SPOUT, spout, 1); builder.setBolt(LOOKUP_BOLT, lookupBolt, 1).shuffleGrouping(WORD_SPOUT); diff --git a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/topology/PersistentWordCount.java b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/topology/PersistentWordCount.java index 93f790b9921..3b4e7f94ee2 100644 --- a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/topology/PersistentWordCount.java +++ b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/topology/PersistentWordCount.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -64,7 +64,8 @@ public static void main(String[] args) throws Exception { if (args.length == 3) { topoName = args[2]; } else if (args.length > 3) { - System.out.println("Usage: PersistentWordCount (topology name)"); + System.out + .println("Usage: PersistentWordCount (topology name)"); return; } Config config = new Config(); diff --git a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/topology/WhitelistWordCount.java b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/topology/WhitelistWordCount.java index 5b47f0cd51e..e600f1192c6 100644 --- a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/topology/WhitelistWordCount.java +++ b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/topology/WhitelistWordCount.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -20,7 +20,6 @@ import java.util.Map; import java.util.Random; - import org.apache.storm.Config; import org.apache.storm.StormSubmitter; import org.apache.storm.redis.bolt.RedisFilterBolt; @@ -53,7 +52,8 @@ public static class PrintWordTotalCountBolt extends BaseRichBolt { private OutputCollector collector; @Override - public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) { + public void prepare(Map topoConf, TopologyContext context, + OutputCollector collector) { this.collector = collector; } @@ -99,7 +99,8 @@ public static void main(String[] args) throws Exception { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout(WORD_SPOUT, spout, 1); builder.setBolt(WHITELIST_BOLT, whitelistBolt, 1).shuffleGrouping(WORD_SPOUT); - builder.setBolt(COUNT_BOLT, wordCounterBolt, 1).fieldsGrouping(WHITELIST_BOLT, new Fields("word")); + builder.setBolt(COUNT_BOLT, wordCounterBolt, 1).fieldsGrouping(WHITELIST_BOLT, + new Fields("word")); PrintWordTotalCountBolt printBolt = new PrintWordTotalCountBolt(); builder.setBolt(PRINT_BOLT, printBolt, 1).shuffleGrouping(COUNT_BOLT); @@ -107,7 +108,8 @@ public static void main(String[] args) throws Exception { if (args.length == 3) { topoName = args[2]; } else if (args.length > 3) { - System.out.println("Usage: WhitelistWordCount [topology name]"); + System.out + .println("Usage: WhitelistWordCount [topology name]"); return; } Config config = new Config(); diff --git a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/topology/WordCounter.java b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/topology/WordCounter.java index a0b9714c804..2d102234215 100644 --- a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/topology/WordCounter.java +++ b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/topology/WordCounter.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -19,9 +19,7 @@ package org.apache.storm.redis.topology; import com.google.common.collect.Maps; - import java.util.Map; - import org.apache.storm.task.TopologyContext; import org.apache.storm.topology.BasicOutputCollector; import org.apache.storm.topology.IBasicBolt; diff --git a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/topology/WordSpout.java b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/topology/WordSpout.java index ff3ec39101b..78f8f84bbcc 100644 --- a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/topology/WordSpout.java +++ b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/topology/WordSpout.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -21,7 +21,6 @@ import java.util.Map; import java.util.Random; import java.util.UUID; - import org.apache.storm.spout.SpoutOutputCollector; import org.apache.storm.task.TopologyContext; import org.apache.storm.topology.IRichSpout; @@ -32,7 +31,8 @@ public class WordSpout implements IRichSpout { boolean isDistributed; SpoutOutputCollector collector; - public static final String[] words = new String[] { "apple", "orange", "pineapple", "banana", "watermelon" }; + public static final String[] words = + new String[] { "apple", "orange", "pineapple", "banana", "watermelon" }; public WordSpout() { this(true); @@ -47,7 +47,8 @@ public boolean isDistributed() { } @Override - public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { + public void open(Map conf, TopologyContext context, + SpoutOutputCollector collector) { this.collector = collector; } diff --git a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/PrintFunction.java b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/PrintFunction.java index f32cfe8e037..b7229d09e97 100644 --- a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/PrintFunction.java +++ b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/PrintFunction.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -19,11 +19,9 @@ package org.apache.storm.redis.trident; import java.util.Random; - import org.apache.storm.trident.operation.BaseFunction; import org.apache.storm.trident.operation.TridentCollector; import org.apache.storm.trident.tuple.TridentTuple; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountLookupMapper.java b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountLookupMapper.java index 04874d5f13c..909ffb6330b 100644 --- a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountLookupMapper.java +++ b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountLookupMapper.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -20,7 +20,6 @@ import java.util.ArrayList; import java.util.List; - import org.apache.storm.redis.common.mapper.RedisDataTypeDescription; import org.apache.storm.redis.common.mapper.RedisLookupMapper; import org.apache.storm.topology.OutputFieldsDeclarer; diff --git a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountStoreMapper.java b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountStoreMapper.java index 84538e232e4..d99454ad3e4 100644 --- a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountStoreMapper.java +++ b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountStoreMapper.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and diff --git a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountTridentRedis.java b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountTridentRedis.java index 1e9991fd3aa..4302486bb2f 100644 --- a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountTridentRedis.java +++ b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountTridentRedis.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -82,6 +82,7 @@ public static void main(String[] args) throws Exception { Config conf = new Config(); conf.setMaxSpoutPending(5); conf.setNumWorkers(3); - StormSubmitter.submitTopology("test_wordCounter_for_redis", conf, buildTopology(redisHost, redisPort)); + StormSubmitter.submitTopology("test_wordCounter_for_redis", conf, buildTopology(redisHost, + redisPort)); } } diff --git a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountTridentRedisCluster.java b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountTridentRedisCluster.java index 7ced2afdadf..fe8fdbbeeb7 100644 --- a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountTridentRedisCluster.java +++ b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountTridentRedisCluster.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -21,7 +21,6 @@ import java.net.InetSocketAddress; import java.util.HashSet; import java.util.Set; - import org.apache.storm.Config; import org.apache.storm.StormSubmitter; import org.apache.storm.generated.StormTopology; @@ -89,7 +88,8 @@ public static void main(String[] args) throws Exception { Config conf = new Config(); conf.setMaxSpoutPending(5); conf.setNumWorkers(3); - StormSubmitter.submitTopology("test_wordCounter_for_redis", conf, buildTopology(redisHostPort)); + StormSubmitter.submitTopology("test_wordCounter_for_redis", conf, + buildTopology(redisHostPort)); } } diff --git a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountTridentRedisClusterMap.java b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountTridentRedisClusterMap.java index b1425c9bf34..9b5cadb3261 100644 --- a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountTridentRedisClusterMap.java +++ b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountTridentRedisClusterMap.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -21,7 +21,6 @@ import java.net.InetSocketAddress; import java.util.HashSet; import java.util.Set; - import org.apache.storm.Config; import org.apache.storm.StormSubmitter; import org.apache.storm.generated.StormTopology; @@ -59,7 +58,8 @@ public static StormTopology buildTopology(String redisHostPort) { .build(); RedisDataTypeDescription dataTypeDescription = new RedisDataTypeDescription( RedisDataTypeDescription.RedisDataType.HASH, "test"); - StateFactory factory = RedisClusterMapState.transactional(clusterConfig, dataTypeDescription); + StateFactory factory = RedisClusterMapState.transactional(clusterConfig, + dataTypeDescription); TridentTopology topology = new TridentTopology(); Stream stream = topology.newStream("spout1", spout); @@ -83,7 +83,8 @@ public static void main(String[] args) throws Exception { Config conf = new Config(); conf.setMaxSpoutPending(5); conf.setNumWorkers(3); - StormSubmitter.submitTopology("test_wordCounter_for_redis", conf, buildTopology(redisHostPort)); + StormSubmitter.submitTopology("test_wordCounter_for_redis", conf, + buildTopology(redisHostPort)); } } diff --git a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountTridentRedisMap.java b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountTridentRedisMap.java index 2a7af97f3b8..02d34d9fcb4 100644 --- a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountTridentRedisMap.java +++ b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountTridentRedisMap.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -77,7 +77,8 @@ public static void main(String[] args) throws Exception { Config conf = new Config(); conf.setMaxSpoutPending(5); conf.setNumWorkers(3); - StormSubmitter.submitTopology("test_wordCounter_for_redis", conf, buildTopology(redisHost, redisPort)); + StormSubmitter.submitTopology("test_wordCounter_for_redis", conf, buildTopology(redisHost, + redisPort)); } } diff --git a/examples/storm-starter/pom.xml b/examples/storm-starter/pom.xml index f7632397f7a..b6dc1e66c3d 100644 --- a/examples/storm-starter/pom.xml +++ b/examples/storm-starter/pom.xml @@ -174,6 +174,16 @@ none + + org.openrewrite.maven + rewrite-maven-plugin + + + + org.codehaus.gmavenplus + gmavenplus-plugin + + org.apache.maven.plugins maven-checkstyle-plugin diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/AnchoredWordCount.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/AnchoredWordCount.java index 1809cf19fa1..d6f7d63e905 100644 --- a/examples/storm-starter/src/jvm/org/apache/storm/starter/AnchoredWordCount.java +++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/AnchoredWordCount.java @@ -1,12 +1,17 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -58,7 +63,6 @@ public static class RandomSentenceSpout extends BaseRichSpout { SpoutOutputCollector collector; Random random; - @Override public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { this.collector = collector; @@ -69,7 +73,8 @@ public void open(Map conf, TopologyContext context, SpoutOutputCollector collect public void nextTuple() { Utils.sleep(10); String[] sentences = new String[]{ - sentence("the cow jumped over the moon"), sentence("an apple a day keeps the doctor away"), + sentence("the cow jumped over the moon"), sentence("an apple a day keeps the " + + "doctor away"), sentence("four score and seven years ago"), sentence("snow white and the seven dwarfs"), sentence("i am at two with nature") }; diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/BasicDRPCTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/BasicDRPCTopology.java index 70e03a4761e..f647fa7d34c 100644 --- a/examples/storm-starter/src/jvm/org/apache/storm/starter/BasicDRPCTopology.java +++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/BasicDRPCTopology.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -24,7 +30,8 @@ import org.apache.storm.utils.DRPCClient; /** - * This topology is a basic example of doing distributed RPC on top of Storm. It implements a function that appends a + * This topology is a basic example of doing distributed RPC on top of Storm. It implements a + * function that appends a * "!" to any string you send the DRPC function. * * @see Distributed RPC @@ -48,13 +55,15 @@ public static void main(String[] args) throws Exception { builder.addBolt(new ExclaimBolt(), 3); conf.setNumWorkers(3); - StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder.createRemoteTopology()); + StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder + .createRemoteTopology()); if (args != null && args.length > 2) { try (DRPCClient drpc = DRPCClient.getConfiguredClient(conf)) { for (int i = 2; i < args.length; i++) { String word = args[i]; - System.out.println("Result for \"" + word + "\": " + drpc.execute(function, word)); + System.out.println("Result for \"" + word + "\": " + drpc.execute(function, + word)); } } } diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/BlobStoreAPIWordCountTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/BlobStoreAPIWordCountTopology.java index f8059f3c1fc..be2b89292be 100644 --- a/examples/storm-starter/src/jvm/org/apache/storm/starter/BlobStoreAPIWordCountTopology.java +++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/BlobStoreAPIWordCountTopology.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -69,7 +75,8 @@ public static void prepare() { // Equivalent create command on command line // storm blobstore create --file blacklist.txt --acl o::rwa key - private static void createBlobWithContent(String blobKey, ClientBlobStore clientBlobStore, File file) + private static void createBlobWithContent(String blobKey, ClientBlobStore clientBlobStore, + File file) throws AuthorizationException, KeyAlreadyExistsException, IOException, KeyNotFoundException { String stringBlobAcl = "o::rwa"; AccessControl blobAcl = BlobStoreAclHandler.parseAccessControl(stringBlobAcl); @@ -83,7 +90,8 @@ private static void createBlobWithContent(String blobKey, ClientBlobStore client // Equivalent update command on command line // storm blobstore update --file blacklist.txt key - private static void updateBlobWithContent(String blobKey, ClientBlobStore clientBlobStore, File file) + private static void updateBlobWithContent(String blobKey, ClientBlobStore clientBlobStore, + File file) throws KeyNotFoundException, AuthorizationException, IOException { AtomicOutputStream blobOutputStream = clientBlobStore.updateBlob(blobKey); blobOutputStream.write(readFile(file).toString().getBytes()); @@ -93,7 +101,8 @@ private static void updateBlobWithContent(String blobKey, ClientBlobStore client private static String getRandomSentence() { String[] sentences = new String[]{ "the cow jumped over the moon", "an apple a day keeps the doctor away", - "four score and seven years ago", "snow white and the seven dwarfs", "i am at two with nature" + "four score and seven years ago", "snow white and the seven dwarfs", "i am at two " + + "with nature" }; String sentence = sentences[new Random().nextInt(sentences.length)]; return sentence; @@ -218,7 +227,8 @@ public static class RandomSentenceSpout extends BaseRichSpout { SpoutOutputCollector collector; @Override - public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { + public void open(Map conf, TopologyContext context, + SpoutOutputCollector collector) { this.collector = collector; } diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/ExclamationTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/ExclamationTopology.java index 31854f613f3..1157b12641c 100644 --- a/examples/storm-starter/src/jvm/org/apache/storm/starter/ExclamationTopology.java +++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/ExclamationTopology.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -58,7 +64,8 @@ public static class ExclamationBolt extends BaseRichBolt { OutputCollector collector; @Override - public void prepare(Map conf, TopologyContext context, OutputCollector collector) { + public void prepare(Map conf, TopologyContext context, + OutputCollector collector) { this.collector = collector; } diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/FastWordCountTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/FastWordCountTopology.java index e5da703380b..a62656515b7 100644 --- a/examples/storm-starter/src/jvm/org/apache/storm/starter/FastWordCountTopology.java +++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/FastWordCountTopology.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -107,7 +113,7 @@ public static void main(String[] args) throws Exception { clusterConf.putAll(Utils.readCommandLineOpts()); Nimbus.Iface client = NimbusClient.Builder.withConf(clusterConf).build().getClient(); - //Sleep for 5 mins + // Sleep for 5 mins for (int i = 0; i < 10; i++) { Thread.sleep(30 * 1000); printMetrics(client, name); @@ -127,7 +133,8 @@ public static class FastRandomSentenceSpout extends BaseRichSpout { Random rand; @Override - public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { + public void open(Map conf, TopologyContext context, + SpoutOutputCollector collector) { this.collector = collector; rand = ThreadLocalRandom.current(); } @@ -140,7 +147,7 @@ public void nextTuple() { @Override public void ack(Object id) { - //Ignored + // Ignored } @Override diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/InOrderDeliveryTest.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/InOrderDeliveryTest.java index 4820aef2026..6e5c141fb16 100644 --- a/examples/storm-starter/src/jvm/org/apache/storm/starter/InOrderDeliveryTest.java +++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/InOrderDeliveryTest.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -100,7 +106,7 @@ public static void main(String[] args) throws Exception { clusterConf.putAll(Utils.readCommandLineOpts()); Nimbus.Iface client = NimbusClient.Builder.withConf(clusterConf).build().getClient(); - //Sleep for 50 mins + // Sleep for 50 mins for (int i = 0; i < 50; i++) { Thread.sleep(30 * 1000); printMetrics(client, name); @@ -114,7 +120,8 @@ public static class InOrderSpout extends BaseRichSpout { int count = 0; @Override - public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { + public void open(Map conf, TopologyContext context, + SpoutOutputCollector collector) { this.collector = collector; base = context.getThisTaskIndex(); } @@ -128,12 +135,12 @@ public void nextTuple() { @Override public void ack(Object id) { - //Ignored + // Ignored } @Override public void fail(Object id) { - //Ignored + // Ignored } @Override @@ -163,7 +170,7 @@ public void execute(Tuple tuple, BasicOutputCollector collector) { @Override public void declareOutputFields(OutputFieldsDeclarer declarer) { - //Empty + // Empty } } } diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/JoinBoltExample.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/JoinBoltExample.java index 6fc373970bf..ccaee7a8584 100644 --- a/examples/storm-starter/src/jvm/org/apache/storm/starter/JoinBoltExample.java +++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/JoinBoltExample.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -50,7 +56,8 @@ public static void main(String[] args) throws Exception { builder.setBolt("printer", new PrinterBolt()).shuffleGrouping("joiner"); Config conf = new Config(); - StormSubmitter.submitTopologyWithProgressBar("join-example", conf, builder.createTopology()); + StormSubmitter.submitTopologyWithProgressBar("join-example", conf, builder + .createTopology()); generateGenderData(genderSpout); diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/LambdaTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/LambdaTopology.java index d5dec296896..78a7b9c74c5 100644 --- a/examples/storm-starter/src/jvm/org/apache/storm/starter/LambdaTopology.java +++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/LambdaTopology.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/ManualDRPC.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/ManualDRPC.java index e685ca16c2e..640404667a3 100644 --- a/examples/storm-starter/src/jvm/org/apache/storm/starter/ManualDRPC.java +++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/ManualDRPC.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/MultiThreadWordCountTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/MultiThreadWordCountTopology.java index 528b98d2b9f..b89c4591d7d 100644 --- a/examples/storm-starter/src/jvm/org/apache/storm/starter/MultiThreadWordCountTopology.java +++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/MultiThreadWordCountTopology.java @@ -30,8 +30,10 @@ Licensed to the Apache Software Foundation (ASF) under one or more contributor l import org.apache.storm.tuple.Values; /** - * Some topologies might spawn some threads within bolts to do some work and emit tuples from those threads. - * This is a simple wordcount topology example that mimics those use cases and might help us catch possible race conditions. + * Some topologies might spawn some threads within bolts to do some work and emit tuples from those + * threads. + * This is a simple wordcount topology example that mimics those use cases and might help us catch + * possible race conditions. */ public class MultiThreadWordCountTopology extends ConfigurableTopology { public static void main(String[] args) { @@ -45,12 +47,14 @@ protected int run(String[] args) { builder.setSpout("spout", new RandomSentenceSpout(), 1); builder.setBolt("split", new MultiThreadedSplitSentence(), 1).shuffleGrouping("spout"); - builder.setBolt("count", new WordCountBolt(), 1).fieldsGrouping("split", new Fields("word")); + builder.setBolt("count", new WordCountBolt(), 1).fieldsGrouping("split", + new Fields("word")); - //this makes sure there is only one executor per worker, easier to debug - //problems involving serialization/deserialization will only happen in inter-worker data transfer + // this makes sure there is only one executor per worker, easier to debug + // problems involving serialization/deserialization will only happen in inter-worker data + // transfer conf.put(Config.TOPOLOGY_RAS_ONE_EXECUTOR_PER_WORKER, true); - //this involves metricsTick + // this involves metricsTick conf.registerMetricsConsumer(LoggingMetricsConsumer.class); conf.setTopologyWorkerMaxHeapSize(128); @@ -69,11 +73,13 @@ public static class MultiThreadedSplitSentence implements IRichBolt { private ExecutorService executor; @Override - public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) { + public void prepare(Map topoConf, TopologyContext context, + OutputCollector collector) { this.collector = collector; executor = Executors.newFixedThreadPool(6); - //This makes sure metricsTick to be called every 1 second - //it makes the race condition between metricsTick and outputCollector easier to happen if any + // This makes sure metricsTick to be called every 1 second + // it makes the race condition between metricsTick and outputCollector easier to happen + // if any context.registerMetric("dummy-counter", () -> 0, 1); } @@ -82,7 +88,7 @@ public void execute(Tuple input) { String str = input.getString(0); String[] splits = str.split("\\s+"); for (String s : splits) { - //spawn other threads to do the work and emit + // spawn other threads to do the work and emit Runnable runnableTask = () -> { collector.emit(new Values(s)); }; diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/MultipleLoggerTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/MultipleLoggerTopology.java index 9410931553f..f7491b8994d 100644 --- a/examples/storm-starter/src/jvm/org/apache/storm/starter/MultipleLoggerTopology.java +++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/MultipleLoggerTopology.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -57,7 +63,8 @@ public static class ExclamationLoggingBolt extends BaseRichBolt { Logger subLogger = LoggerFactory.getLogger("com.myapp.sub"); @Override - public void prepare(Map conf, TopologyContext context, OutputCollector collector) { + public void prepare(Map conf, TopologyContext context, + OutputCollector collector) { this.collector = collector; } diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/PersistentWindowingTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/PersistentWindowingTopology.java index 62b3ed577ec..37f0eabacd8 100644 --- a/examples/storm-starter/src/jvm/org/apache/storm/starter/PersistentWindowingTopology.java +++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/PersistentWindowingTopology.java @@ -41,14 +41,18 @@ import org.slf4j.LoggerFactory; /** - * An example that demonstrates the usage of {@link org.apache.storm.topology.IStatefulWindowedBolt} with window persistence. - *

- * The framework automatically checkpoints the tuples in the window along with the bolt's state and restores the same during restarts. + * An example that demonstrates the usage of {@link org.apache.storm.topology.IStatefulWindowedBolt} + * with window persistence. + * + *

The framework automatically checkpoints the tuples in the window along with the bolt's state + * and + * restores the same during restarts. *

* - *

- * This topology uses 'redis' for state persistence, so you should also start a redis instance before deploying. If you are running in local - * mode you can just start a redis server locally which will be used for storing the state. The default RedisKeyValueStateProvider + *

This topology uses 'redis' for state persistence, so you should also start a redis instance + * before deploying. If you are running in local + * mode you can just start a redis server locally which will be used for storing the state. The + * default RedisKeyValueStateProvider * parameters can be overridden by setting {@link Config#TOPOLOGY_STATE_PROVIDER_CONFIG}, for e.g. *

  * {
@@ -88,7 +92,8 @@ public static void main(String[] args) throws Exception {
                .shuffleGrouping("spout");
 
         // print the values to stdout
-        builder.setBolt("printer", (x, y) -> System.out.println(x.getValue(0)), 1).shuffleGrouping("avgbolt");
+        builder.setBolt("printer", (x, y) -> System.out.println(x.getValue(0)), 1)
+                .shuffleGrouping("avgbolt");
 
         Config conf = new Config();
         conf.setDebug(false);
@@ -97,7 +102,8 @@ public static void main(String[] args) throws Exception {
         conf.put(Config.TOPOLOGY_STATE_CHECKPOINT_INTERVAL, 5000);
 
         // use redis for state persistence
-        conf.put(Config.TOPOLOGY_STATE_PROVIDER, "org.apache.storm.redis.state.RedisKeyValueStateProvider");
+        conf.put(Config.TOPOLOGY_STATE_PROVIDER,
+                "org.apache.storm.redis.state.RedisKeyValueStateProvider");
 
         String topoName = "test";
         if (args != null && args.length > 0) {
@@ -119,7 +125,8 @@ private static class Averages {
 
         @Override
         public String toString() {
-            return "Averages{" + "global=" + String.format("%.2f", global) + ", window=" + String.format("%.2f", window) + '}';
+            return "Averages{" + "global=" + String.format("%.2f", global) + ", window=" + String
+                    .format("%.2f", window) + '}';
         }
     }
 
@@ -134,7 +141,8 @@ private static class AvgBolt extends BaseStatefulWindowedBolt globalAvg;
 
         @Override
-        public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) {
+        public void prepare(Map topoConf, TopologyContext context,
+            OutputCollector collector) {
             this.collector = collector;
         }
 
@@ -142,7 +150,8 @@ public void prepare(Map topoConf, TopologyContext context, Outpu
         public void initState(KeyValueState> state) {
             this.state = state;
             globalAvg = state.get(STATE_KEY, Pair.of(0L, 0L));
-            LOG.info("initState with global avg [" + (double) globalAvg.getFirst() / globalAvg.getSecond() + "]");
+            LOG.info("initState with global avg [" + (double) globalAvg.getFirst() / globalAvg
+                    .getSecond() + "]");
         }
 
         @Override
@@ -161,7 +170,8 @@ public void execute(TupleWindow window) {
             // update the value in state
             state.put(STATE_KEY, globalAvg);
             // emit the averages downstream
-            collector.emit(new Values(new Averages((double) globalAvg.getFirst() / globalAvg.getSecond(), (double) sum / count)));
+            collector.emit(new Values(new Averages((double) globalAvg.getFirst() / globalAvg
+                    .getSecond(), (double) sum / count)));
         }
 
         @Override
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/Prefix.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/Prefix.java
index 31f200a83dc..1ce4d9e9de7 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/Prefix.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/Prefix.java
@@ -1,12 +1,18 @@
 /**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.  The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License.  You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
  *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * 

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/ReachTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/ReachTopology.java index 59051326942..f2a1bc2ac32 100644 --- a/examples/storm-starter/src/jvm/org/apache/storm/starter/ReachTopology.java +++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/ReachTopology.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -33,18 +39,24 @@ import org.apache.storm.utils.DRPCClient; /** - * This is a good example of doing complex Distributed RPC on top of Storm. This program creates a topology that can + * This is a good example of doing complex Distributed RPC on top of Storm. This program creates a + * topology that can * compute the reach for any URL on Twitter in realtime by parallelizing the whole computation. * - *

Reach is the number of unique people exposed to a URL on Twitter. To compute reach, you have to get all the people - * who tweeted the URL, get all the followers of all those people, unique that set of followers, and then count the - * unique set. It's an intense computation that can involve thousands of database calls and tens of millions of follower + *

Reach is the number of unique people exposed to a URL on Twitter. To compute reach, you have + * to get all the people + * who tweeted the URL, get all the followers of all those people, unique that set of followers, and + * then count the + * unique set. It's an intense computation that can involve thousands of database calls and tens of + * millions of follower * records. * - *

This Storm topology does every piece of that computation in parallel, turning what would be a computation that takes + *

This Storm topology does every piece of that computation in parallel, turning what would be a + * computation that takes * minutes on a single machine into one that takes just a couple seconds. * - *

For the purposes of demonstration, this topology replaces the use of actual DBs with in-memory hashmaps. + *

For the purposes of demonstration, this topology replaces the use of actual DBs with in-memory + * hashmaps. * * @see Distributed RPC */ @@ -52,7 +64,8 @@ public class ReachTopology { public static Map> TWEETERS_DB = new HashMap>() { { put("foo.com/blog/1", Arrays.asList("sally", "bob", "tim", "george", "nathan")); - put("engineering.twitter.com/blog/5", Arrays.asList("adam", "david", "sally", "nathan")); + put("engineering.twitter.com/blog/5", Arrays.asList("adam", "david", "sally", + "nathan")); put("tech.backtype.com/blog/123", Arrays.asList("tim", "mike", "john")); } }; @@ -62,7 +75,8 @@ public class ReachTopology { put("sally", Arrays.asList("bob", "tim", "alice", "adam", "jim", "chris", "jai")); put("bob", Arrays.asList("sally", "nathan", "jim", "mary", "david", "vivian")); put("tim", Arrays.asList("alex")); - put("nathan", Arrays.asList("sally", "bob", "adam", "harry", "chris", "vivian", "emily", "jordan")); + put("nathan", Arrays.asList("sally", "bob", "adam", "harry", "chris", "vivian", "emily", + "jordan")); put("adam", Arrays.asList("david", "carissa")); put("mike", Arrays.asList("john", "bob")); put("john", Arrays.asList("alice", "nathan", "jim", "mike", "bob")); @@ -87,10 +101,12 @@ public static void main(String[] args) throws Exception { if (args.length > 0) { topoName = args[0]; } - StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder.createRemoteTopology()); + StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder + .createRemoteTopology()); try (DRPCClient drpc = DRPCClient.getConfiguredClient(conf)) { - String[] urlsToTry = new String[]{ "foo.com/blog/1", "engineering.twitter.com/blog/5", "notaurl.com" }; + String[] urlsToTry = + new String[]{ "foo.com/blog/1", "engineering.twitter.com/blog/5", "notaurl.com" }; for (String url : urlsToTry) { System.out.println("Reach of " + url + ": " + drpc.execute("reach", url)); } @@ -141,7 +157,8 @@ public static class PartialUniquer extends BaseBatchBolt { Set followers = new HashSet(); @Override - public void prepare(Map conf, TopologyContext context, BatchOutputCollector collector, Object id) { + public void prepare(Map conf, TopologyContext context, + BatchOutputCollector collector, Object id) { this.collector = collector; this.id = id; } @@ -168,7 +185,8 @@ public static class CountAggregator extends BaseBatchBolt { int count = 0; @Override - public void prepare(Map conf, TopologyContext context, BatchOutputCollector collector, Object id) { + public void prepare(Map conf, TopologyContext context, + BatchOutputCollector collector, Object id) { this.collector = collector; this.id = id; } diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/ResourceAwareExampleTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/ResourceAwareExampleTopology.java index 20f951e7602..92659f5b667 100644 --- a/examples/storm-starter/src/jvm/org/apache/storm/starter/ResourceAwareExampleTopology.java +++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/ResourceAwareExampleTopology.java @@ -1,20 +1,26 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.starter; import java.util.Iterator; -import java.util.Map; import java.util.Map.Entry; +import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import org.apache.storm.Config; import org.apache.storm.StormSubmitter; @@ -35,13 +41,13 @@ public class ResourceAwareExampleTopology { public static void main(String[] args) throws Exception { TopologyBuilder builder = new TopologyBuilder(); - //A topology can set resources in terms of CPU and Memory for each component + // A topology can set resources in terms of CPU and Memory for each component // These can be chained (like with setting the CPU requirement) SpoutDeclarer spout = builder.setSpout("word", new TestWordSpout(), 10).setCPULoad(20); // Or done separately like with setting the // onheap and offheap memory requirement spout.setMemoryLoad(64, 16); - //On heap memory is used to help calculate the heap of the java process for the worker + // On heap memory is used to help calculate the heap of the java process for the worker // off heap memory is for things like JNI memory allocated off heap, or when using the // ShellBolt or ShellSpout. In this case the 16 MB of off heap is just as an example // as we are not using it. @@ -54,7 +60,8 @@ public static void main(String[] args) throws Exception { SharedOffHeapWithinNode notImplementedButJustAnExample = new SharedOffHeapWithinNode(500, "not-implemented-node-level-cache"); - //If CPU or memory is not set the values stored in topology.component.resources.onheap.memory.mb, + // If CPU or memory is not set the values stored in + // topology.component.resources.onheap.memory.mb, // topology.component.resources.offheap.memory.mb and topology.component.cpu.pcore.percent // will be used instead builder @@ -72,26 +79,34 @@ public static void main(String[] args) throws Exception { Config conf = new Config(); conf.setDebug(true); - //Under RAS the number of workers is determined by the scheduler and the settings in the conf are ignored - //conf.setNumWorkers(3); + // Under RAS the number of workers is determined by the scheduler and the settings in the + // conf are ignored + // conf.setNumWorkers(3); - //Instead the scheduler lets you set the maximum heap size for any worker. + // Instead the scheduler lets you set the maximum heap size for any worker. conf.setTopologyWorkerMaxHeapSize(1024.0); - //The scheduler generally will try to pack executors into workers until the max heap size is met, but + // The scheduler generally will try to pack executors into workers until the max heap size + // is met, but // this can vary depending on the specific scheduling strategy selected. - // The reason for this is to try and balance the maximum pause time GC might take (which is larger for larger heaps) + // The reason for this is to try and balance the maximum pause time GC might take (which is + // larger for larger heaps) // against better performance because of not needing to serialize/deserialize tuples. - //The priority of a topology describes the importance of the topology in decreasing importance - // starting from 0 (i.e. 0 is the highest priority and the priority importance decreases as the priority number increases). - //Recommended range of 0-29 but no hard limit set. - // If there are not enough resources in a cluster the priority in combination with how far over a guarantees + // The priority of a topology describes the importance of the topology in decreasing + // importance + // starting from 0 (i.e. 0 is the highest priority and the priority importance decreases as + // the priority number increases). + // Recommended range of 0-29 but no hard limit set. + // If there are not enough resources in a cluster the priority in combination with how far + // over a guarantees // a user is will decide which topologies are run and which ones are not. conf.setTopologyPriority(29); - //set to use the default resource aware strategy when using the MultitenantResourceAwareBridgeScheduler + // set to use the default resource aware strategy when using the + // MultitenantResourceAwareBridgeScheduler conf.setTopologyStrategy( - "org.apache.storm.scheduler.resource.strategies.scheduling.DefaultResourceAwareStrategy"); + "org.apache.storm.scheduler.resource.strategies.scheduling.DefaultResourceAwareStrateg" + + "y"); String topoName = "test"; if (args != null && args.length > 0) { @@ -102,7 +117,7 @@ public static void main(String[] args) throws Exception { } public static class ExclamationBolt extends BaseRichBolt { - //Have a crummy cache to show off shared memory accounting + // Have a crummy cache to show off shared memory accounting private static final ConcurrentHashMap myCrummyCache = new ConcurrentHashMap<>(); private static final int CACHE_SIZE = 100_000; @@ -116,7 +131,7 @@ protected static void addToCache(String key, String value) { myCrummyCache.putIfAbsent(key, value); int numToRemove = myCrummyCache.size() - CACHE_SIZE; if (numToRemove > 0) { - //Remove something randomly... + // Remove something randomly... Iterator> it = myCrummyCache.entrySet().iterator(); for (; numToRemove > 0 && it.hasNext(); numToRemove--) { it.next(); @@ -126,7 +141,8 @@ protected static void addToCache(String key, String value) { } @Override - public void prepare(Map conf, TopologyContext context, OutputCollector collector) { + public void prepare(Map conf, TopologyContext context, + OutputCollector collector) { this.collector = collector; } diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/RollingTopWords.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/RollingTopWords.java index ea40e8bfaad..79ffdaf26cf 100644 --- a/examples/storm-starter/src/jvm/org/apache/storm/starter/RollingTopWords.java +++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/RollingTopWords.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -51,7 +57,8 @@ public static void main(String[] args) throws Exception { *

Examples: * ``` * # Runs in remote/cluster mode, with topology name "production-topology" - * $ storm jar storm-starter-jar-with-dependencies.jar org.apache.storm.starter.RollingTopWords production-topology ``` + * $ storm jar storm-starter-jar-with-dependencies.jar org.apache.storm.starter.RollingTopWords + * production-topology ``` * * @param args * First positional argument (optional) is topology name, second @@ -69,11 +76,14 @@ protected int run(String[] args) { String counterId = "counter"; String intermediateRankerId = "intermediateRanker"; builder.setSpout(spoutId, new TestWordSpout(), 5); - builder.setBolt(counterId, new RollingCountBolt(9, 3), 4).fieldsGrouping(spoutId, new Fields("word")); - builder.setBolt(intermediateRankerId, new IntermediateRankingsBolt(TOP_N), 4).fieldsGrouping(counterId, + builder.setBolt(counterId, new RollingCountBolt(9, 3), 4).fieldsGrouping(spoutId, + new Fields("word")); + builder.setBolt(intermediateRankerId, new IntermediateRankingsBolt(TOP_N), 4) + .fieldsGrouping(counterId, new Fields("obj")); String totalRankerId = "finalRanker"; - builder.setBolt(totalRankerId, new TotalRankingsBolt(TOP_N)).globalGrouping(intermediateRankerId); + builder.setBolt(totalRankerId, new TotalRankingsBolt(TOP_N)) + .globalGrouping(intermediateRankerId); LOG.info("Topology name: " + topologyName); return submit(topologyName, conf, builder); diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/SingleJoinExample.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/SingleJoinExample.java index ad3ab3d48e2..37433a59504 100644 --- a/examples/storm-starter/src/jvm/org/apache/storm/starter/SingleJoinExample.java +++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/SingleJoinExample.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -37,7 +43,8 @@ public static void main(String[] args) throws Exception { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("gender", genderSpout); builder.setSpout("age", ageSpout); - builder.setBolt("join", new SingleJoinBolt(new Fields("gender", "age"))).fieldsGrouping("gender", new Fields("id")) + builder.setBolt("join", new SingleJoinBolt(new Fields("gender", "age"))) + .fieldsGrouping("gender", new Fields("id")) .fieldsGrouping("age", new Fields("id")); Config conf = new Config(); diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/SkewedRollingTopWords.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/SkewedRollingTopWords.java index a983b7c3d24..ce71516abfc 100644 --- a/examples/storm-starter/src/jvm/org/apache/storm/starter/SkewedRollingTopWords.java +++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/SkewedRollingTopWords.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -55,7 +61,8 @@ public static void main(String[] args) throws Exception { * *

``` * # Runs in remote/cluster mode, with topology name "production-topology" - * $ storm jar storm-starter-jar-with-dependencies.jar org.apache.storm.starter.SkewedRollingTopWords production-topology ``` + * $ storm jar storm-starter-jar-with-dependencies.jar + * org.apache.storm.starter.SkewedRollingTopWords production-topology ``` * * @param args * First positional argument (optional) is topology name, second @@ -73,12 +80,16 @@ protected int run(String[] args) { String counterId = "counter"; String aggId = "aggregator"; builder.setSpout(spoutId, new TestWordSpout(), 5); - builder.setBolt(counterId, new RollingCountBolt(9, 3), 4).partialKeyGrouping(spoutId, new Fields("word")); - builder.setBolt(aggId, new RollingCountAggBolt(), 4).fieldsGrouping(counterId, new Fields("obj")); + builder.setBolt(counterId, new RollingCountBolt(9, 3), 4).partialKeyGrouping(spoutId, + new Fields("word")); + builder.setBolt(aggId, new RollingCountAggBolt(), 4).fieldsGrouping(counterId, + new Fields("obj")); String intermediateRankerId = "intermediateRanker"; - builder.setBolt(intermediateRankerId, new IntermediateRankingsBolt(TOP_N), 4).fieldsGrouping(aggId, new Fields("obj")); + builder.setBolt(intermediateRankerId, new IntermediateRankingsBolt(TOP_N), 4) + .fieldsGrouping(aggId, new Fields("obj")); String totalRankerId = "finalRanker"; - builder.setBolt(totalRankerId, new TotalRankingsBolt(TOP_N)).globalGrouping(intermediateRankerId); + builder.setBolt(totalRankerId, new TotalRankingsBolt(TOP_N)) + .globalGrouping(intermediateRankerId); LOG.info("Topology name: " + topologyName); return submit(topologyName, conf, builder); diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/SlidingTupleTsTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/SlidingTupleTsTopology.java index d179899d69b..742935cb26f 100644 --- a/examples/storm-starter/src/jvm/org/apache/storm/starter/SlidingTupleTsTopology.java +++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/SlidingTupleTsTopology.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -19,8 +25,8 @@ import org.apache.storm.starter.bolt.SlidingWindowSumBolt; import org.apache.storm.starter.spout.RandomIntegerSpout; import org.apache.storm.topology.TopologyBuilder; -import org.apache.storm.topology.base.BaseWindowedBolt; import org.apache.storm.topology.base.BaseWindowedBolt.Duration; +import org.apache.storm.topology.base.BaseWindowedBolt; /** * Windowing based on tuple timestamp (e.g. the time when tuple is generated diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/SlidingWindowTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/SlidingWindowTopology.java index fda309df70f..e646d17e84c 100644 --- a/examples/storm-starter/src/jvm/org/apache/storm/starter/SlidingWindowTopology.java +++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/SlidingWindowTopology.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -23,8 +29,8 @@ import org.apache.storm.task.TopologyContext; import org.apache.storm.topology.OutputFieldsDeclarer; import org.apache.storm.topology.TopologyBuilder; -import org.apache.storm.topology.base.BaseWindowedBolt; import org.apache.storm.topology.base.BaseWindowedBolt.Count; +import org.apache.storm.topology.base.BaseWindowedBolt; import org.apache.storm.tuple.Fields; import org.apache.storm.tuple.Tuple; import org.apache.storm.tuple.Values; @@ -43,9 +49,11 @@ public class SlidingWindowTopology { public static void main(String[] args) throws Exception { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("integer", new RandomIntegerSpout(), 1); - builder.setBolt("slidingsum", new SlidingWindowSumBolt().withWindow(Count.of(30), Count.of(10)), 1) + builder.setBolt("slidingsum", new SlidingWindowSumBolt().withWindow(Count.of(30), Count + .of(10)), 1) .shuffleGrouping("integer"); - builder.setBolt("tumblingavg", new TumblingWindowAvgBolt().withTumblingWindow(Count.of(3)), 1) + builder.setBolt("tumblingavg", new TumblingWindowAvgBolt().withTumblingWindow(Count.of(3)), + 1) .shuffleGrouping("slidingsum"); builder.setBolt("printer", new PrinterBolt(), 1).shuffleGrouping("tumblingavg"); Config conf = new Config(); @@ -65,7 +73,8 @@ private static class TumblingWindowAvgBolt extends BaseWindowedBolt { private OutputCollector collector; @Override - public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) { + public void prepare(Map topoConf, TopologyContext context, + OutputCollector collector) { this.collector = collector; } diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/StatefulTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/StatefulTopology.java index be8d28ed848..ba5c82fca58 100644 --- a/examples/storm-starter/src/jvm/org/apache/storm/starter/StatefulTopology.java +++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/StatefulTopology.java @@ -37,7 +37,8 @@ import org.slf4j.LoggerFactory; /** - * An example topology that demonstrates the use of {@link org.apache.storm.topology.IStatefulBolt} to manage state. To run the example, + * An example topology that demonstrates the use of {@link org.apache.storm.topology.IStatefulBolt} + * to manage state. To run the example, *

  * $ storm jar examples/storm-starter/storm-starter-topologies-*.jar storm.starter.StatefulTopology statetopology
  * 
@@ -92,7 +93,8 @@ private static class StatefulSumBolt extends BaseStatefulBolt topoConf, TopologyContext context, OutputCollector collector) { + public void prepare(Map topoConf, TopologyContext context, + OutputCollector collector) { this.collector = collector; } diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/StatefulWindowingTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/StatefulWindowingTopology.java index eb0132e4f53..6910248961c 100644 --- a/examples/storm-starter/src/jvm/org/apache/storm/starter/StatefulWindowingTopology.java +++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/StatefulWindowingTopology.java @@ -39,14 +39,16 @@ import org.slf4j.LoggerFactory; /** - * A simple example that demonstrates the usage of {@link org.apache.storm.topology.IStatefulWindowedBolt} to save the state of the + * A simple example that demonstrates the usage of {@link + * org.apache.storm.topology.IStatefulWindowedBolt} to save the state of the * windowing operation to avoid re-computation in case of failures. - *

- * The framework internally manages the window boundaries and does not invoke + * + *

The framework internally manages the window boundaries and does not invoke * {@link org.apache.storm.topology.IWindowedBolt#execute(TupleWindow)} * for the already evaluated windows in case of restarts during failures. The * {@link org.apache.storm.topology.IStatefulBolt#initState(State)} - * is invoked with the previously saved state of the bolt after prepare, before the execute() method is invoked. + * is invoked with the previously saved state of the bolt after prepare, before the execute() method + * is invoked. *

*/ public class StatefulWindowingTopology { @@ -56,11 +58,13 @@ public static void main(String[] args) throws Exception { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("spout", new RandomIntegerSpout()); builder.setBolt("sumbolt", new WindowSumBolt().withWindow(new Count(5), new Count(3)) - .withMessageIdField("msgid"), 1).shuffleGrouping("spout"); + .withMessageIdField("msgid"), 1) + .shuffleGrouping("spout"); builder.setBolt("printer", new PrinterBolt(), 1).shuffleGrouping("sumbolt"); Config conf = new Config(); conf.setDebug(false); - //conf.put(Config.TOPOLOGY_STATE_PROVIDER, "org.apache.storm.redis.state.RedisKeyValueStateProvider"); + // conf.put(Config.TOPOLOGY_STATE_PROVIDER, + // "org.apache.storm.redis.state.RedisKeyValueStateProvider"); String topoName = "test"; if (args != null && args.length > 0) { @@ -77,7 +81,8 @@ private static class WindowSumBolt extends BaseStatefulWindowedBolt topoConf, TopologyContext context, OutputCollector collector) { + public void prepare(Map topoConf, TopologyContext context, + OutputCollector collector) { this.collector = collector; } diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/WordCountTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/WordCountTopology.java index 71a3b42ed28..21d56dc7598 100644 --- a/examples/storm-starter/src/jvm/org/apache/storm/starter/WordCountTopology.java +++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/WordCountTopology.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -44,7 +50,8 @@ protected int run(String[] args) throws Exception { builder.setSpout("spout", new RandomSentenceSpout(), 5); builder.setBolt("split", new SplitSentence(), 8).shuffleGrouping("spout"); - builder.setBolt("count", new WordCountBolt(), 12).fieldsGrouping("split", new Fields("word")); + builder.setBolt("count", new WordCountBolt(), 12).fieldsGrouping("split", + new Fields("word")); conf.setDebug(true); diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/WordCountTopologyNode.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/WordCountTopologyNode.java index 6df76789311..f738e80b15f 100644 --- a/examples/storm-starter/src/jvm/org/apache/storm/starter/WordCountTopologyNode.java +++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/WordCountTopologyNode.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/AbstractRankerBolt.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/AbstractRankerBolt.java index b04196ca335..42c7cd5db5e 100644 --- a/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/AbstractRankerBolt.java +++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/AbstractRankerBolt.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -26,10 +32,13 @@ import org.apache.storm.utils.TupleUtils; /** - * This abstract bolt provides the basic behavior of bolts that rank objects according to their count. + * This abstract bolt provides the basic behavior of bolts that rank objects according to their + * count. *

- * It uses a template method design pattern for {@link AbstractRankerBolt#execute(Tuple, BasicOutputCollector)} to allow - * actual bolt implementations to specify how incoming tuples are processed, i.e. how the objects embedded within those + * It uses a template method design pattern for {@link AbstractRankerBolt#execute(Tuple, + * BasicOutputCollector)} to allow + * actual bolt implementations to specify how incoming tuples are processed, i.e. how the objects + * embedded within those * tuples are retrieved and counted. */ public abstract class AbstractRankerBolt extends BaseBasicBolt { @@ -56,7 +65,8 @@ public AbstractRankerBolt(int topN, int emitFrequencyInSeconds) { } if (emitFrequencyInSeconds < 1) { throw new IllegalArgumentException( - "The emit frequency must be >= 1 seconds (you requested " + emitFrequencyInSeconds + " seconds)"); + "The emit frequency must be >= 1 seconds (you requested " + emitFrequencyInSeconds + + " seconds)"); } count = topN; this.emitFrequencyInSeconds = emitFrequencyInSeconds; diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/IntermediateRankingsBolt.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/IntermediateRankingsBolt.java index a6a8b49f9b0..076a1b40300 100644 --- a/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/IntermediateRankingsBolt.java +++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/IntermediateRankingsBolt.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -20,7 +26,8 @@ /** * This bolt ranks incoming objects by their count. *

- * It assumes the input tuples to adhere to the following format: (object, object_count, additionalField1, + * It assumes the input tuples to adhere to the following format: (object, object_count, + * additionalField1, * additionalField2, ..., additionalFieldN). */ public final class IntermediateRankingsBolt extends AbstractRankerBolt { diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/PrinterBolt.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/PrinterBolt.java index 8364222a8b7..06167986b23 100644 --- a/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/PrinterBolt.java +++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/PrinterBolt.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -17,7 +23,6 @@ import org.apache.storm.topology.base.BaseBasicBolt; import org.apache.storm.tuple.Tuple; - public class PrinterBolt extends BaseBasicBolt { @Override diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/RollingCountAggBolt.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/RollingCountAggBolt.java index 9db1dd62963..c62c98d49a7 100644 --- a/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/RollingCountAggBolt.java +++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/RollingCountAggBolt.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -29,13 +35,13 @@ public class RollingCountAggBolt extends BaseRichBolt { private static final long serialVersionUID = 5537727428628598519L; private static final Logger LOG = Logger.getLogger(RollingCountAggBolt.class); - //Mapping of key->upstreamBolt->count + // Mapping of key->upstreamBolt->count private Map> counts = new HashMap>(); private OutputCollector collector; - @Override - public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) { + public void prepare(Map topoConf, TopologyContext context, + OutputCollector collector) { this.collector = collector; } @@ -49,9 +55,9 @@ public void execute(Tuple tuple) { subCounts = new HashMap(); counts.put(obj, subCounts); } - //Update the current count for this object + // Update the current count for this object subCounts.put(source, count); - //Output the sum of all the known counts so for this key + // Output the sum of all the known counts so for this key long sum = 0; for (Long val : subCounts.values()) { sum += val; diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/RollingCountBolt.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/RollingCountBolt.java index 0e5cef57d9f..cb1646e0f75 100644 --- a/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/RollingCountBolt.java +++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/RollingCountBolt.java @@ -1,20 +1,26 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.starter.bolt; import java.util.HashMap; -import java.util.Map; import java.util.Map.Entry; +import java.util.Map; import org.apache.log4j.Logger; import org.apache.storm.Config; import org.apache.storm.starter.tools.NthLastModifiedTimeTracker; @@ -31,20 +37,31 @@ /** * This bolt performs rolling counts of incoming objects, i.e. sliding window based counting. *

- * The bolt is configured by two parameters, the length of the sliding window in seconds (which influences the output - * data of the bolt, i.e. how it will count objects) and the emit frequency in seconds (which influences how often the - * bolt will output the latest window counts). For instance, if the window length is set to an equivalent of five - * minutes and the emit frequency to one minute, then the bolt will output the latest five-minute sliding window every + * The bolt is configured by two parameters, the length of the sliding window in seconds (which + * influences the output + * data of the bolt, i.e. how it will count objects) and the emit frequency in seconds (which + * influences how often the + * bolt will output the latest window counts). For instance, if the window length is set to an + * equivalent of five + * minutes and the emit frequency to one minute, then the bolt will output the latest five-minute + * sliding window every * minute. *

- * The bolt emits a rolling count tuple per object, consisting of the object itself, its latest rolling count, and the - * actual duration of the sliding window. The latter is included in case the expected sliding window length (as - * configured by the user) is different from the actual length, e.g. due to high system load. Note that the actual - * window length is tracked and calculated for the window, and not individually for each object within a window. + * The bolt emits a rolling count tuple per object, consisting of the object itself, its latest + * rolling count, and the + * actual duration of the sliding window. The latter is included in case the expected sliding window + * length (as + * configured by the user) is different from the actual length, e.g. due to high system load. Note + * that the actual + * window length is tracked and calculated for the window, and not individually for each object + * within a window. *

- * Note: During the startup phase you will usually observe that the bolt warns you about the actual sliding window - * length being smaller than the expected length. This behavior is expected and is caused by the way the sliding window - * counts are initially "loaded up". You can safely ignore this warning during startup (e.g. you will see this warning + * Note: During the startup phase you will usually observe that the bolt warns you about the actual + * sliding window + * length being smaller than the expected length. This behavior is expected and is caused by the way + * the sliding window + * counts are initially "loaded up". You can safely ignore this warning during startup (e.g. you + * will see this warning * during the first ~ five minutes of startup time if the window length is set to five minutes). */ public class RollingCountBolt extends BaseRichBolt { @@ -53,7 +70,8 @@ public class RollingCountBolt extends BaseRichBolt { private static final Logger LOG = Logger.getLogger(RollingCountBolt.class); private static final int NUM_WINDOW_CHUNKS = 5; private static final int DEFAULT_SLIDING_WINDOW_IN_SECONDS = NUM_WINDOW_CHUNKS * 60; - private static final int DEFAULT_EMIT_FREQUENCY_IN_SECONDS = DEFAULT_SLIDING_WINDOW_IN_SECONDS / NUM_WINDOW_CHUNKS; + private static final int DEFAULT_EMIT_FREQUENCY_IN_SECONDS = + DEFAULT_SLIDING_WINDOW_IN_SECONDS / NUM_WINDOW_CHUNKS; private static final String WINDOW_LENGTH_WARNING_TEMPLATE = "Actual window length is %d seconds when it should be %d seconds" + " (you can safely ignore this warning during the startup phase)"; @@ -71,18 +89,22 @@ public RollingCountBolt() { public RollingCountBolt(int windowLengthInSeconds, int emitFrequencyInSeconds) { this.windowLengthInSeconds = windowLengthInSeconds; this.emitFrequencyInSeconds = emitFrequencyInSeconds; - counter = new SlidingWindowCounter(deriveNumWindowChunksFrom(this.windowLengthInSeconds, + counter = + new SlidingWindowCounter(deriveNumWindowChunksFrom(this.windowLengthInSeconds, this.emitFrequencyInSeconds)); } - private int deriveNumWindowChunksFrom(int windowLengthInSeconds, int windowUpdateFrequencyInSeconds) { + private int deriveNumWindowChunksFrom(int windowLengthInSeconds, + int windowUpdateFrequencyInSeconds) { return windowLengthInSeconds / windowUpdateFrequencyInSeconds; } @Override - public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) { + public void prepare(Map topoConf, TopologyContext context, + OutputCollector collector) { this.collector = collector; - lastModifiedTracker = new NthLastModifiedTimeTracker(deriveNumWindowChunksFrom(this.windowLengthInSeconds, + lastModifiedTracker = + new NthLastModifiedTimeTracker(deriveNumWindowChunksFrom(this.windowLengthInSeconds, this.emitFrequencyInSeconds)); } @@ -101,7 +123,8 @@ private void emitCurrentWindowCounts() { int actualWindowLengthInSeconds = lastModifiedTracker.secondsSinceOldestModification(); lastModifiedTracker.markAsModified(); if (actualWindowLengthInSeconds != windowLengthInSeconds) { - LOG.warn(String.format(WINDOW_LENGTH_WARNING_TEMPLATE, actualWindowLengthInSeconds, windowLengthInSeconds)); + LOG.warn(String.format(WINDOW_LENGTH_WARNING_TEMPLATE, actualWindowLengthInSeconds, + windowLengthInSeconds)); } emit(counts, actualWindowLengthInSeconds); } diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/SingleJoinBolt.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/SingleJoinBolt.java index 5f9b22525ad..0449a13d326 100644 --- a/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/SingleJoinBolt.java +++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/SingleJoinBolt.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -45,15 +51,18 @@ public SingleJoinBolt(Fields outFields) { } @Override - public void prepare(Map conf, TopologyContext context, OutputCollector collector) { + public void prepare(Map conf, TopologyContext context, + OutputCollector collector) { fieldLocations = new HashMap(); this.collector = collector; int timeout = ((Number) conf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS)).intValue(); - pending = new TimeCacheMap, Map>(timeout, new ExpireCallback()); + pending = new TimeCacheMap, Map>(timeout, + new ExpireCallback()); numSources = context.getThisSources().size(); Set idFields = null; for (GlobalStreamId source : context.getThisSources().keySet()) { - Fields fields = context.getComponentOutputFields(source.get_componentId(), source.get_streamId()); + Fields fields = context.getComponentOutputFields(source.get_componentId(), source + .get_streamId()); Set setFields = new HashSet(fields.toList()); if (idFields == null) { idFields = setFields; @@ -79,7 +88,8 @@ public void prepare(Map conf, TopologyContext context, OutputCol @Override public void execute(Tuple tuple) { List id = tuple.select(idFields); - GlobalStreamId streamId = new GlobalStreamId(tuple.getSourceComponent(), tuple.getSourceStreamId()); + GlobalStreamId streamId = new GlobalStreamId(tuple.getSourceComponent(), tuple + .getSourceStreamId()); if (!pending.containsKey(id)) { pending.put(id, new HashMap()); } diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/SlidingWindowSumBolt.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/SlidingWindowSumBolt.java index 27021817350..40fe8633a26 100644 --- a/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/SlidingWindowSumBolt.java +++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/SlidingWindowSumBolt.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -35,7 +41,8 @@ public class SlidingWindowSumBolt extends BaseWindowedBolt { private OutputCollector collector; @Override - public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) { + public void prepare(Map topoConf, TopologyContext context, + OutputCollector collector) { this.collector = collector; } diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/TotalRankingsBolt.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/TotalRankingsBolt.java index e185a9ec85d..2bd680d926d 100644 --- a/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/TotalRankingsBolt.java +++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/TotalRankingsBolt.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -19,8 +25,10 @@ /** * This bolt merges incoming {@link Rankings}. *

- * It can be used to merge intermediate rankings generated by {@link IntermediateRankingsBolt} into a final, - * consolidated ranking. To do so, configure this bolt with a globalGrouping on {@link IntermediateRankingsBolt}. + * It can be used to merge intermediate rankings generated by {@link IntermediateRankingsBolt} into + * a final, + * consolidated ranking. To do so, configure this bolt with a globalGrouping on {@link + * IntermediateRankingsBolt}. */ public final class TotalRankingsBolt extends AbstractRankerBolt { diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/WordCountBolt.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/WordCountBolt.java index 870d5bd1931..a8f506dbe70 100644 --- a/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/WordCountBolt.java +++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/WordCountBolt.java @@ -1,12 +1,17 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/spout/RandomIntegerSpout.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/spout/RandomIntegerSpout.java index e1a9ee51404..7ceb5031275 100644 --- a/examples/storm-starter/src/jvm/org/apache/storm/starter/spout/RandomIntegerSpout.java +++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/spout/RandomIntegerSpout.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -40,7 +46,8 @@ public void declareOutputFields(OutputFieldsDeclarer declarer) { } @Override - public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { + public void open(Map conf, TopologyContext context, + SpoutOutputCollector collector) { this.collector = collector; this.rand = new Random(); } @@ -48,7 +55,8 @@ public void open(Map conf, TopologyContext context, SpoutOutputC @Override public void nextTuple() { Utils.sleep(100); - collector.emit(new Values(rand.nextInt(1000), System.currentTimeMillis() - (24 * 60 * 60 * 1000), ++msgId), msgId); + collector.emit(new Values(rand.nextInt(1000), System + .currentTimeMillis() - (24 * 60 * 60 * 1000), ++msgId), msgId); } @Override diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/spout/RandomNumberGeneratorSpout.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/spout/RandomNumberGeneratorSpout.java index e6c48048c33..e32035cbb90 100644 --- a/examples/storm-starter/src/jvm/org/apache/storm/starter/spout/RandomNumberGeneratorSpout.java +++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/spout/RandomNumberGeneratorSpout.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/spout/RandomSentenceSpout.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/spout/RandomSentenceSpout.java index 92af62655e7..2be8a8d55be 100644 --- a/examples/storm-starter/src/jvm/org/apache/storm/starter/spout/RandomSentenceSpout.java +++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/spout/RandomSentenceSpout.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -32,9 +38,9 @@ public class RandomSentenceSpout extends BaseRichSpout { SpoutOutputCollector collector; Random rand; - @Override - public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { + public void open(Map conf, TopologyContext context, + SpoutOutputCollector collector) { this.collector = collector; rand = new Random(); } @@ -43,7 +49,8 @@ public void open(Map conf, TopologyContext context, SpoutOutputC public void nextTuple() { Utils.sleep(100); String[] sentences = new String[]{ - sentence("the cow jumped over the moon"), sentence("an apple a day keeps the doctor away"), + sentence("the cow jumped over the moon"), sentence("an apple a day keeps the doctor " + + "away"), sentence("four score and seven years ago"), sentence("snow white and the seven dwarfs"), sentence("i am at two with nature") }; final String sentence = sentences[rand.nextInt(sentences.length)]; diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/AggregateExample.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/AggregateExample.java index 338a894031e..a8a89e9b0e1 100644 --- a/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/AggregateExample.java +++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/AggregateExample.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -30,8 +36,10 @@ public class AggregateExample { public static void main(String[] args) throws Exception { StreamBuilder builder = new StreamBuilder(); /* - * Computes average of the stream of numbers emitted by the spout. Internally the per-partition - * sum and counts are accumulated and emitted to a downstream task where the partially accumulated + * Computes average of the stream of numbers emitted by the spout. Internally the + * per-partition + * sum and counts are accumulated and emitted to a downstream task where the partially + * accumulated * results are merged and the final result is emitted. */ builder.newStream(new RandomIntegerSpout(), new ValueMapper(0), 2) @@ -61,7 +69,8 @@ public Pair apply(Pair sumAndCount, Integer } @Override - public Pair merge(Pair sumAndCount1, Pair sumAndCount2) { + public Pair merge(Pair sumAndCount1, Pair sumAndCount2) { System.out.println("Merge " + sumAndCount1 + " and " + sumAndCount2); return Pair.of( sumAndCount1.value1 + sumAndCount2.value1, diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/BranchExample.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/BranchExample.java index fc7e74a5a7f..7187af65742 100644 --- a/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/BranchExample.java +++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/BranchExample.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/GroupByKeyAndWindowExample.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/GroupByKeyAndWindowExample.java index 02617a2cf57..23ecd68aebb 100644 --- a/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/GroupByKeyAndWindowExample.java +++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/GroupByKeyAndWindowExample.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -47,7 +53,8 @@ public static void main(String[] args) throws Exception { * together and the corresponding values will be merged. * * The result is a PairStream> with - * 'stock symbol' as the key and 'stock prices' for that symbol within the window as the value. + * 'stock symbol' as the key and 'stock prices' for that symbol within the window as + * the value. */ .groupByKeyAndWindow(SlidingWindows.of(Count.of(6), Count.of(3))) .print(); @@ -59,9 +66,11 @@ public static void main(String[] args) throws Exception { * together and their values will be reduced using the given reduce function. * * Here the result is a PairStream with - * 'stock symbol' as the key and the maximum price for that symbol within the window as the value. + * 'stock symbol' as the key and the maximum price for that symbol within the window + * as the value. */ - .reduceByKeyAndWindow((x, y) -> x > y ? x : y, SlidingWindows.of(Count.of(6), Count.of(3))) + .reduceByKeyAndWindow((x, y) -> x > y ? x : y, SlidingWindows.of(Count.of(6), Count + .of(3))) .print(); Config config = new Config(); @@ -75,15 +84,19 @@ public static void main(String[] args) throws Exception { private static class StockQuotes extends BaseRichSpout { private final List> values = Arrays.asList( - Arrays.asList(new Values("AAPL", 100.0), new Values("GOOG", 780.0), new Values("FB", 125.0)), - Arrays.asList(new Values("AAPL", 105.0), new Values("GOOG", 790.0), new Values("FB", 130.0)), - Arrays.asList(new Values("AAPL", 102.0), new Values("GOOG", 788.0), new Values("FB", 128.0)) + Arrays.asList(new Values("AAPL", 100.0), new Values("GOOG", 780.0), new Values("FB", + 125.0)), + Arrays.asList(new Values("AAPL", 105.0), new Values("GOOG", 790.0), new Values("FB", + 130.0)), + Arrays.asList(new Values("AAPL", 102.0), new Values("GOOG", 788.0), new Values("FB", + 128.0)) ); private SpoutOutputCollector collector; private int index = 0; @Override - public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { + public void open(Map conf, TopologyContext context, + SpoutOutputCollector collector) { this.collector = collector; } diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/JoinExample.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/JoinExample.java index 49085c5102e..23db7b778cd 100644 --- a/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/JoinExample.java +++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/JoinExample.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -77,7 +83,8 @@ private static class NumberSpout extends BaseRichSpout { } @Override - public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { + public void open(Map conf, TopologyContext context, + SpoutOutputCollector collector) { this.collector = collector; } diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/StateQueryExample.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/StateQueryExample.java index b72b67614e0..7bf5bc8828d 100644 --- a/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/StateQueryExample.java +++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/StateQueryExample.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -30,7 +36,7 @@ import org.apache.storm.utils.Utils; /** - * An example that uses {@link Stream#stateQuery(StreamState)} to query the state + * An example that uses {@link Stream#stateQuery(StreamState)} to query the state. * *

You should start a local redis instance before running the 'storm jar' command. By default * the connection will be attempted at localhost:6379. The default @@ -46,15 +52,19 @@ public class StateQueryExample { public static void main(String[] args) throws Exception { StreamBuilder builder = new StreamBuilder(); - StreamState ss = builder.newStream(new TestWordSpout(), new ValueMapper(0), 2) + StreamState ss = builder.newStream(new TestWordSpout(), + new ValueMapper(0), 2) /* - * Transform the stream of words to a stream of (word, 1) pairs + * Transform the stream of words to a stream of (word, + * 1) pairs */ .mapToPair(w -> Pair.of(w, 1)) /* - * Update the count in the state. Here the first argument 0L is the initial value for the + * Update the count in the state. Here the first + * argument 0L is the initial value for the * count and - * the second argument is a function that increments the count for each value received. + * the second argument is a function that increments + * the count for each value received. */ .updateStateByKey(0L, (count, val) -> count + 1); @@ -72,7 +82,8 @@ public static void main(String[] args) throws Exception { Config config = new Config(); // use redis based state store for persistence - config.put(Config.TOPOLOGY_STATE_PROVIDER, "org.apache.storm.redis.state.RedisKeyValueStateProvider"); + config.put(Config.TOPOLOGY_STATE_PROVIDER, + "org.apache.storm.redis.state.RedisKeyValueStateProvider"); String topoName = "test"; if (args.length > 0) { topoName = args[0]; @@ -86,7 +97,8 @@ private static class QuerySpout extends BaseRichSpout { private SpoutOutputCollector collector; @Override - public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { + public void open(Map conf, TopologyContext context, + SpoutOutputCollector collector) { this.collector = collector; } diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/StatefulWordCount.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/StatefulWordCount.java index 39534b38066..8e1df052304 100644 --- a/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/StatefulWordCount.java +++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/StatefulWordCount.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -55,7 +61,8 @@ public static void main(String[] args) throws Exception { /* * update the word counts in the state. * Here the first argument 0L is the initial value for the state - * and the second argument is a function that adds the count to the current value in the state. + * and the second argument is a function that adds the count to the current value in + * the state. */ .updateStateByKey(0L, (state, count) -> state + count) /* @@ -66,7 +73,8 @@ public static void main(String[] args) throws Exception { Config config = new Config(); // use redis based state store for persistence - config.put(Config.TOPOLOGY_STATE_PROVIDER, "org.apache.storm.redis.state.RedisKeyValueStateProvider"); + config.put(Config.TOPOLOGY_STATE_PROVIDER, + "org.apache.storm.redis.state.RedisKeyValueStateProvider"); String topoName = "test"; if (args.length > 0) { topoName = args[0]; diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/TypedTupleExample.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/TypedTupleExample.java index f3e8a5ac5fe..4c4c0674091 100644 --- a/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/TypedTupleExample.java +++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/TypedTupleExample.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -25,19 +31,24 @@ import org.apache.storm.topology.base.BaseWindowedBolt.Count; /** - * An example that illustrates the usage of typed tuples (TupleN<..>) and {@link TupleValueMappers}. + * An example that illustrates the usage of typed tuples (TupleN<..>) and {@link + * TupleValueMappers}. */ public class TypedTupleExample { /** - * The spout emits sequences of (Integer, Long, Long). TupleValueMapper can be used to extract fields - * from the values and produce a stream of typed tuple (Tuple3<Integer, Long, Long> in this case. + * The spout emits sequences of (Integer, Long, Long). TupleValueMapper can be used to extract + * fields + * from the values and produce a stream of typed tuple (Tuple3<Integer, Long, Long> in + * this case. */ public static void main(String[] args) throws Exception { StreamBuilder builder = new StreamBuilder(); - Stream> stream = builder.newStream(new RandomIntegerSpout(), TupleValueMappers.of(0, 1, 2)); + Stream> stream = builder.newStream(new RandomIntegerSpout(), + TupleValueMappers.of(0, 1, 2)); - PairStream pairs = stream.mapToPair(t -> Pair.of(t.value2 / 10000, t.value1)); + PairStream pairs = stream.mapToPair(t -> Pair.of(t.value2 / 10000, + t.value1)); pairs.window(TumblingWindows.of(Count.of(10))).groupByKey().print(); diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/WindowedWordCount.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/WindowedWordCount.java index ef8edd4a6fb..e70f326f669 100644 --- a/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/WindowedWordCount.java +++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/WindowedWordCount.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/WordCountToBolt.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/WordCountToBolt.java index 997e642d4f8..cb77ccd0322 100644 --- a/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/WordCountToBolt.java +++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/WordCountToBolt.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -74,7 +80,8 @@ private static class WordCountStoreMapper implements RedisStoreMapper { private final String hashKey = "wordCount"; WordCountStoreMapper() { - description = new RedisDataTypeDescription(RedisDataTypeDescription.RedisDataType.HASH, hashKey); + description = new RedisDataTypeDescription(RedisDataTypeDescription.RedisDataType.HASH, + hashKey); } @Override diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/NthLastModifiedTimeTracker.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/NthLastModifiedTimeTracker.java index 67260a3b580..d9ccaacf118 100644 --- a/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/NthLastModifiedTimeTracker.java +++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/NthLastModifiedTimeTracker.java @@ -1,19 +1,24 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.starter.tools; import java.util.concurrent.ArrayBlockingQueue; - import org.apache.storm.utils.Time; /** @@ -21,7 +26,8 @@ *

* For example, create a 5-slot tracker to track the five most recent time-since-last-modify. *

- * You must manually "mark" that the "something" that you want to track -- in terms of modification times -- has just + * You must manually "mark" that the "something" that you want to track -- in terms of modification + * times -- has just * been modified. */ public class NthLastModifiedTimeTracker { @@ -33,7 +39,8 @@ public class NthLastModifiedTimeTracker { public NthLastModifiedTimeTracker(int numTimesToTrack) { if (numTimesToTrack < 1) { throw new IllegalArgumentException( - "numTimesToTrack must be greater than zero (you requested " + numTimesToTrack + ")"); + "numTimesToTrack must be greater than zero (you requested " + numTimesToTrack + + ")"); } lastModifiedTimesMillis = new ArrayBlockingQueue<>(numTimesToTrack); initLastModifiedTimesMillis(numTimesToTrack); diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/Rankable.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/Rankable.java index ea9e6d66d1f..7c81cd66c0b 100644 --- a/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/Rankable.java +++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/Rankable.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/RankableObjectWithFields.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/RankableObjectWithFields.java index fea589691c9..8fcb18ce427 100644 --- a/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/RankableObjectWithFields.java +++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/RankableObjectWithFields.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -21,7 +27,8 @@ /** * This class wraps an objects and its associated count, including any additional data fields. *

- * This class can be used, for instance, to track the number of occurrences of an object in a Storm topology. + * This class can be used, for instance, to track the number of occurrences of an object in a Storm + * topology. */ public class RankableObjectWithFields implements Rankable, Serializable { @@ -48,9 +55,12 @@ public RankableObjectWithFields(Object obj, long count, Object... otherFields) { /** * Construct a new instance based on the provided {@link Tuple}. *

- * This method expects the object to be ranked in the first field (index 0) of the provided tuple, and the number of - * occurrences of the object (its count) in the second field (index 1). Any further fields in the tuple will be - * extracted and tracked, too. These fields can be accessed via {@link RankableObjectWithFields#getFields()}. + * This method expects the object to be ranked in the first field (index 0) of the provided + * tuple, and the number of + * occurrences of the object (its count) in the second field (index 1). Any further fields in + * the tuple will be + * extracted and tracked, too. These fields can be accessed via {@link + * RankableObjectWithFields#getFields()}. * * @param tuple * @@ -75,7 +85,9 @@ public long getCount() { /** * Get fields. - * @return an immutable list of any additional data fields of the object (may be empty but will never be null) + * + * @return an immutable list of any additional data fields of the object (may be empty but will + * never be null) */ public List getFields() { return fields; @@ -130,8 +142,10 @@ public String toString() { } /** - * Note: We do not defensively copy the wrapped object and any accompanying fields. We do guarantee, however, - * do return a defensive (shallow) copy of the List object that is wrapping any accompanying fields. + * Note: We do not defensively copy the wrapped object and any accompanying fields. We do + * guarantee, however, + * do return a defensive (shallow) copy of the List object that is wrapping any accompanying + * fields. */ @Override public Rankable copy() { diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/Rankings.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/Rankings.java index 89bba59da90..5503cd1a577 100644 --- a/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/Rankings.java +++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/Rankings.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -47,6 +53,7 @@ public Rankings(Rankings other) { /** * Get max size. + * * @return the maximum possible number (size) of ranked objects this instance can hold */ public int maxSize() { @@ -55,6 +62,7 @@ public int maxSize() { /** * Get size. + * * @return the number (size) of ranked objects this instance is currently holding */ public int size() { @@ -62,9 +70,12 @@ public int size() { } /** - * The returned defensive copy is only "somewhat" defensive. We do, for instance, return a defensive copy of the - * enclosing List instance, and we do try to defensively copy any contained Rankable objects, too. However, the - * contract of {@link org.apache.storm.starter.tools.Rankable#copy()} does not guarantee that any Object's embedded within + * The returned defensive copy is only "somewhat" defensive. We do, for instance, return a + * defensive copy of the + * enclosing List instance, and we do try to defensively copy any contained Rankable objects, + * too. However, the + * contract of {@link org.apache.storm.starter.tools.Rankable#copy()} does not guarantee that + * any Object's embedded within * a Rankable will be defensively copied, too. * * @return a somewhat defensive copy of ranked items diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/SlidingWindowCounter.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/SlidingWindowCounter.java index 136d586abd4..210ed8ae392 100644 --- a/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/SlidingWindowCounter.java +++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/SlidingWindowCounter.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -15,22 +21,29 @@ import java.io.Serializable; import java.util.Map; - /** * This class counts objects in a sliding window fashion. *

- * It is designed 1) to give multiple "producer" threads write access to the counter, i.e. being able to increment - * counts of objects, and 2) to give a single "consumer" thread (e.g. {@link org.apache.storm.starter.bolt.RollingCountBolt}) - * read access to the counter. Whenever the consumer thread performs a read operation, this class will advance the head slot - * of the sliding window counter. This means that the consumer thread indirectly controls where writes of the producer threads + * It is designed 1) to give multiple "producer" threads write access to the counter, i.e. being + * able to increment + * counts of objects, and 2) to give a single "consumer" thread (e.g. {@link + * org.apache.storm.starter.bolt.RollingCountBolt}) + * read access to the counter. Whenever the consumer thread performs a read operation, this class + * will advance the head slot + * of the sliding window counter. This means that the consumer thread indirectly controls where + * writes of the producer threads * will go to. Also, by itself this class will not advance the head slot. *

- * A note for analyzing data based on a sliding window count: During the initial windowLengthInSlots - * iterations, this sliding window counter will always return object counts that are equal or greater than in the - * previous iteration. This is the effect of the counter "loading up" at the very start of its existence. Conceptually, + * A note for analyzing data based on a sliding window count: During the initial + * windowLengthInSlots + * iterations, this sliding window counter will always return object counts that are equal or + * greater than in the + * previous iteration. This is the effect of the counter "loading up" at the very start of its + * existence. Conceptually, * this is the desired behavior. *

- * To give an example, using a counter with 5 slots which for the sake of this example represent 1 minute of time each: + * To give an example, using a counter with 5 slots which for the sake of this example represent 1 + * minute of time each: *

*

  * {@code
@@ -47,16 +60,24 @@
  * }
  * 
*

- * As you can see in this example, for the first windowLengthInSlots (here: the first five minutes) the - * counter will always return counts equal or greater than in the previous iteration (1, 2, 3, 4, 4). This initial load - * effect needs to be accounted for whenever you want to perform analyses such as trending topics; otherwise your - * analysis algorithm might falsely identify the object to be trending as the counter seems to observe continuously - * increasing counts. Also, note that during the initial load phase every object will exhibit increasing + * As you can see in this example, for the first windowLengthInSlots (here: the first + * five minutes) the + * counter will always return counts equal or greater than in the previous iteration (1, 2, 3, 4, + * 4). This initial load + * effect needs to be accounted for whenever you want to perform analyses such as trending topics; + * otherwise your + * analysis algorithm might falsely identify the object to be trending as the counter seems to + * observe continuously + * increasing counts. Also, note that during the initial load phase every object will + * exhibit increasing * counts. *

- * On a high-level, the counter exhibits the following behavior: If you asked the example counter after two minutes, - * "how often did you count the object during the past five minutes?", then it should reply "I have counted it 2 times - * in the past five minutes", implying that it can only account for the last two of those five minutes because the + * On a high-level, the counter exhibits the following behavior: If you asked the example counter + * after two minutes, + * "how often did you count the object during the past five minutes?", then it should reply "I have + * counted it 2 times + * in the past five minutes", implying that it can only account for the last two of those five + * minutes because the * counter was not running before that time. * * @param The type of those objects we want to count. @@ -73,7 +94,8 @@ public final class SlidingWindowCounter implements Serializable { public SlidingWindowCounter(int windowLengthInSlots) { if (windowLengthInSlots < 2) { throw new IllegalArgumentException( - "Window length in slots must be at least two (you requested " + windowLengthInSlots + ")"); + "Window length in slots must be at least two (you requested " + windowLengthInSlots + + ")"); } this.windowLengthInSlots = windowLengthInSlots; this.objCounter = new SlotBasedCounter(this.windowLengthInSlots); @@ -89,8 +111,10 @@ public void incrementCount(T obj) { /** * Return the current (total) counts of all tracked objects, then advance the window. *

- * Whenever this method is called, we consider the counts of the current sliding window to be available to and - * successfully processed "upstream" (i.e. by the caller). Knowing this we will start counting any subsequent + * Whenever this method is called, we consider the counts of the current sliding window to be + * available to and + * successfully processed "upstream" (i.e. by the caller). Knowing this we will start counting + * any subsequent * objects within the next "chunk" of the sliding window. * * @return The current (total) counts of all tracked objects. diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/SlotBasedCounter.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/SlotBasedCounter.java index 6f48c4784ea..f60dd589b15 100644 --- a/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/SlotBasedCounter.java +++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/SlotBasedCounter.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -20,7 +26,8 @@ /** * This class provides per-slot counts of the occurrences of objects. *

- * It can be used, for instance, as a building block for implementing sliding window counting of objects. + * It can be used, for instance, as a building block for implementing sliding window counting of + * objects. * * @param The type of those objects we want to count. */ @@ -33,7 +40,8 @@ public final class SlotBasedCounter implements Serializable { public SlotBasedCounter(int numSlots) { if (numSlots <= 0) { - throw new IllegalArgumentException("Number of slots must be greater than zero (you requested " + numSlots + ")"); + throw new IllegalArgumentException("Number of slots must be greater than zero (you " + + "requested " + numSlots + ")"); } this.numSlots = numSlots; } @@ -95,7 +103,8 @@ private boolean shouldBeRemovedFromCounter(T obj) { * Remove any object from the counter whose total count is zero (to free up memory). */ public void wipeZeros() { - for (Iterator> it = objToCounts.entrySet().iterator(); it.hasNext(); ) { + for (Iterator> it = objToCounts.entrySet().iterator(); it + .hasNext(); ) { Map.Entry entry = it.next(); if (shouldBeRemovedFromCounter(entry.getKey())) { it.remove(); diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/DebugMemoryMapState.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/DebugMemoryMapState.java index 3a7aeb0e53a..842b08b1464 100644 --- a/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/DebugMemoryMapState.java +++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/DebugMemoryMapState.java @@ -45,7 +45,8 @@ public List multiUpdate(List> keys, List updaters) print(keys, updaters); if ((updateCount++ % 5) == 0) { LOG.error("Throwing FailedException"); - throw new FailedException("Enforced State Update Fail. On retrial should replay the exact same batch."); + throw new FailedException("Enforced State Update Fail. On retrial should replay the " + + "exact same batch."); } return super.multiUpdate(keys, updaters); } @@ -54,7 +55,8 @@ private void print(List> keys, List updaters) { for (int i = 0; i < keys.size(); i++) { ValueUpdater valueUpdater = updaters.get(i); Object arg = ((CombinerValueUpdater) valueUpdater).getArg(); - LOG.info("updateCount = {}, keys = {} => updaterArgs = {}", updateCount, keys.get(i), arg); + LOG.info("updateCount = {}, keys = {} => updaterArgs = {}", updateCount, keys.get(i), + arg); } } @@ -66,7 +68,8 @@ public Factory() { } @Override - public State makeState(Map conf, IMetricsContext metrics, int partitionIndex, int numPartitions) { + public State makeState(Map conf, IMetricsContext metrics, + int partitionIndex, int numPartitions) { return new DebugMemoryMapState(id + partitionIndex); } } diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentMapExample.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentMapExample.java index 067eeff3d5c..9ec4d2f6a21 100644 --- a/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentMapExample.java +++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentMapExample.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -36,7 +42,8 @@ import org.apache.storm.utils.DRPCClient; /** - * A simple example that demonstrates the usage of {@link org.apache.storm.trident.Stream#map(MapFunction)} and + * A simple example that demonstrates the usage of {@link + * org.apache.storm.trident.Stream#map(MapFunction)} and * {@link org.apache.storm.trident.Stream#flatMap(FlatMapFunction)} functions. */ public class TridentMapExample { @@ -69,7 +76,8 @@ public boolean isKeep(TridentTuple tuple) { public static StormTopology buildTopology() { FixedBatchSpout spout = new FixedBatchSpout( new Fields("word"), 3, new Values("the cow jumped over the moon"), - new Values("the man went to the store and bought some candy"), new Values("four score and seven years ago"), + new Values("the man went to the store and bought some candy"), new Values("four score " + + "and seven years ago"), new Values("how many apples can you eat"), new Values("to be or not to be the person")); spout.setCycle(true); @@ -85,7 +93,8 @@ public void accept(TridentTuple input) { } }) .groupBy(new Fields("uppercased")) - .persistentAggregate(new MemoryMapState.Factory(), new Count(), new Fields("count")) + .persistentAggregate(new MemoryMapState.Factory(), + new Count(), new Fields("count")) .parallelismHint(16); topology.newDRPCStream("words") diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentMinMaxOfDevicesTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentMinMaxOfDevicesTopology.java index 5944408ee71..e1eb5302e0b 100644 --- a/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentMinMaxOfDevicesTopology.java +++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentMinMaxOfDevicesTopology.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -37,7 +43,8 @@ public class TridentMinMaxOfDevicesTopology { /** - * Creates a topology with device-id and count (which are whole numbers) as tuple fields in a stream and it finally + * Creates a topology with device-id and count (which are whole numbers) as tuple fields in a + * stream and it finally * generates result stream based on min amd max with device-id and count values. */ public static StormTopology buildDevicesTopology() { @@ -64,8 +71,10 @@ public static StormTopology buildDevicesTopology() { } /** - * Creates a topology which demonstrates min/max operations on tuples of stream which contain vehicle and driver fields - * with values {@link TridentMinMaxOfDevicesTopology.Vehicle} and {@link TridentMinMaxOfDevicesTopology.Driver} respectively. + * Creates a topology which demonstrates min/max operations on tuples of stream which contain + * vehicle and driver fields + * with values {@link TridentMinMaxOfDevicesTopology.Vehicle} and {@link + * TridentMinMaxOfDevicesTopology.Driver} respectively. */ public static StormTopology buildVehiclesTopology() { Fields driverField = new Fields(Driver.FIELD_NAME); diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentMinMaxOfVehiclesTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentMinMaxOfVehiclesTopology.java index 3c0ff31ac83..dd58dc28af9 100644 --- a/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentMinMaxOfVehiclesTopology.java +++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentMinMaxOfVehiclesTopology.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -38,8 +44,10 @@ public class TridentMinMaxOfVehiclesTopology { /** - * Creates a topology which demonstrates min/max operations on tuples of stream which contain vehicle and driver fields - * with values {@link TridentMinMaxOfVehiclesTopology.Vehicle} and {@link TridentMinMaxOfVehiclesTopology.Driver} respectively. + * Creates a topology which demonstrates min/max operations on tuples of stream which contain + * vehicle and driver fields + * with values {@link TridentMinMaxOfVehiclesTopology.Vehicle} and {@link + * TridentMinMaxOfVehiclesTopology.Driver} respectively. */ public static StormTopology buildVehiclesTopology() { Fields driverField = new Fields(Driver.FIELD_NAME); diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentReach.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentReach.java index 1a31bc71310..14022c1d2cb 100644 --- a/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentReach.java +++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentReach.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -41,7 +47,8 @@ public class TridentReach { public static Map> TWEETERS_DB = new HashMap>() { { put("foo.com/blog/1", Arrays.asList("sally", "bob", "tim", "george", "nathan")); - put("engineering.twitter.com/blog/5", Arrays.asList("adam", "david", "sally", "nathan")); + put("engineering.twitter.com/blog/5", Arrays.asList("adam", "david", "sally", + "nathan")); put("tech.backtype.com/blog/123", Arrays.asList("tim", "mike", "john")); } }; @@ -51,7 +58,8 @@ public class TridentReach { put("sally", Arrays.asList("bob", "tim", "alice", "adam", "jim", "chris", "jai")); put("bob", Arrays.asList("sally", "nathan", "jim", "mary", "david", "vivian")); put("tim", Arrays.asList("alex")); - put("nathan", Arrays.asList("sally", "bob", "adam", "harry", "chris", "vivian", "emily", "jordan")); + put("nathan", Arrays.asList("sally", "bob", "adam", "harry", "chris", "vivian", "emily", + "jordan")); put("adam", Arrays.asList("david", "carissa")); put("mike", Arrays.asList("john", "bob")); put("john", Arrays.asList("alice", "nathan", "jim", "mike", "bob")); @@ -60,13 +68,18 @@ public class TridentReach { public static StormTopology buildTopology() { TridentTopology topology = new TridentTopology(); - TridentState urlToTweeters = topology.newStaticState(new StaticSingleKeyMapState.Factory(TWEETERS_DB)); - TridentState tweetersToFollowers = topology.newStaticState(new StaticSingleKeyMapState.Factory(FOLLOWERS_DB)); - - - topology.newDRPCStream("reach").stateQuery(urlToTweeters, new Fields("args"), new MapGet(), new Fields( - "tweeters")).each(new Fields("tweeters"), new ExpandList(), new Fields("tweeter")).shuffle().stateQuery( - tweetersToFollowers, new Fields("tweeter"), new MapGet(), new Fields("followers")).each(new Fields("followers"), + TridentState urlToTweeters = topology.newStaticState(new StaticSingleKeyMapState + .Factory(TWEETERS_DB)); + TridentState tweetersToFollowers = topology.newStaticState(new StaticSingleKeyMapState + .Factory(FOLLOWERS_DB)); + + + topology.newDRPCStream("reach").stateQuery(urlToTweeters, new Fields("args"), new MapGet(), + new Fields( + "tweeters")).each(new Fields("tweeters"), new ExpandList(), new Fields("tweeter")) + .shuffle().stateQuery( + tweetersToFollowers, new Fields("tweeter"), new MapGet(), new Fields("followers")) + .each(new Fields("followers"), new ExpandList(), new Fields("follower")) .groupBy(new Fields("follower")).aggregate(new One(), new Fields( @@ -111,7 +124,8 @@ public Factory(Map map) { } @Override - public State makeState(Map conf, IMetricsContext metrics, int partitionIndex, int numPartitions) { + public State makeState(Map conf, IMetricsContext metrics, + int partitionIndex, int numPartitions) { return new StaticSingleKeyMapState(map); } diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentWindowingInmemoryStoreTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentWindowingInmemoryStoreTopology.java index 862d09a237d..5265c31cd58 100644 --- a/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentWindowingInmemoryStoreTopology.java +++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentWindowingInmemoryStoreTopology.java @@ -1,12 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -35,20 +40,26 @@ * Sample application of trident windowing which uses inmemory store for storing tuples in window. */ public class TridentWindowingInmemoryStoreTopology { - private static final Logger LOG = LoggerFactory.getLogger(TridentWindowingInmemoryStoreTopology.class); + private static final Logger LOG = LoggerFactory + .getLogger(TridentWindowingInmemoryStoreTopology.class); - public static StormTopology buildTopology(WindowsStoreFactory windowStore, WindowConfig windowConfig) throws Exception { - FixedBatchSpout spout = new FixedBatchSpout(new Fields("sentence"), 3, new Values("the cow jumped over the moon"), - new Values("the man went to the store and bought some candy"), + public static StormTopology buildTopology(WindowsStoreFactory windowStore, + WindowConfig windowConfig) throws Exception { + FixedBatchSpout spout = new FixedBatchSpout(new Fields("sentence"), 3, + new Values("the cow jumped over the moon"), + new Values("the man went to the store and " + + "bought some candy"), new Values("four score and seven years ago"), new Values("how many apples can you eat"), new Values("to be or not to be the person")); spout.setCycle(true); TridentTopology topology = new TridentTopology(); - Stream stream = topology.newStream("spout1", spout).parallelismHint(16).each(new Fields("sentence"), + Stream stream = topology.newStream("spout1", spout).parallelismHint(16) + .each(new Fields("sentence"), new Split(), new Fields("word")) - .window(windowConfig, windowStore, new Fields("word"), new CountAsAggregator(), new Fields("count")) + .window(windowConfig, windowStore, new Fields("word"), + new CountAsAggregator(), new Fields("count")) .peek(new Consumer() { @Override public void accept(TridentTuple input) { @@ -68,6 +79,7 @@ public static void main(String[] args) throws Exception { } conf.setNumWorkers(3); - StormSubmitter.submitTopologyWithProgressBar(topoName, conf, buildTopology(mapState, SlidingCountWindow.of(1000, 100))); + StormSubmitter.submitTopologyWithProgressBar(topoName, conf, buildTopology(mapState, + SlidingCountWindow.of(1000, 100))); } } diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentWordCount.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentWordCount.java index bafeba29c24..9c3e4f8824e 100644 --- a/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentWordCount.java +++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentWordCount.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -29,19 +35,22 @@ import org.apache.storm.tuple.Values; import org.apache.storm.utils.DRPCClient; - public class TridentWordCount { public static StormTopology buildTopology() { - FixedBatchSpout spout = new FixedBatchSpout(new Fields("sentence"), 3, new Values("the cow jumped over the moon"), - new Values("the man went to the store and bought some candy"), + FixedBatchSpout spout = new FixedBatchSpout(new Fields("sentence"), 3, + new Values("the cow jumped over the moon"), + new Values("the man went to the store and " + + "bought some candy"), new Values("four score and seven years ago"), new Values("how many apples can you eat"), new Values("to be or not to be the person")); spout.setCycle(true); TridentTopology topology = new TridentTopology(); - TridentState wordCounts = topology.newStream("spout1", spout).parallelismHint(16).each(new Fields("sentence"), + TridentState wordCounts = topology.newStream("spout1", spout).parallelismHint(16) + .each(new Fields("sentence"), new Split(), new Fields("word")) - .groupBy(new Fields("word")).persistentAggregate(new MemoryMapState.Factory(), + .groupBy(new Fields("word")) + .persistentAggregate(new MemoryMapState.Factory(), new Count(), new Fields("count")) .parallelismHint(16); diff --git a/examples/storm-starter/test/jvm/org/apache/storm/starter/bolt/IntermediateRankingsBoltTest.java b/examples/storm-starter/test/jvm/org/apache/storm/starter/bolt/IntermediateRankingsBoltTest.java index 18834d3095c..2f45b35c21f 100644 --- a/examples/storm-starter/test/jvm/org/apache/storm/starter/bolt/IntermediateRankingsBoltTest.java +++ b/examples/storm-starter/test/jvm/org/apache/storm/starter/bolt/IntermediateRankingsBoltTest.java @@ -1,17 +1,31 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.starter.bolt; +import static org.fest.assertions.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + import com.google.common.collect.Lists; import java.util.Map; import org.apache.storm.Config; @@ -24,14 +38,6 @@ import org.testng.annotations.DataProvider; import org.testng.annotations.Test; -import static org.fest.assertions.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoInteractions; -import static org.mockito.Mockito.when; - public class IntermediateRankingsBoltTest { private static final String ANY_NON_SYSTEM_COMPONENT_ID = "irrelevant_component_id"; @@ -41,7 +47,8 @@ public class IntermediateRankingsBoltTest { private static final long ANY_COUNT = 42; private Tuple mockRankableTuple(Object obj, long count) { - Tuple tuple = MockTupleHelpers.mockTuple(ANY_NON_SYSTEM_COMPONENT_ID, ANY_NON_SYSTEM_STREAM_ID); + Tuple tuple = MockTupleHelpers.mockTuple(ANY_NON_SYSTEM_COMPONENT_ID, + ANY_NON_SYSTEM_STREAM_ID); when(tuple.getValues()).thenReturn(Lists.newArrayList(ANY_OBJECT, ANY_COUNT)); return tuple; } @@ -61,7 +68,8 @@ public Object[][] illegalEmitFrequency() { return new Object[][]{ { -10 }, { -3 }, { -2 }, { -1 }, { 0 } }; } - @Test(expectedExceptions = IllegalArgumentException.class, dataProvider = "illegalEmitFrequency") + @Test(expectedExceptions = IllegalArgumentException.class, + dataProvider = "illegalEmitFrequency") public void negativeOrZeroEmitFrequencyShouldThrowIAE(int emitFrequencyInSeconds) { new IntermediateRankingsBolt(ANY_TOPN, emitFrequencyInSeconds); } @@ -138,7 +146,8 @@ public void shouldSetTickTupleFrequencyInComponentConfigurationToNonZeroValue() // then assertThat(componentConfig).containsKey(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS); - Integer emitFrequencyInSeconds = (Integer) componentConfig.get(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS); + Integer emitFrequencyInSeconds = (Integer) componentConfig + .get(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS); assertThat(emitFrequencyInSeconds).isGreaterThan(0); } } diff --git a/examples/storm-starter/test/jvm/org/apache/storm/starter/bolt/RollingCountBoltTest.java b/examples/storm-starter/test/jvm/org/apache/storm/starter/bolt/RollingCountBoltTest.java index 56d65606520..f8f5dd3fc15 100644 --- a/examples/storm-starter/test/jvm/org/apache/storm/starter/bolt/RollingCountBoltTest.java +++ b/examples/storm-starter/test/jvm/org/apache/storm/starter/bolt/RollingCountBoltTest.java @@ -1,17 +1,31 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.starter.bolt; +import static org.fest.assertions.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + import java.util.Map; import org.apache.storm.Config; import org.apache.storm.task.OutputCollector; @@ -23,21 +37,14 @@ import org.apache.storm.utils.MockTupleHelpers; import org.testng.annotations.Test; -import static org.fest.assertions.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoInteractions; -import static org.mockito.Mockito.when; - public class RollingCountBoltTest { private static final String ANY_NON_SYSTEM_COMPONENT_ID = "irrelevant_component_id"; private static final String ANY_NON_SYSTEM_STREAM_ID = "irrelevant_stream_id"; private Tuple mockNormalTuple(Object obj) { - Tuple tuple = MockTupleHelpers.mockTuple(ANY_NON_SYSTEM_COMPONENT_ID, ANY_NON_SYSTEM_STREAM_ID); + Tuple tuple = MockTupleHelpers.mockTuple(ANY_NON_SYSTEM_COMPONENT_ID, + ANY_NON_SYSTEM_STREAM_ID); when(tuple.getValue(0)).thenReturn(obj); return tuple; } @@ -105,7 +112,8 @@ public void shouldSetTickTupleFrequencyInComponentConfigurationToNonZeroValue() // then assertThat(componentConfig).containsKey(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS); - Integer emitFrequencyInSeconds = (Integer) componentConfig.get(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS); + Integer emitFrequencyInSeconds = (Integer) componentConfig + .get(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS); assertThat(emitFrequencyInSeconds).isGreaterThan(0); } } diff --git a/examples/storm-starter/test/jvm/org/apache/storm/starter/bolt/TotalRankingsBoltTest.java b/examples/storm-starter/test/jvm/org/apache/storm/starter/bolt/TotalRankingsBoltTest.java index d74e41e5a19..43a3d9b5263 100644 --- a/examples/storm-starter/test/jvm/org/apache/storm/starter/bolt/TotalRankingsBoltTest.java +++ b/examples/storm-starter/test/jvm/org/apache/storm/starter/bolt/TotalRankingsBoltTest.java @@ -1,17 +1,31 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.starter.bolt; +import static org.fest.assertions.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + import java.util.Map; import org.apache.storm.Config; import org.apache.storm.starter.tools.Rankings; @@ -24,14 +38,6 @@ import org.testng.annotations.DataProvider; import org.testng.annotations.Test; -import static org.fest.assertions.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoInteractions; -import static org.mockito.Mockito.when; - public class TotalRankingsBoltTest { private static final String ANY_NON_SYSTEM_COMPONENT_ID = "irrelevant_component_id"; @@ -41,7 +47,8 @@ public class TotalRankingsBoltTest { private static final long ANY_COUNT = 42; private Tuple mockRankingsTuple(Object obj, long count) { - Tuple tuple = MockTupleHelpers.mockTuple(ANY_NON_SYSTEM_COMPONENT_ID, ANY_NON_SYSTEM_STREAM_ID); + Tuple tuple = MockTupleHelpers.mockTuple(ANY_NON_SYSTEM_COMPONENT_ID, + ANY_NON_SYSTEM_STREAM_ID); Rankings rankings = mock(Rankings.class); when(tuple.getValue(0)).thenReturn(rankings); return tuple; @@ -62,7 +69,8 @@ public Object[][] illegalEmitFrequency() { return new Object[][]{ { -10 }, { -3 }, { -2 }, { -1 }, { 0 } }; } - @Test(expectedExceptions = IllegalArgumentException.class, dataProvider = "illegalEmitFrequency") + @Test(expectedExceptions = IllegalArgumentException.class, + dataProvider = "illegalEmitFrequency") public void negativeOrZeroEmitFrequencyShouldThrowIAE(int emitFrequencyInSeconds) { new TotalRankingsBolt(ANY_TOPN, emitFrequencyInSeconds); } @@ -139,7 +147,8 @@ public void shouldSetTickTupleFrequencyInComponentConfigurationToNonZeroValue() // then assertThat(componentConfig).containsKey(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS); - Integer emitFrequencyInSeconds = (Integer) componentConfig.get(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS); + Integer emitFrequencyInSeconds = (Integer) componentConfig + .get(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS); assertThat(emitFrequencyInSeconds).isGreaterThan(0); } } diff --git a/examples/storm-starter/test/jvm/org/apache/storm/starter/tools/NthLastModifiedTimeTrackerTest.java b/examples/storm-starter/test/jvm/org/apache/storm/starter/tools/NthLastModifiedTimeTrackerTest.java index f4393ed41b4..fa908c9a6da 100644 --- a/examples/storm-starter/test/jvm/org/apache/storm/starter/tools/NthLastModifiedTimeTrackerTest.java +++ b/examples/storm-starter/test/jvm/org/apache/storm/starter/tools/NthLastModifiedTimeTrackerTest.java @@ -1,24 +1,30 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.starter.tools; -import org.apache.storm.utils.Time; +import static org.fest.assertions.api.Assertions.assertThat; + import org.apache.storm.utils.Time.SimulatedTime; +import org.apache.storm.utils.Time; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; -import static org.fest.assertions.api.Assertions.assertThat; - public class NthLastModifiedTimeTrackerTest { private static final int ANY_NUM_TIMES_TO_TRACK = 3; @@ -53,7 +59,8 @@ public Object[][] whenNotYetMarkedAsModifiedData() { public void shouldReturnCorrectModifiedTimeEvenWhenNotYetMarkedAsModified(int secondsToAdvance) { // given try (SimulatedTime t = new SimulatedTime()) { - NthLastModifiedTimeTracker tracker = new NthLastModifiedTimeTracker(ANY_NUM_TIMES_TO_TRACK); + NthLastModifiedTimeTracker tracker = + new NthLastModifiedTimeTracker(ANY_NUM_TIMES_TO_TRACK); // when Time.advanceTimeSecs(secondsToAdvance); diff --git a/examples/storm-starter/test/jvm/org/apache/storm/starter/tools/RankableObjectWithFieldsTest.java b/examples/storm-starter/test/jvm/org/apache/storm/starter/tools/RankableObjectWithFieldsTest.java index cf275d26d72..f715e8fe937 100644 --- a/examples/storm-starter/test/jvm/org/apache/storm/starter/tools/RankableObjectWithFieldsTest.java +++ b/examples/storm-starter/test/jvm/org/apache/storm/starter/tools/RankableObjectWithFieldsTest.java @@ -1,17 +1,29 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.starter.tools; +import static org.fest.assertions.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; + import com.google.common.collect.Lists; import java.util.ArrayList; import java.util.List; @@ -19,12 +31,6 @@ import org.testng.annotations.DataProvider; import org.testng.annotations.Test; -import static org.fest.assertions.api.Assertions.assertThat; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; -import static org.testng.Assert.assertFalse; -import static org.testng.Assert.assertTrue; - public class RankableObjectWithFieldsTest { private static final Object ANY_OBJECT = new Object(); @@ -67,14 +73,15 @@ public Object[][] otherClassesData() { return new Object[][]{ {"foo"}, { new Object() }, {4}, { Lists.newArrayList(7, 8, 9) - } + } }; } @Test(dataProvider = "otherClassesData") public void shouldNotBeEqualToInstancesOfOtherClasses(Object notARankable) { RankableObjectWithFields r = new RankableObjectWithFields(ANY_OBJECT, ANY_COUNT); - assertFalse(r.equals(notARankable), r + " is equal to " + notARankable + " but it should not be"); + assertFalse(r.equals(notARankable), r + " is equal to " + notARankable + + " but it should not be"); } @DataProvider @@ -85,15 +92,17 @@ public Object[][] falseDuplicatesData() { { new RankableObjectWithFields("foo", 1), new RankableObjectWithFields("FOO", 1) }, { new RankableObjectWithFields("foo", 1), new RankableObjectWithFields("bar", 1) }, { new RankableObjectWithFields("", 0), new RankableObjectWithFields("", 1) }, { - new RankableObjectWithFields("", + new RankableObjectWithFields("", 1), new RankableObjectWithFields("bar", 1) - } + } }; } @Test(dataProvider = "falseDuplicatesData") - public void shouldNotBeEqualToFalseDuplicates(RankableObjectWithFields r, RankableObjectWithFields falseDuplicate) { - assertFalse(r.equals(falseDuplicate), r + " is equal to " + falseDuplicate + " but it should not be"); + public void shouldNotBeEqualToFalseDuplicates(RankableObjectWithFields r, + RankableObjectWithFields falseDuplicate) { + assertFalse(r.equals(falseDuplicate), r + " is equal to " + falseDuplicate + + " but it should not be"); } @Test(dataProvider = "falseDuplicatesData") @@ -106,17 +115,21 @@ public void shouldHaveDifferentHashCodeThanFalseDuplicates(RankableObjectWithFie public Object[][] trueDuplicatesData() { return new Object[][]{ { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("foo", 0) }, - { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("foo", 0, "someOtherField") }, + { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("foo", 0, + "someOtherField") }, { - new RankableObjectWithFields("foo", 0, "someField"), new RankableObjectWithFields("foo", 0, + new RankableObjectWithFields("foo", 0, + "someField"), new RankableObjectWithFields("foo", 0, "someOtherField") } }; } @Test(dataProvider = "trueDuplicatesData") - public void shouldBeEqualToTrueDuplicates(RankableObjectWithFields r, RankableObjectWithFields trueDuplicate) { - assertTrue(r.equals(trueDuplicate), r + " is not equal to " + trueDuplicate + " but it should be"); + public void shouldBeEqualToTrueDuplicates(RankableObjectWithFields r, + RankableObjectWithFields trueDuplicate) { + assertTrue(r.equals(trueDuplicate), r + " is not equal to " + trueDuplicate + + " but it should be"); } @Test(dataProvider = "trueDuplicatesData") @@ -132,25 +145,32 @@ public Object[][] compareToData() { new RankableObjectWithFields("foo", 1000), new RankableObjectWithFields("foo", 0), GREATER_THAN }, { - new RankableObjectWithFields("foo", 1), new RankableObjectWithFields("foo", 0), - GREATER_THAN - }, { - new RankableObjectWithFields("foo", 1000), new RankableObjectWithFields("bar", 0), - GREATER_THAN - }, { - new RankableObjectWithFields("foo", 1), new RankableObjectWithFields("bar", 0), - GREATER_THAN - }, { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("foo", 0), EQUAL_TO }, - { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("bar", 0), EQUAL_TO }, - { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("foo", 1000), SMALLER_THAN }, - { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("foo", 1), SMALLER_THAN }, - { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("bar", 1), SMALLER_THAN }, - { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("bar", 1000), SMALLER_THAN }, + new RankableObjectWithFields("foo", 1), new RankableObjectWithFields("foo", 0), + GREATER_THAN + }, { + new RankableObjectWithFields("foo", 1000), new RankableObjectWithFields("bar", 0), + GREATER_THAN + }, { + new RankableObjectWithFields("foo", 1), new RankableObjectWithFields("bar", 0), + GREATER_THAN + }, { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("foo", + 0), EQUAL_TO }, + { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("bar", + 0), EQUAL_TO }, + { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("foo", + 1000), SMALLER_THAN }, + { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("foo", + 1), SMALLER_THAN }, + { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("bar", + 1), SMALLER_THAN }, + { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("bar", + 1000), SMALLER_THAN }, }; } @Test(dataProvider = "compareToData") - public void verifyCompareTo(RankableObjectWithFields first, RankableObjectWithFields second, int expCompareToValue) { + public void verifyCompareTo(RankableObjectWithFields first, RankableObjectWithFields second, + int expCompareToValue) { assertThat(first.compareTo(second)).isEqualTo(expCompareToValue); } @@ -251,13 +271,14 @@ public void shouldCreateRankableObjectFromTuple() { public Object[][] copyData() { return new Object[][]{ { new RankableObjectWithFields("foo", 0) }, { - new RankableObjectWithFields("foo", 3, + new RankableObjectWithFields("foo", 3, "someOtherField") - }, { new RankableObjectWithFields("foo", 0, "someField") } + }, { new RankableObjectWithFields("foo", 0, "someField") } }; } - // TODO: What would be a good test to ensure that RankableObjectWithFields is at least somewhat defensively copied? + // TODO: What would be a good test to ensure that RankableObjectWithFields is at least somewhat + // defensively copied? // The contract of Rankable#copy() returns a Rankable value, not a RankableObjectWithFields. @Test(dataProvider = "copyData") public void copyShouldReturnCopy(RankableObjectWithFields original) { diff --git a/examples/storm-starter/test/jvm/org/apache/storm/starter/tools/RankingsTest.java b/examples/storm-starter/test/jvm/org/apache/storm/starter/tools/RankingsTest.java index 206d45c77d4..565a781ee7d 100644 --- a/examples/storm-starter/test/jvm/org/apache/storm/starter/tools/RankingsTest.java +++ b/examples/storm-starter/test/jvm/org/apache/storm/starter/tools/RankingsTest.java @@ -1,17 +1,25 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.starter.tools; +import static org.fest.assertions.api.Assertions.assertThat; + import com.google.common.base.Throwables; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; @@ -20,12 +28,11 @@ import org.testng.annotations.DataProvider; import org.testng.annotations.Test; -import static org.fest.assertions.api.Assertions.assertThat; - public class RankingsTest { private static final int ANY_TOPN = 42; - private static final Rankable ANY_RANKABLE = new RankableObjectWithFields("someObject", ANY_TOPN); + private static final Rankable ANY_RANKABLE = new RankableObjectWithFields("someObject", + ANY_TOPN); private static final Rankable ZERO = new RankableObjectWithFields("ZERO_COUNT", 0); private static final Rankable A = new RankableObjectWithFields("A", 1); private static final Rankable B = new RankableObjectWithFields("B", 2); @@ -50,7 +57,8 @@ public void constructorWithNegativeOrZeroTopNShouldThrowIAE(int topN) { public Object[][] copyRankingsData() { return new Object[][]{ { 5, Lists.newArrayList(A, B, C) }, { 2, Lists.newArrayList(A, B, C, D) }, - { 1, Lists.newArrayList() }, { 1, Lists.newArrayList(A) }, { 1, Lists.newArrayList(A, B) } + { 1, Lists.newArrayList() }, { 1, Lists.newArrayList(A) }, { 1, Lists.newArrayList(A, + B) } }; } @@ -74,20 +82,21 @@ public void copyConstructorShouldReturnCopy(int topN, List rankables) public Object[][] defensiveCopyRankingsData() { return new Object[][]{ { 5, Lists.newArrayList(A, B, C), Lists.newArrayList(D) }, { - 2, Lists.newArrayList(A, B, C, + 2, Lists.newArrayList(A, B, C, D), Lists.newArrayList(E, F) - }, { 1, Lists.newArrayList(), Lists.newArrayList(A) }, { - 1, Lists.newArrayList(A), + }, { 1, Lists.newArrayList(), Lists.newArrayList(A) }, { + 1, Lists.newArrayList(A), Lists.newArrayList(B) - }, { 1, Lists.newArrayList(ZERO), Lists.newArrayList(B) }, { - 1, Lists.newArrayList(ZERO), + }, { 1, Lists.newArrayList(ZERO), Lists.newArrayList(B) }, { + 1, Lists.newArrayList(ZERO), Lists.newArrayList() - } + } }; } @Test(dataProvider = "defensiveCopyRankingsData") - public void copyConstructorShouldReturnDefensiveCopy(int topN, List rankables, List changes) { + public void copyConstructorShouldReturnDefensiveCopy(int topN, List rankables, + List changes) { // given Rankings original = new Rankings(topN); for (Rankable r : rankables) { @@ -139,14 +148,16 @@ public void defaultConstructorShouldSetPositiveTopN() { public Object[][] rankingsGrowData() { return new Object[][]{ { - 2, Lists.newArrayList(new RankableObjectWithFields("A", 1), new RankableObjectWithFields( - "B", 2), new RankableObjectWithFields("C", 3)) + 2, Lists.newArrayList(new RankableObjectWithFields("A", 1), + new RankableObjectWithFields( + "B", 2), new RankableObjectWithFields("C", 3)) }, { - 2, Lists.newArrayList(new RankableObjectWithFields("A", 1), - new RankableObjectWithFields("B", 2), new RankableObjectWithFields("C", 3), + 2, Lists.newArrayList(new RankableObjectWithFields("A", 1), + new RankableObjectWithFields("B", + 2), new RankableObjectWithFields("C", 3), new RankableObjectWithFields("D", 4)) - } + } }; } @@ -154,7 +165,8 @@ public Object[][] rankingsGrowData() { public void sizeOfRankingsShouldNotGrowBeyondTopN(int topN, List rankables) { // sanity check of the provided test data assertThat(rankables.size()).overridingErrorMessage( - "The supplied test data is not correct: the number of rankables <%d> should be greater than <%d>", + "The supplied test data is not correct: the number of rankables <%d> should be " + + "greater than <%d>", rankables.size(), topN).isGreaterThan(topN); // given @@ -175,15 +187,17 @@ public Object[][] simulatedRankingsData() { { Lists.newArrayList(A), Lists.newArrayList(A) }, { Lists.newArrayList(B, D, A, C), Lists.newArrayList(D, C, B, A) - }, { + }, { Lists.newArrayList(B, F, A, C, D, E), Lists.newArrayList(F, E, D, C, B, A) - }, { Lists.newArrayList(G, B, F, A, C, D, E, H), Lists.newArrayList(H, G, F, E, D, C, B, A) } + }, { Lists.newArrayList(G, B, F, A, C, D, E, H), Lists.newArrayList(H, G, F, E, D, C, B, + A) } }; } @Test(dataProvider = "simulatedRankingsData") - public void shouldCorrectlyRankWhenUpdatedWithRankables(List unsorted, List expSorted) { + public void shouldCorrectlyRankWhenUpdatedWithRankables(List unsorted, + List expSorted) { // given Rankings rankings = new Rankings(unsorted.size()); @@ -214,7 +228,8 @@ public void shouldCorrectlyRankWhenEmptyAndUpdatedWithOtherRankings(List unsorted, List expSorted) { + public void shouldCorrectlyRankWhenUpdatedWithEmptyOtherRankings(List unsorted, + List expSorted) { // given Rankings rankings = new Rankings(unsorted.size()); for (Rankable r : unsorted) { @@ -233,13 +248,15 @@ public void shouldCorrectlyRankWhenUpdatedWithEmptyOtherRankings(List public Object[][] simulatedRankingsAndOtherRankingsData() { return new Object[][]{ { Lists.newArrayList(A), Lists.newArrayList(A), Lists.newArrayList(A) }, - { Lists.newArrayList(A, C), Lists.newArrayList(B, D), Lists.newArrayList(D, C, B, A) }, { + { Lists.newArrayList(A, C), Lists.newArrayList(B, D), Lists.newArrayList(D, C, B, + A) }, { Lists.newArrayList(B, - F, A), Lists.newArrayList(C, D, E), Lists.newArrayList(F, E, D, C, B, A) - }, { + F, A), Lists.newArrayList(C, D, E), Lists + .newArrayList(F, E, D, C, B, A) + }, { Lists.newArrayList(G, B, F, A, C), Lists.newArrayList(D, E, H), Lists.newArrayList(H, G, F, E, D, C, B, A) - } + } }; } @@ -273,7 +290,7 @@ public Object[][] duplicatesData() { { Lists.newArrayList(ANY_RANKABLE, ANY_RANKABLE, ANY_RANKABLE) }, { Lists.newArrayList(A1, A2, A3) - }, + }, }; } @@ -297,18 +314,20 @@ public Object[][] removeZeroRankingsData() { { Lists.newArrayList(A, ZERO), Lists.newArrayList(A) }, { Lists.newArrayList(A), Lists.newArrayList(A) - }, { Lists.newArrayList(ZERO, A), Lists.newArrayList(A) }, { + }, { Lists.newArrayList(ZERO, A), Lists.newArrayList(A) }, { Lists.newArrayList(ZERO), Lists.newArrayList() - }, { + }, { Lists.newArrayList(ZERO, new RankableObjectWithFields("ZERO2", 0)), Lists.newArrayList() - }, { + }, { Lists.newArrayList(B, ZERO, new RankableObjectWithFields("ZERO2", 0), D, - new RankableObjectWithFields("ZERO3", 0), new RankableObjectWithFields("ZERO4", 0), C), Lists.newArrayList(D, + new RankableObjectWithFields("ZERO3", + 0), new RankableObjectWithFields("ZERO4", 0), C), Lists + .newArrayList(D, C, B) - }, { Lists.newArrayList(A, ZERO, B), Lists.newArrayList(B, A) } + }, { Lists.newArrayList(A, ZERO, B), Lists.newArrayList(B, A) } }; } @@ -333,8 +352,10 @@ public void updatingWithNewRankablesShouldBeThreadSafe() throws InterruptedExcep final List entries = ImmutableList.of(A, B, C, D); final Rankings rankings = new Rankings(entries.size()); - // We are capturing exceptions thrown in Blitzer's child threads into this data structure so that we can properly - // pass/fail this test. The reason is that Blitzer doesn't report exceptions, which is a known bug in Blitzer + // We are capturing exceptions thrown in Blitzer's child threads into this data structure so + // that we can properly + // pass/fail this test. The reason is that Blitzer doesn't report exceptions, which is a + // known bug in Blitzer // (JMOCK-263). See https://github.com/jmock-developers/jmock-library/issues/22 for more information. final List exceptions = Lists.newArrayList(); Blitzer blitzer = new Blitzer(1000); @@ -383,7 +404,8 @@ public void copyShouldReturnCopy(int topN, List rankables) { } @Test(dataProvider = "defensiveCopyRankingsData") - public void copyShouldReturnDefensiveCopy(int topN, List rankables, List changes) { + public void copyShouldReturnDefensiveCopy(int topN, List rankables, + List changes) { // given Rankings original = new Rankings(topN); for (Rankable r : rankables) { diff --git a/examples/storm-starter/test/jvm/org/apache/storm/starter/tools/SlidingWindowCounterTest.java b/examples/storm-starter/test/jvm/org/apache/storm/starter/tools/SlidingWindowCounterTest.java index 3bdb4ff25e6..2c0a5f30073 100644 --- a/examples/storm-starter/test/jvm/org/apache/storm/starter/tools/SlidingWindowCounterTest.java +++ b/examples/storm-starter/test/jvm/org/apache/storm/starter/tools/SlidingWindowCounterTest.java @@ -1,23 +1,29 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.starter.tools; +import static org.fest.assertions.api.Assertions.assertThat; + import java.util.Map; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; -import static org.fest.assertions.api.Assertions.assertThat; - public class SlidingWindowCounterTest { private static final int ANY_WINDOW_LENGTH_IN_SLOTS = 2; @@ -28,7 +34,8 @@ public Object[][] illegalWindowLengths() { return new Object[][]{ { -10 }, { -3 }, { -2 }, { -1 }, { 0 }, { 1 } }; } - @Test(expectedExceptions = IllegalArgumentException.class, dataProvider = "illegalWindowLengths") + @Test(expectedExceptions = IllegalArgumentException.class, + dataProvider = "illegalWindowLengths") public void lessThanTwoSlotsShouldThrowIAE(int windowLengthInSlots) { new SlidingWindowCounter(windowLengthInSlots); } @@ -46,7 +53,8 @@ public void twoOrMoreSlotsShouldBeValid(int windowLengthInSlots) { @Test public void newInstanceShouldHaveEmptyCounts() { // given - SlidingWindowCounter counter = new SlidingWindowCounter(ANY_WINDOW_LENGTH_IN_SLOTS); + SlidingWindowCounter counter = + new SlidingWindowCounter(ANY_WINDOW_LENGTH_IN_SLOTS); // when Map counts = counter.getCountsThenAdvanceWindow(); @@ -73,7 +81,8 @@ public Object[][] simulatedCounterIterations() { public void testCounterWithSimulatedRuns(int windowLengthInSlots, int[] incrementsPerIteration, long[] expCountsPerIteration) { // given - SlidingWindowCounter counter = new SlidingWindowCounter(windowLengthInSlots); + SlidingWindowCounter counter = + new SlidingWindowCounter(windowLengthInSlots); int numIterations = incrementsPerIteration.length; for (int i = 0; i < numIterations; i++) { @@ -81,7 +90,8 @@ public void testCounterWithSimulatedRuns(int windowLengthInSlots, int[] incremen long expCounts = expCountsPerIteration[i]; // Objects are absent if they were zero both this iteration // and the last -- if only this one, we need to report zero. - boolean expAbsent = ((expCounts == 0) && ((i == 0) || (expCountsPerIteration[i - 1] == 0))); + boolean expAbsent = ((expCounts == 0) && ((i == 0) + || (expCountsPerIteration[i - 1] == 0))); // given (for this iteration) for (int j = 0; j < numIncrements; j++) { diff --git a/examples/storm-starter/test/jvm/org/apache/storm/starter/tools/SlotBasedCounterTest.java b/examples/storm-starter/test/jvm/org/apache/storm/starter/tools/SlotBasedCounterTest.java index df2ab21d65c..90d21d1eb0f 100644 --- a/examples/storm-starter/test/jvm/org/apache/storm/starter/tools/SlotBasedCounterTest.java +++ b/examples/storm-starter/test/jvm/org/apache/storm/starter/tools/SlotBasedCounterTest.java @@ -1,23 +1,29 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.starter.tools; +import static org.fest.assertions.api.Assertions.assertThat; + import java.util.Map; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; -import static org.fest.assertions.api.Assertions.assertThat; - public class SlotBasedCounterTest { private static final int ANY_NUM_SLOTS = 1; diff --git a/external/storm-autocreds/pom.xml b/external/storm-autocreds/pom.xml index 80ecdc4f1ba..bdf1c3cff82 100644 --- a/external/storm-autocreds/pom.xml +++ b/external/storm-autocreds/pom.xml @@ -112,6 +112,16 @@ + + org.openrewrite.maven + rewrite-maven-plugin + + + + org.codehaus.gmavenplus + gmavenplus-plugin + + org.apache.maven.plugins maven-checkstyle-plugin diff --git a/external/storm-autocreds/src/main/java/org/apache/storm/common/AbstractHadoopAutoCreds.java b/external/storm-autocreds/src/main/java/org/apache/storm/common/AbstractHadoopAutoCreds.java index e8a7ac791d6..0680b3eea43 100644 --- a/external/storm-autocreds/src/main/java/org/apache/storm/common/AbstractHadoopAutoCreds.java +++ b/external/storm-autocreds/src/main/java/org/apache/storm/common/AbstractHadoopAutoCreds.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -26,7 +26,6 @@ import java.util.Set; import javax.security.auth.Subject; import javax.xml.bind.DatatypeConverter; - import org.apache.commons.math3.util.Pair; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; @@ -37,7 +36,8 @@ import org.slf4j.LoggerFactory; /** - * The base class that for auto credential plugins that abstracts out some of the common functionality. + * The base class that for auto credential plugins that abstracts out some of the common + * functionality. */ public abstract class AbstractHadoopAutoCreds implements IAutoCredentials, CredentialKeyProvider { private static final Logger LOG = LoggerFactory.getLogger(AbstractHadoopAutoCreds.class); @@ -118,13 +118,15 @@ private void addTokensToUgi(Subject subject) { continue; } - LOG.debug("Current user: {}", UserGroupInformation.getCurrentUser()); + LOG.debug("Current user: {}", UserGroupInformation + .getCurrentUser()); LOG.debug("Token from Credentials : {}", token); TokenIdentifier tokenId = token.decodeIdentifier(); if (tokenId != null) { LOG.debug("Token identifier : {}", tokenId); - LOG.debug("Username in token identifier : {}", tokenId.getUser()); + LOG.debug("Username in token identifier : {}", tokenId + .getUser()); } UserGroupInformation.getCurrentUser().addToken(token); diff --git a/external/storm-autocreds/src/main/java/org/apache/storm/common/AbstractHadoopNimbusPluginAutoCreds.java b/external/storm-autocreds/src/main/java/org/apache/storm/common/AbstractHadoopNimbusPluginAutoCreds.java index ad3bd17886f..605e6bb16b4 100644 --- a/external/storm-autocreds/src/main/java/org/apache/storm/common/AbstractHadoopNimbusPluginAutoCreds.java +++ b/external/storm-autocreds/src/main/java/org/apache/storm/common/AbstractHadoopNimbusPluginAutoCreds.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -25,7 +25,6 @@ import java.util.Map; import java.util.Set; import javax.xml.bind.DatatypeConverter; - import org.apache.commons.math3.util.Pair; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; @@ -38,11 +37,13 @@ import org.slf4j.LoggerFactory; /** - * The base class that for auto credential plugins that abstracts out some of the common functionality. + * The base class that for auto credential plugins that abstracts out some of the common + * functionality. */ public abstract class AbstractHadoopNimbusPluginAutoCreds implements INimbusCredentialPlugin, ICredentialsRenewer, CredentialKeyProvider { - private static final Logger LOG = LoggerFactory.getLogger(AbstractHadoopNimbusPluginAutoCreds.class); + private static final Logger LOG = LoggerFactory + .getLogger(AbstractHadoopNimbusPluginAutoCreds.class); public static final String CONFIG_KEY_RESOURCES = "resources"; @Override @@ -59,11 +60,13 @@ public void populateCredentials(Map credentials, if (!configKeys.isEmpty()) { for (String configKey : configKeys) { credentials.put(getCredentialKey(configKey), - DatatypeConverter.printBase64Binary(getHadoopCredentials(topologyConf, configKey))); + DatatypeConverter.printBase64Binary(getHadoopCredentials(topologyConf, + configKey))); } } else { credentials.put(getCredentialKey(""), - DatatypeConverter.printBase64Binary(getHadoopCredentials(topologyConf, topologyOwnerPrincipal))); + DatatypeConverter.printBase64Binary(getHadoopCredentials(topologyConf, + topologyOwnerPrincipal))); } LOG.info("Tokens added to credentials map."); } catch (Exception e) { @@ -72,7 +75,8 @@ public void populateCredentials(Map credentials, } @Override - public void renew(Map credentials, Map topologyConf, final String topologyOwnerPrincipal) { + public void renew(Map credentials, Map topologyConf, + final String topologyOwnerPrincipal) { doRenew(credentials, topologyConf, topologyOwnerPrincipal); } @@ -81,9 +85,11 @@ protected Set> getCredentials(Map cred return HadoopCredentialUtil.getCredential(this, credentials, configKeys); } - protected void fillHadoopConfiguration(Map topologyConf, String configKey, Configuration configuration) { + protected void fillHadoopConfiguration(Map topologyConf, String configKey, + Configuration configuration) { Map config = (Map) topologyConf.get(configKey); - LOG.info("TopoConf {}, got config {}, for configKey {}", ConfigUtils.maskPasswords(topologyConf), + LOG.info("TopoConf {}, got config {}, for configKey {}", ConfigUtils + .maskPasswords(topologyConf), ConfigUtils.maskPasswords(config), configKey); if (config != null) { List resourcesToLoad = new ArrayList<>(); @@ -118,11 +124,14 @@ protected void fillHadoopConfiguration(Map topologyConf, String configKey, Confi */ protected abstract String getConfigKeyString(); - protected abstract byte[] getHadoopCredentials(Map topologyConf, String configKey, String topologyOwnerPrincipal); + protected abstract byte[] getHadoopCredentials(Map topologyConf, + String configKey, String topologyOwnerPrincipal); - protected abstract byte[] getHadoopCredentials(Map topologyConf, String topologyOwnerPrincipal); + protected abstract byte[] getHadoopCredentials(Map topologyConf, + String topologyOwnerPrincipal); - protected abstract void doRenew(Map credentials, Map topologyConf, String topologyOwnerPrincipal); + protected abstract void doRenew(Map credentials, Map topologyConf, String topologyOwnerPrincipal); protected List getConfigKeys(Map conf) { String configKeyString = getConfigKeyString(); diff --git a/external/storm-autocreds/src/main/java/org/apache/storm/common/HadoopCredentialUtil.java b/external/storm-autocreds/src/main/java/org/apache/storm/common/HadoopCredentialUtil.java index 8e7c64bc2f1..7dfb0c340b6 100644 --- a/external/storm-autocreds/src/main/java/org/apache/storm/common/HadoopCredentialUtil.java +++ b/external/storm-autocreds/src/main/java/org/apache/storm/common/HadoopCredentialUtil.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -25,7 +25,6 @@ import java.util.Map; import java.util.Set; import javax.xml.bind.DatatypeConverter; - import org.apache.commons.math3.util.Pair; import org.apache.hadoop.security.Credentials; import org.slf4j.Logger; diff --git a/external/storm-autocreds/src/main/java/org/apache/storm/hbase/security/AutoHBase.java b/external/storm-autocreds/src/main/java/org/apache/storm/hbase/security/AutoHBase.java index 549f2d94f5f..2c9bb0197f8 100644 --- a/external/storm-autocreds/src/main/java/org/apache/storm/hbase/security/AutoHBase.java +++ b/external/storm-autocreds/src/main/java/org/apache/storm/hbase/security/AutoHBase.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -21,7 +21,6 @@ import static org.apache.storm.hbase.security.HBaseSecurityUtil.HBASE_CREDENTIALS; import java.util.Map; - import org.apache.storm.common.AbstractHadoopAutoCreds; /** diff --git a/external/storm-autocreds/src/main/java/org/apache/storm/hbase/security/AutoHBaseCommand.java b/external/storm-autocreds/src/main/java/org/apache/storm/hbase/security/AutoHBaseCommand.java index 518f2130b03..815574b5b79 100644 --- a/external/storm-autocreds/src/main/java/org/apache/storm/hbase/security/AutoHBaseCommand.java +++ b/external/storm-autocreds/src/main/java/org/apache/storm/hbase/security/AutoHBaseCommand.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -24,7 +24,6 @@ import java.util.HashMap; import java.util.Map; import javax.security.auth.Subject; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -50,7 +49,8 @@ public static void main(String[] args) throws Exception { autoHBaseNimbus.prepare(conf); Map creds = new HashMap<>(); - autoHBaseNimbus.populateCredentials(creds, conf, args[0]); //with realm e.g. storm@WITZEND.COM + autoHBaseNimbus.populateCredentials(creds, conf, + args[0]); // with realm e.g. storm@WITZEND.COM LOG.info("Got HBase credentials" + autoHBase.getCredentials(creds)); Subject s = new Subject(); diff --git a/external/storm-autocreds/src/main/java/org/apache/storm/hbase/security/AutoHBaseNimbus.java b/external/storm-autocreds/src/main/java/org/apache/storm/hbase/security/AutoHBaseNimbus.java index 84ce57e0844..6f102a64954 100644 --- a/external/storm-autocreds/src/main/java/org/apache/storm/hbase/security/AutoHBaseNimbus.java +++ b/external/storm-autocreds/src/main/java/org/apache/storm/hbase/security/AutoHBaseNimbus.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -26,7 +26,6 @@ import java.io.ObjectOutputStream; import java.net.InetAddress; import java.util.Map; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.client.Connection; @@ -61,37 +60,43 @@ protected String getConfigKeyString() { @Override public void shutdown() { - //no op. + // no op. } @Override - protected byte[] getHadoopCredentials(Map conf, String configKey, final String topologyOwnerPrincipal) { + protected byte[] getHadoopCredentials(Map conf, String configKey, + final String topologyOwnerPrincipal) { Configuration configuration = getHadoopConfiguration(conf, configKey); return getHadoopCredentials(conf, configuration, topologyOwnerPrincipal); } @Override - protected byte[] getHadoopCredentials(Map conf, final String topologyOwnerPrincipal) { + protected byte[] getHadoopCredentials(Map conf, + final String topologyOwnerPrincipal) { return getHadoopCredentials(conf, HBaseConfiguration.create(), topologyOwnerPrincipal); } @SuppressWarnings("unchecked") - protected byte[] getHadoopCredentials(Map conf, Configuration hbaseConf, final String topologySubmitterUser) { + protected byte[] getHadoopCredentials(Map conf, Configuration hbaseConf, + final String topologySubmitterUser) { try { if (UserGroupInformation.isSecurityEnabled()) { UserProvider provider = UserProvider.instantiate(hbaseConf); - provider.login(HBASE_KEYTAB_FILE_KEY, HBASE_PRINCIPAL_KEY, InetAddress.getLocalHost().getCanonicalHostName()); + provider.login(HBASE_KEYTAB_FILE_KEY, HBASE_PRINCIPAL_KEY, InetAddress + .getLocalHost().getCanonicalHostName()); LOG.info("Logged into Hbase as principal = " + hbaseConf.get(HBASE_PRINCIPAL_KEY)); UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); - final UserGroupInformation proxyUser = UserGroupInformation.createProxyUser(topologySubmitterUser, ugi); + final UserGroupInformation proxyUser = UserGroupInformation + .createProxyUser(topologySubmitterUser, ugi); User user = User.create(proxyUser); if (user.isHBaseSecurityEnabled(hbaseConf)) { - final Connection connection = ConnectionFactory.createConnection(hbaseConf, user); + final Connection connection = ConnectionFactory.createConnection(hbaseConf, + user); ClientTokenUtil.obtainAndCacheToken(connection, user); LOG.info("Obtained HBase tokens, adding to user credentials."); @@ -127,8 +132,9 @@ private Configuration getHadoopConfiguration(Map topoConf, Strin } @Override - public void doRenew(Map credentials, Map topologyConf, final String topologySubmitterUser) { - //HBASE tokens are not renewable so we always have to get new ones. + public void doRenew(Map credentials, Map topologyConf, + final String topologySubmitterUser) { + // HBASE tokens are not renewable so we always have to get new ones. populateCredentials(credentials, topologyConf, topologySubmitterUser); } diff --git a/external/storm-autocreds/src/main/java/org/apache/storm/hbase/security/HBaseSecurityUtil.java b/external/storm-autocreds/src/main/java/org/apache/storm/hbase/security/HBaseSecurityUtil.java index 1d8e5b865a8..323e3a70d84 100644 --- a/external/storm-autocreds/src/main/java/org/apache/storm/hbase/security/HBaseSecurityUtil.java +++ b/external/storm-autocreds/src/main/java/org/apache/storm/hbase/security/HBaseSecurityUtil.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -24,7 +24,6 @@ import java.net.InetAddress; import java.util.List; import java.util.Map; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.security.UserProvider; import org.apache.hadoop.security.UserGroupInformation; @@ -52,14 +51,17 @@ public class HBaseSecurityUtil { private HBaseSecurityUtil() { } - public static UserProvider login(Map conf, Configuration hbaseConfig) throws IOException { - //Allowing keytab based login for backward compatibility. + public static UserProvider login(Map conf, + Configuration hbaseConfig) throws IOException { + // Allowing keytab based login for backward compatibility. if (UserGroupInformation.isSecurityEnabled()) { List autoCredentials = (List) conf.get(TOPOLOGY_AUTO_CREDENTIALS); if ((autoCredentials == null) - || (!autoCredentials.contains(AutoHBase.class.getName()) && !autoCredentials.contains(AutoTGT.class.getName()))) { - LOG.info("Logging in using keytab as neither AutoHBase or AutoTGT is specified for " + TOPOLOGY_AUTO_CREDENTIALS); - //insure that if keytab is used only one login per process executed + || (!autoCredentials.contains(AutoHBase.class.getName()) && !autoCredentials + .contains(AutoTGT.class.getName()))) { + LOG.info("Logging in using keytab as neither AutoHBase or AutoTGT is specified " + + "for " + TOPOLOGY_AUTO_CREDENTIALS); + // insure that if keytab is used only one login per process executed if (legacyProvider == null) { synchronized (HBaseSecurityUtil.class) { if (legacyProvider == null) { diff --git a/external/storm-autocreds/src/main/java/org/apache/storm/hdfs/security/AutoHDFS.java b/external/storm-autocreds/src/main/java/org/apache/storm/hdfs/security/AutoHDFS.java index a5d1a03a335..b90fbdf8e49 100644 --- a/external/storm-autocreds/src/main/java/org/apache/storm/hdfs/security/AutoHDFS.java +++ b/external/storm-autocreds/src/main/java/org/apache/storm/hdfs/security/AutoHDFS.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -21,7 +21,6 @@ import static org.apache.storm.hdfs.security.HdfsSecurityUtil.HDFS_CREDENTIALS; import java.util.Map; - import org.apache.storm.common.AbstractHadoopAutoCreds; /** diff --git a/external/storm-autocreds/src/main/java/org/apache/storm/hdfs/security/AutoHDFSCommand.java b/external/storm-autocreds/src/main/java/org/apache/storm/hdfs/security/AutoHDFSCommand.java index b8e4396a930..6e593e10c0f 100644 --- a/external/storm-autocreds/src/main/java/org/apache/storm/hdfs/security/AutoHDFSCommand.java +++ b/external/storm-autocreds/src/main/java/org/apache/storm/hdfs/security/AutoHDFSCommand.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -24,7 +24,6 @@ import java.util.HashMap; import java.util.Map; import javax.security.auth.Subject; - import org.apache.storm.Config; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -42,7 +41,7 @@ private AutoHDFSCommand() { @SuppressWarnings("unchecked") public static void main(String[] args) throws Exception { Map conf = new HashMap<>(); - conf.put(STORM_USER_NAME_KEY, args[1]); //with realm e.g. hdfs@WITZEND.COM + conf.put(STORM_USER_NAME_KEY, args[1]); // with realm e.g. hdfs@WITZEND.COM conf.put(STORM_KEYTAB_FILE_KEY, args[2]); // /etc/security/keytabs/storm.keytab AutoHDFS autoHdfs = new AutoHDFS(); diff --git a/external/storm-autocreds/src/main/java/org/apache/storm/hdfs/security/AutoHDFSNimbus.java b/external/storm-autocreds/src/main/java/org/apache/storm/hdfs/security/AutoHDFSNimbus.java index 200ae4747d8..a91c46ea7fa 100644 --- a/external/storm-autocreds/src/main/java/org/apache/storm/hdfs/security/AutoHDFSNimbus.java +++ b/external/storm-autocreds/src/main/java/org/apache/storm/hdfs/security/AutoHDFSNimbus.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -31,7 +31,6 @@ import java.util.Collection; import java.util.List; import java.util.Map; - import org.apache.commons.math3.util.Pair; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@ -71,22 +70,25 @@ protected String getConfigKeyString() { @Override public void shutdown() { - //no op. + // no op. } @Override - protected byte[] getHadoopCredentials(Map conf, String configKey, final String topologyOwnerPrincipal) { + protected byte[] getHadoopCredentials(Map conf, String configKey, + final String topologyOwnerPrincipal) { Configuration configuration = getHadoopConfiguration(conf, configKey); return getHadoopCredentials(conf, configuration, topologyOwnerPrincipal); } @Override - protected byte[] getHadoopCredentials(Map conf, final String topologyOwnerPrincipal) { + protected byte[] getHadoopCredentials(Map conf, + final String topologyOwnerPrincipal) { return getHadoopCredentials(conf, new Configuration(), topologyOwnerPrincipal); } @SuppressWarnings("unchecked") - private byte[] getHadoopCredentials(Map conf, final Configuration configuration, final String topologySubmitterUser) { + private byte[] getHadoopCredentials(Map conf, final Configuration configuration, + final String topologySubmitterUser) { try { if (UserGroupInformation.isSecurityEnabled()) { login(configuration); @@ -97,7 +99,8 @@ private byte[] getHadoopCredentials(Map conf, final Configuratio UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); - final UserGroupInformation proxyUser = UserGroupInformation.createProxyUser(topologySubmitterUser, ugi); + final UserGroupInformation proxyUser = UserGroupInformation + .createProxyUser(topologySubmitterUser, ugi); Credentials creds = (Credentials) proxyUser.doAs(new PrivilegedAction() { @Override @@ -110,8 +113,10 @@ public Object run() { configuration.set(STORM_USER_NAME_KEY, hdfsPrincipal); } - fileSystem.addDelegationTokens(configuration.get(STORM_USER_NAME_KEY), credential); - LOG.info("Delegation tokens acquired for user {}", topologySubmitterUser); + fileSystem.addDelegationTokens(configuration.get(STORM_USER_NAME_KEY), + credential); + LOG.info("Delegation tokens acquired for user {}", + topologySubmitterUser); return credential; } catch (IOException e) { throw new RuntimeException(e); @@ -146,26 +151,30 @@ private Configuration getHadoopConfiguration(Map topoConf, Strin * {@inheritDoc} */ @Override - public void doRenew(Map credentials, Map topologyConf, final String topologyOwnerPrincipal) { + public void doRenew(Map credentials, Map topologyConf, + final String topologyOwnerPrincipal) { List confKeys = getConfigKeys(topologyConf); for (Pair cred : getCredentials(credentials, confKeys)) { try { Configuration configuration = getHadoopConfiguration(topologyConf, cred.getFirst()); - Collection> tokens = cred.getSecond().getAllTokens(); + Collection> tokens = cred.getSecond() + .getAllTokens(); if (tokens != null && !tokens.isEmpty()) { for (Token token : tokens) { - //We need to re-login some other thread might have logged into hadoop using + // We need to re-login some other thread might have logged into hadoop using // their credentials (e.g. AutoHBase might be also part of nimbu auto creds) login(configuration); long expiration = token.renew(configuration); - LOG.info("HDFS delegation token renewed, new expiration time {}", expiration); + LOG.info("HDFS delegation token renewed, new expiration time {}", + expiration); } } else { LOG.debug("No tokens found for credentials, skipping renewal."); } } catch (Exception e) { - LOG.warn("could not renew the credentials, one of the possible reason is tokens are beyond " + LOG.warn("could not renew the credentials, one of the possible reason is tokens " + + "are beyond " + "renewal period so attempting to get new tokens.", e); populateCredentials(credentials, topologyConf, topologyOwnerPrincipal); diff --git a/external/storm-autocreds/src/main/java/org/apache/storm/hdfs/security/HdfsSecurityUtil.java b/external/storm-autocreds/src/main/java/org/apache/storm/hdfs/security/HdfsSecurityUtil.java index c89b845273d..1737c77ea64 100644 --- a/external/storm-autocreds/src/main/java/org/apache/storm/hdfs/security/HdfsSecurityUtil.java +++ b/external/storm-autocreds/src/main/java/org/apache/storm/hdfs/security/HdfsSecurityUtil.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -24,13 +24,10 @@ import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; - import org.apache.storm.security.auth.kerberos.AutoTGT; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -51,15 +48,20 @@ public final class HdfsSecurityUtil { private HdfsSecurityUtil() { } - public static void login(Map conf, Configuration hdfsConfig) throws IOException { - //If AutoHDFS is specified, do not attempt to login using keytabs, only kept for backward compatibility. + public static void login(Map conf, + Configuration hdfsConfig) throws IOException { + // If AutoHDFS is specified, do not attempt to login using keytabs, only kept for backward + // compatibility. if (conf.get(TOPOLOGY_AUTO_CREDENTIALS) == null - || (!(((List) conf.get(TOPOLOGY_AUTO_CREDENTIALS)).contains(AutoHDFS.class.getName())) - && !(((List) conf.get(TOPOLOGY_AUTO_CREDENTIALS)).contains(AutoTGT.class.getName())))) { + || (!(((List) conf.get(TOPOLOGY_AUTO_CREDENTIALS)).contains(AutoHDFS.class + .getName())) + && !(((List) conf.get(TOPOLOGY_AUTO_CREDENTIALS)).contains(AutoTGT.class + .getName())))) { if (UserGroupInformation.isSecurityEnabled()) { // compareAndSet added because of https://issues.apache.org/jira/browse/STORM-1535 if (isLoggedIn.compareAndSet(false, true)) { - LOG.info("Logging in using keytab as AutoHDFS is not specified for " + TOPOLOGY_AUTO_CREDENTIALS); + LOG.info("Logging in using keytab as AutoHDFS is not specified for " + + TOPOLOGY_AUTO_CREDENTIALS); String keytab = (String) conf.get(STORM_KEYTAB_FILE_KEY); if (keytab != null) { hdfsConfig.set(STORM_KEYTAB_FILE_KEY, keytab); diff --git a/external/storm-blobstore-migration/pom.xml b/external/storm-blobstore-migration/pom.xml index ae3bfa3b8a3..9150aa0a220 100644 --- a/external/storm-blobstore-migration/pom.xml +++ b/external/storm-blobstore-migration/pom.xml @@ -146,6 +146,16 @@ limitations under the License. + + org.openrewrite.maven + rewrite-maven-plugin + + + + org.codehaus.gmavenplus + gmavenplus-plugin + + org.apache.maven.plugins maven-checkstyle-plugin diff --git a/external/storm-blobstore-migration/src/main/java/org/apache/storm/blobstore/ListHDFS.java b/external/storm-blobstore-migration/src/main/java/org/apache/storm/blobstore/ListHDFS.java index cfa71312c24..78bf6e31642 100644 --- a/external/storm-blobstore-migration/src/main/java/org/apache/storm/blobstore/ListHDFS.java +++ b/external/storm-blobstore-migration/src/main/java/org/apache/storm/blobstore/ListHDFS.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -19,9 +19,7 @@ package org.apache.storm.blobstore; import java.util.Map; - import javax.security.auth.Subject; - import org.apache.storm.Config; import org.apache.storm.blobstore.ClientBlobStore; import org.apache.storm.hdfs.blobstore.HdfsBlobStore; @@ -33,12 +31,14 @@ public class ListHDFS { public static void main(String[] args) throws Exception { if (args.length < 1) { - System.out.println("Need at least 1 argument (hdfs_blobstore_path), but have " + Integer.toString(args.length)); + System.out.println("Need at least 1 argument (hdfs_blobstore_path), but have " + Integer + .toString(args.length)); System.out.println("listHDFS "); System.out.println("Lists blobs in HdfsBlobStore"); System.out.println("Example: listHDFS " + "'hdfs://some-hdfs-namenode:8080/srv/storm/my-storm-blobstore' " - + "'stormUser/my-nimbus-host.example.com@STORM.EXAMPLE.COM' '/srv/my-keytab/stormUser.kt'"); + + "'stormUser/my-nimbus-host.example.com@STORM.EXAMPLE.COM' " + + "'/srv/my-keytab/stormUser.kt'"); System.exit(1); } @@ -46,7 +46,8 @@ public static void main(String[] args) throws Exception { String hdfsBlobstorePath = args[0]; hdfsConf.put(Config.BLOBSTORE_DIR, hdfsBlobstorePath); - hdfsConf.put(Config.STORM_PRINCIPAL_TO_LOCAL_PLUGIN, "org.apache.storm.security.auth.DefaultPrincipalToLocal"); + hdfsConf.put(Config.STORM_PRINCIPAL_TO_LOCAL_PLUGIN, + "org.apache.storm.security.auth.DefaultPrincipalToLocal"); if (args.length >= 2) { System.out.println("SETTING HDFS PRINCIPAL!"); hdfsConf.put(Config.STORM_HDFS_LOGIN_PRINCIPAL, args[1]); diff --git a/external/storm-blobstore-migration/src/main/java/org/apache/storm/blobstore/ListLocalFs.java b/external/storm-blobstore-migration/src/main/java/org/apache/storm/blobstore/ListLocalFs.java index aefddb8869b..e3721b0e03b 100644 --- a/external/storm-blobstore-migration/src/main/java/org/apache/storm/blobstore/ListLocalFs.java +++ b/external/storm-blobstore-migration/src/main/java/org/apache/storm/blobstore/ListLocalFs.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -19,9 +19,7 @@ package org.apache.storm.blobstore; import java.util.Map; - import javax.security.auth.Subject; - import org.apache.storm.Config; import org.apache.storm.blobstore.LocalFsBlobStore; import org.apache.storm.nimbus.NimbusInfo; @@ -41,7 +39,8 @@ public static void main(String[] args) throws Exception { Map lfsConf = Utils.readStormConfig(); lfsConf.put(Config.BLOBSTORE_DIR, args[0]); - lfsConf.put(Config.STORM_PRINCIPAL_TO_LOCAL_PLUGIN, "org.apache.storm.security.auth.DefaultPrincipalToLocal"); + lfsConf.put(Config.STORM_PRINCIPAL_TO_LOCAL_PLUGIN, + "org.apache.storm.security.auth.DefaultPrincipalToLocal"); /* CREATE THE BLOBSTORE */ LocalFsBlobStore lfsBlobStore = new LocalFsBlobStore(); diff --git a/external/storm-blobstore-migration/src/main/java/org/apache/storm/blobstore/MigrateBlobs.java b/external/storm-blobstore-migration/src/main/java/org/apache/storm/blobstore/MigrateBlobs.java index e7a3581637d..9ea2435f105 100644 --- a/external/storm-blobstore-migration/src/main/java/org/apache/storm/blobstore/MigrateBlobs.java +++ b/external/storm-blobstore-migration/src/main/java/org/apache/storm/blobstore/MigrateBlobs.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -21,10 +21,8 @@ import java.io.IOException; import java.io.InputStream; import java.util.Map; - import javax.security.auth.Subject; import javax.security.auth.login.LoginContext; - import org.apache.storm.Config; import org.apache.storm.blobstore.BlobStore; import org.apache.storm.blobstore.LocalFsBlobStore; @@ -39,7 +37,8 @@ public class MigrateBlobs { - protected static void deleteAllBlobStoreKeys(BlobStore bs, Subject who) throws AuthorizationException, KeyNotFoundException { + protected static void deleteAllBlobStoreKeys(BlobStore bs, + Subject who) throws AuthorizationException, KeyNotFoundException { Iterable hdfsKeys = () -> bs.listKeys(); for (String key : hdfsKeys) { System.out.println(key); @@ -63,25 +62,29 @@ protected static void copyBlobStoreKeys(BlobStore bsFrom, System.out.println("DONE CREATING BLOB " + key); } } - - + public static void main(String[] args) throws Exception { Map hdfsConf = Utils.readStormConfig(); if (args.length < 2) { - System.out.println("Need at least 2 arguments, but have " + Integer.toString(args.length)); - System.out.println("migrate "); + System.out.println("Need at least 2 arguments, but have " + Integer + .toString(args.length)); + System.out + .println("migrate " + + " "); System.out.println("Migrates blobs from LocalFsBlobStore to HdfsBlobStore"); System.out.println("Example: migrate '/srv/storm' " + "'hdfs://some-hdfs-namenode:8080/srv/storm/my-storm-blobstore' " - + "'stormUser/my-nimbus-host.example.com@STORM.EXAMPLE.COM' '/srv/my-keytab/stormUser.kt'"); + + "'stormUser/my-nimbus-host.example.com@STORM.EXAMPLE.COM' " + + "'/srv/my-keytab/stormUser.kt'"); System.exit(1); } String hdfsBlobstorePath = args[1]; hdfsConf.put(Config.BLOBSTORE_DIR, hdfsBlobstorePath); - hdfsConf.put(Config.STORM_PRINCIPAL_TO_LOCAL_PLUGIN, "org.apache.storm.security.auth.DefaultPrincipalToLocal"); + hdfsConf.put(Config.STORM_PRINCIPAL_TO_LOCAL_PLUGIN, + "org.apache.storm.security.auth.DefaultPrincipalToLocal"); if (args.length >= 3) { System.out.println("SETTING HDFS PRINCIPAL!"); hdfsConf.put(Config.STORM_HDFS_LOGIN_PRINCIPAL, args[2]); @@ -95,7 +98,8 @@ public static void main(String[] args) throws Exception { Map lfsConf = Utils.readStormConfig(); String localBlobstoreDir = args[0]; lfsConf.put(Config.BLOBSTORE_DIR, localBlobstoreDir); - lfsConf.put(Config.STORM_PRINCIPAL_TO_LOCAL_PLUGIN, "org.apache.storm.security.auth.DefaultPrincipalToLocal"); + lfsConf.put(Config.STORM_PRINCIPAL_TO_LOCAL_PLUGIN, + "org.apache.storm.security.auth.DefaultPrincipalToLocal"); /* CREATE THE BLOBSTORES */ @@ -117,7 +121,9 @@ public static void main(String[] args) throws Exception { System.out.println("Done listing HDFS blobstore keys."); - System.out.println("Going to delete everything in HDFS, then copy all local blobs to HDFS. Continue? [Y/n]"); + System.out + .println("Going to delete everything in HDFS, then copy all local blobs to HDFS. " + + "Continue? [Y/n]"); String resp = System.console().readLine().toLowerCase().trim(); if (!(resp.equals("y") || resp.equals(""))) { System.out.println("Not copying blobs. Exiting. [" + resp.toLowerCase().trim() + "]"); diff --git a/external/storm-blobstore-migration/src/main/java/org/apache/storm/blobstore/MigratorMain.java b/external/storm-blobstore-migration/src/main/java/org/apache/storm/blobstore/MigratorMain.java index 03d163aad27..678bb420f0f 100644 --- a/external/storm-blobstore-migration/src/main/java/org/apache/storm/blobstore/MigratorMain.java +++ b/external/storm-blobstore-migration/src/main/java/org/apache/storm/blobstore/MigratorMain.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -19,7 +19,6 @@ package org.apache.storm.blobstore; import java.util.Arrays; - import javax.security.auth.Subject; public class MigratorMain { diff --git a/external/storm-hdfs-blobstore/pom.xml b/external/storm-hdfs-blobstore/pom.xml index a33bbca96e6..fe31d7fabe3 100644 --- a/external/storm-hdfs-blobstore/pom.xml +++ b/external/storm-hdfs-blobstore/pom.xml @@ -153,6 +153,16 @@ + + org.openrewrite.maven + rewrite-maven-plugin + + + + org.codehaus.gmavenplus + gmavenplus-plugin + + org.apache.maven.plugins maven-checkstyle-plugin diff --git a/external/storm-hdfs-blobstore/src/main/java/org/apache/storm/hdfs/blobstore/HdfsBlobStore.java b/external/storm-hdfs-blobstore/src/main/java/org/apache/storm/hdfs/blobstore/HdfsBlobStore.java index 7d957186a21..dea33e29175 100644 --- a/external/storm-hdfs-blobstore/src/main/java/org/apache/storm/hdfs/blobstore/HdfsBlobStore.java +++ b/external/storm-hdfs-blobstore/src/main/java/org/apache/storm/hdfs/blobstore/HdfsBlobStore.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -59,18 +59,24 @@ * Note that this provides an api for having HDFS be the backing store for the blobstore, * it is not a service/daemon. * - *

We currently have NIMBUS_ADMINS and SUPERVISOR_ADMINS configuration. NIMBUS_ADMINS are given READ, WRITE and ADMIN - * access whereas the SUPERVISOR_ADMINS are given READ access in order to read and download the blobs form the nimbus. + *

We currently have NIMBUS_ADMINS and SUPERVISOR_ADMINS configuration. NIMBUS_ADMINS are given + * READ, WRITE and ADMIN + * access whereas the SUPERVISOR_ADMINS are given READ access in order to read and download the + * blobs form the nimbus. * - *

The ACLs for the blob store are validated against whether the subject is a NIMBUS_ADMIN, SUPERVISOR_ADMIN or USER + *

The ACLs for the blob store are validated against whether the subject is a NIMBUS_ADMIN, + * SUPERVISOR_ADMIN or USER * who has read, write or admin privileges in order to perform respective operations on the blob. * *

For hdfs blob store - * 1. The USER interacts with nimbus to upload and access blobs through NimbusBlobStore Client API. Here, unlike + * 1. The USER interacts with nimbus to upload and access blobs through NimbusBlobStore Client API. + * Here, unlike * local blob store which stores the blobs locally, the nimbus talks to HDFS to upload the blobs. * 2. The USER sets the ACLs, and the blob access is validated against these ACLs. - * 3. The SUPERVISOR interacts with nimbus through HdfsClientBlobStore to download the blobs. Here, unlike local - * blob store the supervisor interacts with HDFS directly to download the blobs. The call to HdfsBlobStore is made as a "null" + * 3. The SUPERVISOR interacts with nimbus through HdfsClientBlobStore to download the blobs. Here, + * unlike local + * blob store the supervisor interacts with HDFS directly to download the blobs. The call to + * HdfsBlobStore is made as a "null" * subject. The blobstore gets the hadoop user and validates permissions for the supervisor. */ public class HdfsBlobStore extends BlobStore { @@ -82,8 +88,10 @@ public class HdfsBlobStore extends BlobStore { private HdfsBlobStoreImpl hbs; private Subject localSubject; private Map conf; - private Cache cacheMetas = CacheBuilder.newBuilder().expireAfterWrite(10, TimeUnit.MINUTES).build(); - private Cache cachedReplicationCount = CacheBuilder.newBuilder().expireAfterWrite(10, TimeUnit.MINUTES).build(); + private Cache cacheMetas = CacheBuilder.newBuilder() + .expireAfterWrite(10, TimeUnit.MINUTES).build(); + private Cache cachedReplicationCount = CacheBuilder.newBuilder() + .expireAfterWrite(10, TimeUnit.MINUTES).build(); /** * If who is null then we want to use the user hadoop says we are. @@ -98,7 +106,8 @@ private Subject checkAndGetSubject(Subject who) { } @Override - public void prepare(Map conf, String overrideBase, NimbusInfo nimbusInfo, ILeaderElector leaderElector) { + public void prepare(Map conf, String overrideBase, NimbusInfo nimbusInfo, + ILeaderElector leaderElector) { this.conf = conf; prepareInternal(conf, overrideBase, null); } @@ -107,7 +116,8 @@ public void prepare(Map conf, String overrideBase, NimbusInfo ni * Allow a Hadoop Configuration to be passed for testing. If it's null then the hadoop configs * must be in your classpath. */ - protected void prepareInternal(Map conf, String overrideBase, Configuration hadoopConf) { + protected void prepareInternal(Map conf, String overrideBase, + Configuration hadoopConf) { this.conf = conf; if (overrideBase == null) { overrideBase = (String) conf.get(Config.BLOBSTORE_DIR); @@ -117,7 +127,7 @@ protected void prepareInternal(Map conf, String overrideBase, Co } LOG.debug("directory is: {}", overrideBase); - //Login to hdfs + // Login to hdfs localSubject = HadoopLoginUtil.loginHadoop(conf); aclHandler = new BlobStoreAclHandler(conf); @@ -166,7 +176,7 @@ public AtomicOutputStream createBlob(String key, SettableBlobMeta meta, Subject try { outputStream.cancel(); } catch (IOException e) { - //Ignored + // Ignored } } } @@ -205,7 +215,8 @@ private SettableBlobMeta getStoredBlobMeta(String key) throws KeyNotFoundExcepti } in.close(); in = null; - SettableBlobMeta blobMeta = Utils.thriftDeserialize(SettableBlobMeta.class, out.toByteArray()); + SettableBlobMeta blobMeta = Utils.thriftDeserialize(SettableBlobMeta.class, out + .toByteArray()); return blobMeta; } catch (IOException e) { throw new RuntimeException(e); @@ -214,7 +225,7 @@ private SettableBlobMeta getStoredBlobMeta(String key) throws KeyNotFoundExcepti try { in.close(); } catch (IOException e) { - //Ignored + // Ignored } } } @@ -324,11 +335,12 @@ public Iterator listKeys() { @Override public void shutdown() { - //Empty + // Empty } @Override - public int getBlobReplication(String key, Subject who) throws AuthorizationException, KeyNotFoundException { + public int getBlobReplication(String key, + Subject who) throws AuthorizationException, KeyNotFoundException { who = checkAndGetSubject(who); validateKey(key); SettableBlobMeta meta = extractBlobMeta(key); @@ -361,7 +373,8 @@ private SettableBlobMeta extractBlobMeta(String key) throws KeyNotFoundException } @Override - public int updateBlobReplication(String key, int replication, Subject who) throws AuthorizationException, KeyNotFoundException { + public int updateBlobReplication(String key, int replication, + Subject who) throws AuthorizationException, KeyNotFoundException { who = checkAndGetSubject(who); validateKey(key); SettableBlobMeta meta = extractBlobMeta(key); @@ -395,7 +408,7 @@ public void writeMetadata(String key, SettableBlobMeta meta) try { outputStream.cancel(); } catch (IOException e) { - //Ignored + // Ignored } } } diff --git a/external/storm-hdfs-blobstore/src/main/java/org/apache/storm/hdfs/blobstore/HdfsBlobStoreFile.java b/external/storm-hdfs-blobstore/src/main/java/org/apache/storm/hdfs/blobstore/HdfsBlobStoreFile.java index f124cdfd0fe..737f8ea23eb 100644 --- a/external/storm-hdfs-blobstore/src/main/java/org/apache/storm/hdfs/blobstore/HdfsBlobStoreFile.java +++ b/external/storm-hdfs-blobstore/src/main/java/org/apache/storm/hdfs/blobstore/HdfsBlobStoreFile.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -22,7 +22,6 @@ import java.io.InputStream; import java.io.OutputStream; import java.util.regex.Matcher; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.FileSystem; @@ -56,7 +55,8 @@ public HdfsBlobStoreFile(Path base, String name, Configuration hconf) { } else { Matcher m = TMP_NAME_PATTERN.matcher(name); if (!m.matches()) { - throw new IllegalArgumentException("File name does not match '" + name + "' !~ " + TMP_NAME_PATTERN); + throw new IllegalArgumentException("File name does not match '" + name + "' !~ " + + TMP_NAME_PATTERN); } isTmp = true; } @@ -136,14 +136,16 @@ public OutputStream getOutputStream() throws IOException { fileSystem.setPermission(path, fileperms); fileSystem.setReplication(path, (short) this.getMetadata().get_replication_factor()); } catch (IOException e) { - //Try to create the parent directory, may not work + // Try to create the parent directory, may not work FsPermission dirperms = new FsPermission(HdfsBlobStoreImpl.BLOBSTORE_DIR_PERMISSION); if (!fileSystem.mkdirs(path.getParent(), dirperms)) { LOG.warn("error creating parent dir: " + path.getParent()); } if (!fileSystem.getFileStatus(path.getParent()).getPermission().equals(dirperms)) { - LOG.warn("Directory {} created with unexpected permission {}.Set permission {} for this directory.", - path.getParent(), fileSystem.getFileStatus(path.getParent()).getPermission(), dirperms); + LOG.warn("Directory {} created with unexpected permission {}.Set permission {} " + + "for this directory.", + path.getParent(), fileSystem.getFileStatus(path.getParent()) + .getPermission(), dirperms); fileSystem.setPermission(path.getParent(), dirperms); } out = fileSystem.create(path, (short) this.getMetadata().get_replication_factor()); diff --git a/external/storm-hdfs-blobstore/src/main/java/org/apache/storm/hdfs/blobstore/HdfsBlobStoreImpl.java b/external/storm-hdfs-blobstore/src/main/java/org/apache/storm/hdfs/blobstore/HdfsBlobStoreImpl.java index 455cdd78569..c9f40fc69d3 100644 --- a/external/storm-hdfs-blobstore/src/main/java/org/apache/storm/hdfs/blobstore/HdfsBlobStoreImpl.java +++ b/external/storm-hdfs-blobstore/src/main/java/org/apache/storm/hdfs/blobstore/HdfsBlobStoreImpl.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -29,7 +29,6 @@ import java.util.NoSuchElementException; import java.util.Timer; import java.util.TimerTask; - import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; @@ -45,7 +44,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; - /** * HDFS blob store impl. */ @@ -144,7 +142,8 @@ public HdfsBlobStoreImpl(Path path, Map conf, FsPermission perms = new FsPermission(BLOBSTORE_DIR_PERMISSION); boolean success = fileSystem.mkdirs(fullPath, perms); if (!fileSystem.getFileStatus(fullPath).getPermission().equals(perms)) { - LOG.warn("Directory {} created with unexpected permission {}.Set permission {} for this directory.", + LOG.warn("Directory {} created with unexpected permission {}.Set permission {} " + + "for this directory.", fullPath, fileSystem.getFileStatus(fullPath).getPermission(), perms); fileSystem.setPermission(fullPath, perms); } @@ -189,7 +188,7 @@ protected Iterator listKeys(Path path) throws IOException { try { ret.add(sub.getPath().getName().toString()); } catch (IllegalArgumentException e) { - //Ignored the file did not match + // Ignored the file did not match LOG.debug("Found an unexpected file in {} {}", path, sub.getPath().getName()); } } @@ -269,7 +268,7 @@ public void fullCleanup(long age) throws IOException { Path keyDir = getKeyDir(key); Iterator i = listBlobStoreFiles(keyDir); if (!i.hasNext()) { - //The dir is empty, so try to delete it, may fail, but that is OK + // The dir is empty, so try to delete it, may fail, but that is OK try { fileSystem.delete(keyDir, true); } catch (Exception e) { @@ -293,10 +292,11 @@ protected Iterator listBlobStoreFiles(Path path) throws IOExcepti if (files != null) { for (FileStatus sub : files) { try { - ret.add(new HdfsBlobStoreFile(sub.getPath().getParent(), sub.getPath().getName(), + ret.add(new HdfsBlobStoreFile(sub.getPath().getParent(), sub.getPath() + .getName(), hadoopConf)); } catch (IllegalArgumentException e) { - //Ignored the file did not match + // Ignored the file did not match LOG.warn("Found an unexpected file in {} {}", path, sub.getPath().getName()); } } @@ -359,14 +359,17 @@ public synchronized void updateLastBlobUpdateTime() throws IOException { Long timestamp = Time.currentTimeMillis(); Path updateTimeFile = new Path(fullPath, BLOBSTORE_UPDATE_TIME_FILE); FSDataOutputStream fsDataOutputStream = fileSystem.create(updateTimeFile, true); - BufferedWriter bufferedWriter = new BufferedWriter(new OutputStreamWriter(fsDataOutputStream, StandardCharsets.UTF_8)); + BufferedWriter bufferedWriter = + new BufferedWriter(new OutputStreamWriter(fsDataOutputStream, + StandardCharsets.UTF_8)); bufferedWriter.write(timestamp.toString()); bufferedWriter.close(); LOG.debug("Updated blobstore update time of {} to {}", updateTimeFile, timestamp); } /** - * Validates that the last updated blob time of the blobstore is up to date with the current existing blobs. + * Validates that the last updated blob time of the blobstore is up to date with the current + * existing blobs. * * @throws IOException on any error */ diff --git a/external/storm-hdfs-blobstore/src/main/java/org/apache/storm/hdfs/blobstore/HdfsClientBlobStore.java b/external/storm-hdfs-blobstore/src/main/java/org/apache/storm/hdfs/blobstore/HdfsClientBlobStore.java index fbdc1866100..12eb7e4b5a4 100644 --- a/external/storm-hdfs-blobstore/src/main/java/org/apache/storm/hdfs/blobstore/HdfsClientBlobStore.java +++ b/external/storm-hdfs-blobstore/src/main/java/org/apache/storm/hdfs/blobstore/HdfsClientBlobStore.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -21,7 +21,6 @@ import java.io.IOException; import java.util.Iterator; import java.util.Map; - import org.apache.storm.blobstore.AtomicOutputStream; import org.apache.storm.blobstore.ClientBlobStore; import org.apache.storm.blobstore.InputStreamWithMeta; @@ -107,7 +106,8 @@ public int getBlobReplication(String key) throws AuthorizationException, KeyNotF } @Override - public int updateBlobReplication(String key, int replication) throws AuthorizationException, KeyNotFoundException { + public int updateBlobReplication(String key, + int replication) throws AuthorizationException, KeyNotFoundException { return blobStore.updateBlobReplication(key, replication, null); } diff --git a/external/storm-hdfs-blobstore/src/test/java/org/apache/storm/hdfs/blobstore/BlobStoreTest.java b/external/storm-hdfs-blobstore/src/test/java/org/apache/storm/hdfs/blobstore/BlobStoreTest.java index 206e6518a0c..c734368ea18 100644 --- a/external/storm-hdfs-blobstore/src/test/java/org/apache/storm/hdfs/blobstore/BlobStoreTest.java +++ b/external/storm-hdfs-blobstore/src/test/java/org/apache/storm/hdfs/blobstore/BlobStoreTest.java @@ -18,7 +18,21 @@ */ package org.apache.storm.hdfs.blobstore; -import org.apache.storm.hdfs.testing.MiniDFSClusterExtension; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import javax.security.auth.Subject; import org.apache.storm.Config; import org.apache.storm.blobstore.AtomicOutputStream; import org.apache.storm.blobstore.BlobStore; @@ -28,25 +42,10 @@ import org.apache.storm.generated.AuthorizationException; import org.apache.storm.generated.KeyNotFoundException; import org.apache.storm.generated.SettableBlobMeta; +import org.apache.storm.hdfs.testing.MiniDFSClusterExtension; import org.apache.storm.security.auth.FixedGroupsMapping; import org.apache.storm.security.auth.NimbusPrincipal; import org.apache.storm.security.auth.SingleUserPrincipal; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.security.auth.Subject; -import java.io.IOException; -import java.io.InputStream; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import static org.junit.jupiter.api.Assertions.*; - import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -54,11 +53,14 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; import org.junit.jupiter.params.provider.ValueSource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class BlobStoreTest { @RegisterExtension - public static final MiniDFSClusterExtension DFS_CLUSTER_EXTENSION = new MiniDFSClusterExtension(); + public static final MiniDFSClusterExtension DFS_CLUSTER_EXTENSION = + new MiniDFSClusterExtension(); private static final Logger LOG = LoggerFactory.getLogger(BlobStoreTest.class); private static final Map CONF = new HashMap<>(); @@ -88,19 +90,21 @@ public static void initializeConfigs() { // Now create a params map to put it in to our conf Map paramMap = new HashMap<>(); paramMap.put(FixedGroupsMapping.STORM_FIXED_GROUP_MAPPING, groupsMapping); - CONF.put(Config.STORM_GROUP_MAPPING_SERVICE_PROVIDER_PLUGIN, "org.apache.storm.security.auth.FixedGroupsMapping"); + CONF.put(Config.STORM_GROUP_MAPPING_SERVICE_PROVIDER_PLUGIN, + "org.apache.storm.security.auth.FixedGroupsMapping"); CONF.put(Config.STORM_GROUP_MAPPING_SERVICE_PARAMS, paramMap); CONF.put(Config.NIMBUS_SUPERVISOR_USERS, "supervisor"); } - //Gets Nimbus Subject with NimbusPrincipal set on it + // Gets Nimbus Subject with NimbusPrincipal set on it public static Subject getNimbusSubject() { Subject nimbus = new Subject(); nimbus.getPrincipals().add(new NimbusPrincipal()); return nimbus; } - // Overloading the assertStoreHasExactly method accommodate Subject in order to check for authorization + // Overloading the assertStoreHasExactly method accommodate Subject in order to check for + // authorization public static void assertStoreHasExactly(BlobStore store, Subject who, String... keys) { Set expected = new HashSet<>(Arrays.asList(keys)); Set found = new HashSet<>(); @@ -121,8 +125,10 @@ public static void assertStoreHasExactly(BlobStore store, String... keys) { assertStoreHasExactly(store, null, keys); } - // Overloading the readInt method accommodate Subject in order to check for authorization (security turned on) - public static int readInt(BlobStore store, Subject who, String key) throws IOException, KeyNotFoundException, AuthorizationException { + // Overloading the readInt method accommodate Subject in order to check for authorization + // (security turned on) + public static int readInt(BlobStore store, Subject who, + String key) throws IOException, KeyNotFoundException, AuthorizationException { try (InputStream in = store.getBlob(key, who)) { return in.read(); } @@ -147,10 +153,12 @@ public void readAssertEqualsWithAuth(BlobStore store, Subject who, String key, i private AutoCloseableBlobStoreContainer initHdfs(String dirName) { Map conf = new HashMap<>(); conf.put(Config.BLOBSTORE_DIR, dirName); - conf.put(Config.STORM_PRINCIPAL_TO_LOCAL_PLUGIN, "org.apache.storm.security.auth.DefaultPrincipalToLocal"); + conf.put(Config.STORM_PRINCIPAL_TO_LOCAL_PLUGIN, + "org.apache.storm.security.auth.DefaultPrincipalToLocal"); conf.put(Config.STORM_BLOBSTORE_REPLICATION_FACTOR, 3); HdfsBlobStore store = new HdfsBlobStore(); - store.prepareInternal(conf, null, DFS_CLUSTER_EXTENSION.getDfscluster().getConfiguration(0)); + store.prepareInternal(conf, null, DFS_CLUSTER_EXTENSION.getDfscluster() + .getConfiguration(0)); return new AutoCloseableBlobStoreContainer(store); } @@ -203,10 +211,11 @@ public void testReplication(String path, BlobStore store) out.write(1); } assertStoreHasExactly(store, "test"); - assertEquals(store.getBlobReplication("test", null), 4, "Blobstore replication not matching"); + assertEquals(store.getBlobReplication("test", null), 4, + "Blobstore replication not matching"); store.deleteBlob("test", null); - //Test for replication with NIMBUS as user + // Test for replication with NIMBUS as user Subject admin = getSubject("admin"); metadata = new SettableBlobMeta(BlobStoreAclHandler.DEFAULT); metadata.set_replication_factor(4); @@ -214,12 +223,14 @@ public void testReplication(String path, BlobStore store) out.write(1); } assertStoreHasExactly(store, "test"); - assertEquals(store.getBlobReplication("test", admin), 4, "Blobstore replication not matching"); + assertEquals(store.getBlobReplication("test", admin), 4, + "Blobstore replication not matching"); store.updateBlobReplication("test", 5, admin); - assertEquals(store.getBlobReplication("test", admin), 5, "Blobstore replication not matching"); + assertEquals(store.getBlobReplication("test", admin), 5, + "Blobstore replication not matching"); store.deleteBlob("test", admin); - //Test for replication using SUPERVISOR access + // Test for replication using SUPERVISOR access Subject supervisor = getSubject("supervisor"); metadata = new SettableBlobMeta(BlobStoreAclHandler.DEFAULT); metadata.set_replication_factor(4); @@ -227,9 +238,11 @@ public void testReplication(String path, BlobStore store) out.write(1); } assertStoreHasExactly(store, "test"); - assertEquals(store.getBlobReplication("test", supervisor), 4, "Blobstore replication not matching"); + assertEquals(store.getBlobReplication("test", supervisor), 4, + "Blobstore replication not matching"); store.updateBlobReplication("test", 5, supervisor); - assertEquals(store.getBlobReplication("test", supervisor), 5, "Blobstore replication not matching"); + assertEquals(store.getBlobReplication("test", supervisor), 5, + "Blobstore replication not matching"); store.deleteBlob("test", supervisor); Subject adminsGroupsUser = getSubject("adminsGroupsUser"); @@ -239,12 +252,14 @@ public void testReplication(String path, BlobStore store) out.write(1); } assertStoreHasExactly(store, "test"); - assertEquals(store.getBlobReplication("test", adminsGroupsUser), 4, "Blobstore replication not matching"); + assertEquals(store.getBlobReplication("test", adminsGroupsUser), 4, + "Blobstore replication not matching"); store.updateBlobReplication("test", 5, adminsGroupsUser); - assertEquals(store.getBlobReplication("test", adminsGroupsUser), 5, "Blobstore replication not matching"); + assertEquals(store.getBlobReplication("test", adminsGroupsUser), 5, + "Blobstore replication not matching"); store.deleteBlob("test", adminsGroupsUser); - //Test for a user having read or write or admin access to read replication for a blob + // Test for a user having read or write or admin access to read replication for a blob String createSubject = "createSubject"; String writeSubject = "writeSubject"; String adminSubject = "adminSubject"; @@ -261,12 +276,14 @@ public void testReplication(String path, BlobStore store) } assertStoreHasExactly(store, "test"); who = getSubject(writeSubject); - assertEquals(store.getBlobReplication("test", who), 4, "Blobstore replication not matching"); + assertEquals(store.getBlobReplication("test", who), 4, + "Blobstore replication not matching"); - //Test for a user having WRITE or ADMIN privileges to change replication of a blob + // Test for a user having WRITE or ADMIN privileges to change replication of a blob who = getSubject(adminSubject); store.updateBlobReplication("test", 5, who); - assertEquals(store.getBlobReplication("test", who), 5, "Blobstore replication not matching"); + assertEquals(store.getBlobReplication("test", who), 5, + "Blobstore replication not matching"); store.deleteBlob("test", getSubject(createSubject)); } @@ -278,13 +295,13 @@ public static Subject getSubject(String name) { } enum AuthenticationTestSubject { - //Nimbus Admin + // Nimbus Admin ADMIN(getSubject("admin")), - //Nimbus groups admin + // Nimbus groups admin ADMIN_GROUPS_USER(getSubject("adminGroupsUser")), - //Supervisor admin + // Supervisor admin SUPERVISOR(getSubject("supervisor")), - //Nimbus itself + // Nimbus itself NIMBUS(getNimbusSubject()); private final Subject subject; @@ -297,7 +314,8 @@ enum AuthenticationTestSubject { @ParameterizedTest @EnumSource(value = AuthenticationTestSubject.class) void testWithAuthentication(AuthenticationTestSubject testSubject) throws Exception { - try (AutoCloseableBlobStoreContainer container = initHdfs("/storm/blobstore-auth-" + testSubject.name())) { + try (AutoCloseableBlobStoreContainer container = initHdfs("/storm/blobstore-auth-" + + testSubject.name())) { BlobStore store = container.blobStore; assertStoreHasExactly(store); SettableBlobMeta metadata = new SettableBlobMeta(BlobStoreAclHandler.DEFAULT); @@ -312,26 +330,32 @@ void testWithAuthentication(AuthenticationTestSubject testSubject) throws Except @ParameterizedTest @ValueSource(booleans = {true, false}) void testWithAuthenticationDummy(boolean securityEnabled) throws Exception { - try (AutoCloseableBlobStoreContainer container = initHdfs("/storm/blobstore-auth-dummy-sec-" + securityEnabled)) { + try (AutoCloseableBlobStoreContainer container = + initHdfs("/storm/blobstore-auth-dummy-sec-" + securityEnabled)) { BlobStore store = container.blobStore; Subject who = getSubject("test_subject"); assertStoreHasExactly(store); // Tests for case when subject != null (security turned on) and // acls for the blob are set to WORLD_EVERYTHING - SettableBlobMeta metadata = new SettableBlobMeta(securityEnabled ? BlobStoreAclHandler.DEFAULT : BlobStoreAclHandler.WORLD_EVERYTHING); + SettableBlobMeta metadata = new SettableBlobMeta(securityEnabled + ? BlobStoreAclHandler.DEFAULT : BlobStoreAclHandler.WORLD_EVERYTHING); try (AtomicOutputStream out = store.createBlob("test", metadata, who)) { out.write(1); } assertStoreHasExactly(store, "test"); if (securityEnabled) { - // Testing whether acls are set to WORLD_EVERYTHING. Here the acl should not contain WORLD_EVERYTHING because - // the subject is neither null nor empty. The ACL should however contain USER_EVERYTHING as user needs to have + // Testing whether acls are set to WORLD_EVERYTHING. Here the acl should not contain + // WORLD_EVERYTHING because + // the subject is neither null nor empty. The ACL should however contain + // USER_EVERYTHING as user needs to have // complete access to the blob - assertFalse(metadata.toString().contains("AccessControl(type:OTHER, access:7)"), "ACL contains WORLD_EVERYTHING"); + assertFalse(metadata.toString().contains("AccessControl(type:OTHER, access:7)"), + "ACL contains WORLD_EVERYTHING"); } else { // Testing whether acls are set to WORLD_EVERYTHING - assertTrue(metadata.toString().contains("AccessControl(type:OTHER, access:7)"), "ACL does not contain WORLD_EVERYTHING"); + assertTrue(metadata.toString().contains("AccessControl(type:OTHER, access:7)"), + "ACL does not contain WORLD_EVERYTHING"); } readAssertEqualsWithAuth(store, who, "test", 1); @@ -377,21 +401,24 @@ void testWithAuthenticationUpdate() throws Exception { @ParameterizedTest @ValueSource(booleans = {true, false}) void testWithAuthenticationNoPrincipal(boolean securityEnabled) throws Exception { - try (AutoCloseableBlobStoreContainer container = initHdfs("/storm/blobstore-auth-no-principal-sec-" + securityEnabled)) { + try (AutoCloseableBlobStoreContainer container = + initHdfs("/storm/blobstore-auth-no-principal-sec-" + securityEnabled)) { BlobStore store = container.blobStore; - //Test for subject with no principals + // Test for subject with no principals Subject who = new Subject(); assertStoreHasExactly(store); // Tests for case when subject != null (security turned on) and // acls for the blob are set to WORLD_EVERYTHING - SettableBlobMeta metadata = new SettableBlobMeta(securityEnabled ? BlobStoreAclHandler.DEFAULT : BlobStoreAclHandler.WORLD_EVERYTHING); + SettableBlobMeta metadata = new SettableBlobMeta(securityEnabled + ? BlobStoreAclHandler.DEFAULT : BlobStoreAclHandler.WORLD_EVERYTHING); try (AtomicOutputStream out = store.createBlob("test", metadata, who)) { out.write(1); } assertStoreHasExactly(store, "test"); // With no principals in the subject ACL should always be set to WORLD_EVERYTHING - assertTrue(metadata.toString().contains("AccessControl(type:OTHER, access:7)"), "ACL does not contain WORLD_EVERYTHING"); + assertTrue(metadata.toString().contains("AccessControl(type:OTHER, access:7)"), + "ACL does not contain WORLD_EVERYTHING"); readAssertEqualsWithAuth(store, who, "test", 1); } @@ -409,7 +436,8 @@ public void testBasic(BlobStore store) } assertStoreHasExactly(store, "test"); // Testing whether acls are set to WORLD_EVERYTHING - assertTrue(metadata.toString().contains("AccessControl(type:OTHER, access:7)"), "ACL does not contain WORLD_EVERYTHING"); + assertTrue(metadata.toString().contains("AccessControl(type:OTHER, access:7)"), + "ACL does not contain WORLD_EVERYTHING"); readAssertEquals(store, "test", 1); LOG.info("Deleting test"); @@ -450,14 +478,16 @@ public void testMultiple(BlobStore store) throws Exception { assertStoreHasExactly(store); LOG.info("Creating test"); - try (AtomicOutputStream out = store.createBlob("test", new SettableBlobMeta(BlobStoreAclHandler.WORLD_EVERYTHING), null)) { + try (AtomicOutputStream out = store.createBlob("test", + new SettableBlobMeta(BlobStoreAclHandler.WORLD_EVERYTHING), null)) { out.write(1); } assertStoreHasExactly(store, "test"); readAssertEquals(store, "test", 1); LOG.info("Creating other"); - try (AtomicOutputStream out = store.createBlob("other", new SettableBlobMeta(BlobStoreAclHandler.WORLD_EVERYTHING), + try (AtomicOutputStream out = store.createBlob("other", + new SettableBlobMeta(BlobStoreAclHandler.WORLD_EVERYTHING), null)) { out.write(2); } @@ -479,7 +509,8 @@ public void testMultiple(BlobStore store) readAssertEquals(store, "other", 5); LOG.info("Creating test again"); - try (AtomicOutputStream out = store.createBlob("test", new SettableBlobMeta(BlobStoreAclHandler.WORLD_EVERYTHING), + try (AtomicOutputStream out = store.createBlob("test", + new SettableBlobMeta(BlobStoreAclHandler.WORLD_EVERYTHING), null)) { out.write(2); } diff --git a/external/storm-hdfs-blobstore/src/test/java/org/apache/storm/hdfs/blobstore/HdfsBlobStoreImplTest.java b/external/storm-hdfs-blobstore/src/test/java/org/apache/storm/hdfs/blobstore/HdfsBlobStoreImplTest.java index f596e4591df..b75323d13b0 100644 --- a/external/storm-hdfs-blobstore/src/test/java/org/apache/storm/hdfs/blobstore/HdfsBlobStoreImplTest.java +++ b/external/storm-hdfs-blobstore/src/test/java/org/apache/storm/hdfs/blobstore/HdfsBlobStoreImplTest.java @@ -8,9 +8,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -18,6 +18,20 @@ */ package org.apache.storm.hdfs.blobstore; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@ -30,22 +44,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; - -import static org.junit.jupiter.api.Assertions.*; - public class HdfsBlobStoreImplTest { @RegisterExtension - public static final MiniDFSClusterExtensionClassLevel DFS_CLUSTER_EXTENSION = new MiniDFSClusterExtensionClassLevel(); + public static final MiniDFSClusterExtensionClassLevel DFS_CLUSTER_EXTENSION = + new MiniDFSClusterExtensionClassLevel(); private static final Logger LOG = LoggerFactory.getLogger(HdfsBlobStoreImplTest.class); public static final String CONCURRENT_TEST_KEY_PREFIX = "concurrent-test-key"; @@ -62,6 +65,7 @@ public class HdfsBlobStoreImplTest { public class TestHdfsBlobStoreImpl extends HdfsBlobStoreImpl implements AutoCloseable { Path basePath; + public TestHdfsBlobStoreImpl(Path path, Map conf) throws IOException { super(path, conf); basePath = path; @@ -90,14 +94,16 @@ public void testMultiple() throws Exception { String testString = "testingblob"; String validKey = "validkeyBasic"; - //Will be closed automatically when shutting down the DFS cluster + // Will be closed automatically when shutting down the DFS cluster FileSystem fs = DFS_CLUSTER_EXTENSION.getDfscluster().getFileSystem(); Map conf = new HashMap<>(); - try (TestHdfsBlobStoreImpl hbs = new TestHdfsBlobStoreImpl(blobDir, conf, DFS_CLUSTER_EXTENSION.getHadoopConf())) { + try (TestHdfsBlobStoreImpl hbs = new TestHdfsBlobStoreImpl(blobDir, conf, + DFS_CLUSTER_EXTENSION.getHadoopConf())) { // should have created blobDir assertTrue(fs.exists(blobDir), "BlobStore dir wasn't created"); - assertEquals(HdfsBlobStoreImpl.BLOBSTORE_DIR_PERMISSION, fs.getFileStatus(blobDir).getPermission(), + assertEquals(HdfsBlobStoreImpl.BLOBSTORE_DIR_PERMISSION, fs.getFileStatus(blobDir) + .getPermission(), "BlobStore dir was created with wrong permissions"); // test exist with non-existent key @@ -126,7 +132,8 @@ public void testMultiple() throws Exception { pfile.commit(); Path dataFile = new Path(new Path(fullKeyDir, validKey), BLOBSTORE_DATA); assertTrue(fs.exists(dataFile), "blob data not committed"); - assertEquals(HdfsBlobStoreFile.BLOBSTORE_FILE_PERMISSION, fs.getFileStatus(dataFile).getPermission(), + assertEquals(HdfsBlobStoreFile.BLOBSTORE_FILE_PERMISSION, fs.getFileStatus(dataFile) + .getPermission(), "BlobStore dir was created with wrong permissions"); assertTrue(hbs.exists(validKey), "key doesn't exist but should"); @@ -162,7 +169,8 @@ public void testMultiple() throws Exception { assertTrue(fs.exists(fullKeyDir), "BlobStore key dir wasn't created"); pfile.commit(); assertTrue(fs.exists(dataFile), "blob data not committed"); - assertEquals(HdfsBlobStoreFile.BLOBSTORE_FILE_PERMISSION, fs.getFileStatus(dataFile).getPermission(), + assertEquals(HdfsBlobStoreFile.BLOBSTORE_FILE_PERMISSION, fs.getFileStatus(dataFile) + .getPermission(), "BlobStore dir was created with wrong permissions"); assertTrue(hbs.exists(validKey), "key doesn't exist but should"); @@ -177,7 +185,8 @@ public void testMultiple() throws Exception { pfile.commit(); Path dataFile2 = new Path(new Path(fullKeyDir, validKey2), BLOBSTORE_DATA); assertTrue(fs.exists(dataFile2), "blob data not committed"); - assertEquals(HdfsBlobStoreFile.BLOBSTORE_FILE_PERMISSION, fs.getFileStatus(dataFile2).getPermission(), + assertEquals(HdfsBlobStoreFile.BLOBSTORE_FILE_PERMISSION, fs.getFileStatus(dataFile2) + .getPermission(), "BlobStore dir was created with wrong permissions"); assertTrue(hbs.exists(validKey2), "key doesn't exist but should"); @@ -227,7 +236,8 @@ public void testGetFileLength() throws Exception { Map conf = new HashMap<>(); String validKey = "validkeyBasic"; String testString = "testingblob"; - try (TestHdfsBlobStoreImpl hbs = new TestHdfsBlobStoreImpl(blobDir, conf, DFS_CLUSTER_EXTENSION.getHadoopConf())) { + try (TestHdfsBlobStoreImpl hbs = new TestHdfsBlobStoreImpl(blobDir, conf, + DFS_CLUSTER_EXTENSION.getHadoopConf())) { BlobStoreFile pfile = hbs.write(validKey, false); // Adding metadata to avoid null pointer exception SettableBlobMeta meta = new SettableBlobMeta(); @@ -241,7 +251,8 @@ public void testGetFileLength() throws Exception { } /** - * Test by listing keys {@link HdfsBlobStoreImpl#listKeys()} in multiple concurrent threads and then ensure that + * Test by listing keys {@link HdfsBlobStoreImpl#listKeys()} in multiple concurrent threads and + * then ensure that * same keys are retrived in all the threads without any exceptions. */ @Test @@ -273,9 +284,10 @@ public void run() { } Map conf = new HashMap<>(); - try (TestHdfsBlobStoreImpl hbs = new TestHdfsBlobStoreImpl(concurrentTestBlobDir, conf, DFS_CLUSTER_EXTENSION.getHadoopConf())) { + try (TestHdfsBlobStoreImpl hbs = new TestHdfsBlobStoreImpl(concurrentTestBlobDir, conf, + DFS_CLUSTER_EXTENSION.getHadoopConf())) { // test write again - for (int i = 0 ; i < keyCount ; i++) { + for (int i = 0; i < keyCount; i++) { String key = CONCURRENT_TEST_KEY_PREFIX + i; String val = "This is string " + i; BlobStoreFile pfile = hbs.write(key, false); @@ -290,27 +302,28 @@ public void run() { ConcurrentListerRunnable[] runnables = new ConcurrentListerRunnable[concurrency]; Thread[] threads = new Thread[concurrency]; - for (int i = 0 ; i < concurrency ; i++) { + for (int i = 0; i < concurrency; i++) { runnables[i] = new ConcurrentListerRunnable(hbs, i); threads[i] = new Thread(runnables[i]); } - for (int i = 0 ; i < concurrency ; i++) { + for (int i = 0; i < concurrency; i++) { threads[i].start(); } - for (int i = 0 ; i < concurrency ; i++) { + for (int i = 0; i < concurrency; i++) { threads[i].join(); } List keys = runnables[0].keys; assertEquals(keyCount, keys.size(), "Number of keys (values=" + keys + ")"); - for (int i = 1 ; i < concurrency ; i++) { + for (int i = 1; i < concurrency; i++) { ConcurrentListerRunnable otherRunnable = runnables[i]; assertEquals(keys, otherRunnable.keys); } - for (int i = 0 ; i < keyCount ; i++) { + for (int i = 0; i < keyCount; i++) { String key = CONCURRENT_TEST_KEY_PREFIX + i; hbs.deleteKey(key); } - LOG.info("All %d threads have %d keys=[%s]\n", concurrency, keys.size(), String.join(",", keys)); + LOG.info("All %d threads have %d keys=[%s]\n", concurrency, keys.size(), String + .join(",", keys)); } } } diff --git a/external/storm-hdfs-blobstore/src/test/java/org/apache/storm/hdfs/testing/MiniDFSClusterExtension.java b/external/storm-hdfs-blobstore/src/test/java/org/apache/storm/hdfs/testing/MiniDFSClusterExtension.java index 8bf6b0b3c5b..4e2791ed4c5 100644 --- a/external/storm-hdfs-blobstore/src/test/java/org/apache/storm/hdfs/testing/MiniDFSClusterExtension.java +++ b/external/storm-hdfs-blobstore/src/test/java/org/apache/storm/hdfs/testing/MiniDFSClusterExtension.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -17,19 +17,18 @@ */ package org.apache.storm.hdfs.testing; +import static org.apache.hadoop.test.GenericTestUtils.DEFAULT_TEST_DATA_DIR; +import static org.apache.hadoop.test.GenericTestUtils.SYSPROP_TEST_DATA_DIR; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.File; +import java.util.function.Supplier; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.junit.jupiter.api.extension.AfterEachCallback; import org.junit.jupiter.api.extension.BeforeEachCallback; import org.junit.jupiter.api.extension.ExtensionContext; -import java.io.File; -import java.util.function.Supplier; - -import static org.apache.hadoop.test.GenericTestUtils.DEFAULT_TEST_DATA_DIR; -import static org.apache.hadoop.test.GenericTestUtils.SYSPROP_TEST_DATA_DIR; -import static org.junit.jupiter.api.Assertions.assertTrue; - public class MiniDFSClusterExtension implements BeforeEachCallback, AfterEachCallback { private static final String TEST_BUILD_DATA = "test.build.data"; @@ -74,6 +73,7 @@ public void afterEach(ExtensionContext arg0) throws Exception { * Get an uncreated directory for tests. * We use this method to get rid of getTestDir() in GenericTestUtils in Hadoop code * which uses assert from junit4. + * * @return the absolute directory for tests. Caller is expected to create it. */ public static File getTestDir(String subdir) { @@ -82,6 +82,7 @@ public static File getTestDir(String subdir) { /** * Get the (created) base directory for tests. + * * @return the absolute directory */ public static File getTestDir() { diff --git a/external/storm-hdfs-blobstore/src/test/java/org/apache/storm/hdfs/testing/MiniDFSClusterExtensionClassLevel.java b/external/storm-hdfs-blobstore/src/test/java/org/apache/storm/hdfs/testing/MiniDFSClusterExtensionClassLevel.java index 1fd13d930ce..b0c992e01c6 100644 --- a/external/storm-hdfs-blobstore/src/test/java/org/apache/storm/hdfs/testing/MiniDFSClusterExtensionClassLevel.java +++ b/external/storm-hdfs-blobstore/src/test/java/org/apache/storm/hdfs/testing/MiniDFSClusterExtensionClassLevel.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -17,17 +17,16 @@ */ package org.apache.storm.hdfs.testing; +import static org.apache.storm.hdfs.testing.MiniDFSClusterExtension.getTestDir; + +import java.io.File; +import java.util.function.Supplier; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.junit.jupiter.api.extension.AfterAllCallback; import org.junit.jupiter.api.extension.BeforeAllCallback; import org.junit.jupiter.api.extension.ExtensionContext; -import java.io.File; -import java.util.function.Supplier; - -import static org.apache.storm.hdfs.testing.MiniDFSClusterExtension.getTestDir; - public class MiniDFSClusterExtensionClassLevel implements BeforeAllCallback, AfterAllCallback { private static final String TEST_BUILD_DATA = "test.build.data"; diff --git a/external/storm-hdfs-oci/pom.xml b/external/storm-hdfs-oci/pom.xml index 73b3f0c95fc..e0d4990a92c 100644 --- a/external/storm-hdfs-oci/pom.xml +++ b/external/storm-hdfs-oci/pom.xml @@ -104,6 +104,16 @@ + + org.openrewrite.maven + rewrite-maven-plugin + + + + org.codehaus.gmavenplus + gmavenplus-plugin + + org.apache.maven.plugins maven-checkstyle-plugin diff --git a/external/storm-hdfs-oci/src/main/java/org/apache/storm/container/oci/HdfsManifestToResourcesPlugin.java b/external/storm-hdfs-oci/src/main/java/org/apache/storm/container/oci/HdfsManifestToResourcesPlugin.java index 0ad78032c22..87fb8304cbc 100644 --- a/external/storm-hdfs-oci/src/main/java/org/apache/storm/container/oci/HdfsManifestToResourcesPlugin.java +++ b/external/storm-hdfs-oci/src/main/java/org/apache/storm/container/oci/HdfsManifestToResourcesPlugin.java @@ -42,9 +42,11 @@ public class HdfsManifestToResourcesPlugin implements OciManifestToResourcesPlug private FileSystem fs; private LoadingCache statCache; - private static final String CONFIG_MEDIA_TYPE = "application/vnd.docker.container.image.v1+json"; + private static final String CONFIG_MEDIA_TYPE = + "application/vnd.docker.container.image.v1+json"; - private static final String LAYER_TAR_GZIP_MEDIA_TYPE = "application/vnd.docker.image.rootfs.diff.tar.gzip"; + private static final String LAYER_TAR_GZIP_MEDIA_TYPE = + "application/vnd.docker.image.rootfs.diff.tar.gzip"; private static final String SHA_256 = "sha256"; @@ -59,10 +61,11 @@ public class HdfsManifestToResourcesPlugin implements OciManifestToResourcesPlug @Override public void init(Map conf) throws IOException { - //login to hdfs + // login to hdfs HadoopLoginUtil.loginHadoop(conf); - String topLevelDir = ObjectReader.getString(conf.get(DaemonConfig.STORM_OCI_IMAGE_HDFS_TOPLEVEL_DIR)); + String topLevelDir = ObjectReader.getString(conf + .get(DaemonConfig.STORM_OCI_IMAGE_HDFS_TOPLEVEL_DIR)); this.layersDir = topLevelDir + "/layers/"; this.configDir = topLevelDir + "/config/"; @@ -108,7 +111,8 @@ public List getLayerResources(ImageManifest manifest) throws IOExce FileStatus stat = statCache.get(path); long timestamp = stat.getModificationTime(); - OciResource ociResource = new OciResource(path.toString(), fileName, size, timestamp, OciResource.OciResourceType.LAYER); + OciResource ociResource = new OciResource(path.toString(), fileName, size, + timestamp, OciResource.OciResourceType.LAYER); ociResources.add(ociResource); } catch (ExecutionException e) { throw new IOException(e); @@ -146,7 +150,8 @@ public OciResource getConfigResource(ImageManifest manifest) throws IOException try { FileStatus stat = statCache.get(path); long timestamp = stat.getModificationTime(); - ociResource = new OciResource(path.toString(), hash, size, timestamp, OciResource.OciResourceType.CONFIG); + ociResource = new OciResource(path.toString(), hash, size, timestamp, + OciResource.OciResourceType.CONFIG); } catch (ExecutionException e) { throw new IOException(e); } diff --git a/external/storm-hdfs-oci/src/main/java/org/apache/storm/container/oci/HdfsOciResourcesLocalizer.java b/external/storm-hdfs-oci/src/main/java/org/apache/storm/container/oci/HdfsOciResourcesLocalizer.java index ef749c25f7a..8f2bd4b2380 100644 --- a/external/storm-hdfs-oci/src/main/java/org/apache/storm/container/oci/HdfsOciResourcesLocalizer.java +++ b/external/storm-hdfs-oci/src/main/java/org/apache/storm/container/oci/HdfsOciResourcesLocalizer.java @@ -21,7 +21,6 @@ import java.io.File; import java.io.IOException; import java.util.Map; - import org.apache.commons.io.FileDeleteStrategy; import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; @@ -31,7 +30,6 @@ import org.apache.storm.utils.ConfigUtils; import org.apache.storm.utils.HadoopLoginUtil; import org.apache.storm.utils.ObjectReader; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -44,25 +42,29 @@ public class HdfsOciResourcesLocalizer implements OciResourcesLocalizerInterface /** * Initialization. + * * @param conf the storm conf. * @throws IOException on I/O exception */ @Override public void init(Map conf) throws IOException { - //login to hdfs + // login to hdfs HadoopLoginUtil.loginHadoop(conf); - String resourcesLocalDir = ObjectReader.getString(conf.get(DaemonConfig.STORM_OCI_RESOURCES_LOCAL_DIR), + String resourcesLocalDir = ObjectReader.getString(conf + .get(DaemonConfig.STORM_OCI_RESOURCES_LOCAL_DIR), ConfigUtils.supervisorLocalDir(conf) + "/oci-resources"); FileUtils.forceMkdir(new File(resourcesLocalDir)); this.layersLocalDir = resourcesLocalDir + "/layers/"; this.configLocalDir = resourcesLocalDir + "/config/"; - String topLevelDir = ObjectReader.getString(conf.get(DaemonConfig.STORM_OCI_IMAGE_HDFS_TOPLEVEL_DIR)); + String topLevelDir = ObjectReader.getString(conf + .get(DaemonConfig.STORM_OCI_IMAGE_HDFS_TOPLEVEL_DIR)); this.fs = new Path(topLevelDir).getFileSystem(new Configuration()); } /** * Download the resources from HDFS to local dir. + * * @param ociResource The oci resource to download * @return the destination of the oci resource * @throws IOException on I/O exception @@ -101,9 +103,10 @@ public synchronized String localize(OciResource ociResource) throws IOException LOG.info("Starting to copy {} from hdfs to {}", ociResource.getPath(), workingDst); copyFileLocallyWithRetry(ociResource, workingDst); - LOG.info("Successfully finished copying {} from hdfs to {}", ociResource.getPath(), workingDst); + LOG.info("Successfully finished copying {} from hdfs to {}", ociResource.getPath(), + workingDst); - //set to readable by anyone + // set to readable by anyone boolean setReadable = workingDst.setReadable(true, false); if (!setReadable) { throw new IOException("Couldn't set " + workingDst + " to be world-readable"); @@ -117,7 +120,8 @@ public synchronized String localize(OciResource ociResource) throws IOException return dst.toString(); } - private synchronized void copyFileLocallyWithRetry(OciResource ociResource, File dst) throws IOException { + private synchronized void copyFileLocallyWithRetry(OciResource ociResource, + File dst) throws IOException { IOException lastIoException = null; for (int retryCount = 0; retryCount < LOCALIZE_MAX_RETRY; retryCount++) { @@ -129,7 +133,8 @@ private synchronized void copyFileLocallyWithRetry(OciResource ociResource, File if (dst.exists()) { FileDeleteStrategy.FORCE.delete(dst); } - LOG.warn("{} occurred at attempt {}, deleted corrupt file {} if present", e.toString(), retryCount, dst); + LOG.warn("{} occurred at attempt {}, deleted corrupt file {} if present", e + .toString(), retryCount, dst); lastIoException = e; try { Thread.sleep(1500); @@ -140,7 +145,8 @@ private synchronized void copyFileLocallyWithRetry(OciResource ociResource, File } } if (lastIoException != null) { - LOG.error("Resource localization of {} to {} failed after {} retries", ociResource, dst, LOCALIZE_MAX_RETRY, lastIoException); + LOG.error("Resource localization of {} to {} failed after {} retries", ociResource, dst, + LOCALIZE_MAX_RETRY, lastIoException); throw lastIoException; } diff --git a/external/storm-hdfs-oci/src/main/java/org/apache/storm/container/oci/LocalOrHdfsImageTagToManifestPlugin.java b/external/storm-hdfs-oci/src/main/java/org/apache/storm/container/oci/LocalOrHdfsImageTagToManifestPlugin.java index c3244436531..928eeac5473 100644 --- a/external/storm-hdfs-oci/src/main/java/org/apache/storm/container/oci/LocalOrHdfsImageTagToManifestPlugin.java +++ b/external/storm-hdfs-oci/src/main/java/org/apache/storm/container/oci/LocalOrHdfsImageTagToManifestPlugin.java @@ -23,8 +23,8 @@ import java.io.File; import java.io.FileReader; import java.io.IOException; -import java.io.UncheckedIOException; import java.io.InputStreamReader; +import java.io.UncheckedIOException; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.Map; @@ -41,7 +41,8 @@ import org.slf4j.LoggerFactory; public class LocalOrHdfsImageTagToManifestPlugin implements OciImageTagToManifestPluginInterface { - private static final Logger LOG = LoggerFactory.getLogger(LocalOrHdfsImageTagToManifestPlugin.class); + private static final Logger LOG = LoggerFactory + .getLogger(LocalOrHdfsImageTagToManifestPlugin.class); private Map manifestCache; private ObjectMapper objMapper; @@ -56,7 +57,8 @@ public class LocalOrHdfsImageTagToManifestPlugin implements OciImageTagToManifes private int ociCacheRefreshIntervalSecs; private long lastRefreshTime; - private static final String LOCAL_OR_HDFS_IMAGE_TAG_TO_MANIFEST_PLUGIN_PREFIX = "storm.oci.local.or.hdfs.image.tag.to.manifest.plugin."; + private static final String LOCAL_OR_HDFS_IMAGE_TAG_TO_MANIFEST_PLUGIN_PREFIX = + "storm.oci.local.or.hdfs.image.tag.to.manifest.plugin."; /** * The HDFS location where the oci image-tag-to-hash file exists. @@ -79,7 +81,8 @@ public class LocalOrHdfsImageTagToManifestPlugin implements OciImageTagToManifes /** * The number of manifests to cache. */ - private static final String OCI_NUM_MANIFESTS_TO_CACHE = LOCAL_OR_HDFS_IMAGE_TAG_TO_MANIFEST_PLUGIN_PREFIX + "num.manifests.to.cache"; + private static final String OCI_NUM_MANIFESTS_TO_CACHE = + LOCAL_OR_HDFS_IMAGE_TAG_TO_MANIFEST_PLUGIN_PREFIX + "num.manifests.to.cache"; private static final int SHA256_HASH_LENGTH = 64; @@ -88,6 +91,7 @@ public class LocalOrHdfsImageTagToManifestPlugin implements OciImageTagToManifes /** * Check that a string can be used as an image hash, i.e. it consists of exactly * {@link #SHA256_HASH_LENGTH} alphanumeric characters. + * * @param hash the string to check * @return true if the string has the shape of an image hash */ @@ -99,7 +103,7 @@ private static boolean isValidHash(String hash) { public void init(Map conf) throws IOException { this.conf = conf; - //login to hdfs + // login to hdfs HadoopLoginUtil.loginHadoop(conf); localImageTagToHashFile = (String) conf.get(LOCAL_OCI_IMAGE_TAG_TO_HASH_FILE); @@ -113,7 +117,8 @@ public void init(Map conf) throws IOException { if (hdfsImageToHashFile == null && localImageTagToHashFile == null) { throw new IllegalArgumentException("No valid image-tag-to-hash files"); } - manifestDir = ObjectReader.getString(conf.get(DaemonConfig.STORM_OCI_IMAGE_HDFS_TOPLEVEL_DIR)) + "/manifests/"; + manifestDir = ObjectReader.getString(conf + .get(DaemonConfig.STORM_OCI_IMAGE_HDFS_TOPLEVEL_DIR)) + "/manifests/"; int numManifestsToCache = ObjectReader.getInt(conf.get(OCI_NUM_MANIFESTS_TO_CACHE), 10); this.objMapper = new ObjectMapper(); this.manifestCache = new LruCache(numManifestsToCache, 0.75f); @@ -123,7 +128,8 @@ public void init(Map conf) throws IOException { private boolean loadImageToHashFiles() throws IOException { boolean ret = false; try (BufferedReader localBr = getLocalImageToHashReader()) { - Map localImageToHash = readImageToHashFile(localBr, localImageTagToHashFile); + Map localImageToHash = readImageToHashFile(localBr, + localImageTagToHashFile); if (localImageToHash != null && !localImageToHash.equals(localImageToHashCache)) { localImageToHashCache = localImageToHash; LOG.info("Reloaded local image tag to hash cache"); @@ -173,7 +179,8 @@ private BufferedReader getHdfsImageToHashReader() throws IOException { Path imageToHash = new Path(hdfsImageToHashFile); FileSystem fs = imageToHash.getFileSystem(new Configuration()); if (!fs.exists(imageToHash)) { - String message = "Could not load hdfs image to hash file, " + hdfsImageToHashFile + " doesn't exist"; + String message = "Could not load hdfs image to hash file, " + hdfsImageToHashFile + + " doesn't exist"; LOG.error(message); throw new IOException(message); } @@ -199,7 +206,8 @@ private BufferedReader getHdfsImageToHashReader() throws IOException { * *

This will map both foo/bar:current and fizz/gig:latest to 123456789 */ - private static Map readImageToHashFile(BufferedReader br, String filePath) throws IOException { + private static Map readImageToHashFile(BufferedReader br, + String filePath) throws IOException { if (br == null) { return null; } @@ -236,7 +244,6 @@ private static Map readImageToHashFile(BufferedReader br, String return imageToHashCache; } - @Override public synchronized ImageManifest getManifestFromImageTag(String imageTag) throws IOException { String hash = getHashFromImageTag(imageTag); @@ -274,7 +281,7 @@ public synchronized String getHashFromImageTag(String imageTag) { LOG.debug("Refreshing local and hdfs image-tag-to-hash cache"); try { boolean loaded = loadImageToHashFiles(); - //If this is the first time trying to load the files and yet it failed + // If this is the first time trying to load the files and yet it failed if (!loaded && lastRefreshTime == 0) { throw new RuntimeException("Couldn't load any image-tag-to-hash-files"); } diff --git a/external/storm-hdfs-oci/src/test/java/org/apache/storm/container/oci/LocalOrHdfsImageTagToManifestPluginTest.java b/external/storm-hdfs-oci/src/test/java/org/apache/storm/container/oci/LocalOrHdfsImageTagToManifestPluginTest.java index 2cda2b06378..955e0956b20 100644 --- a/external/storm-hdfs-oci/src/test/java/org/apache/storm/container/oci/LocalOrHdfsImageTagToManifestPluginTest.java +++ b/external/storm-hdfs-oci/src/test/java/org/apache/storm/container/oci/LocalOrHdfsImageTagToManifestPluginTest.java @@ -34,9 +34,11 @@ public class LocalOrHdfsImageTagToManifestPluginTest { - private static final String KNOWN_HASH = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; - private static final String UNKNOWN_HASH = "fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210"; - //same length as a hash, but made of characters that would still escape the manifest directory + private static final String KNOWN_HASH = + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + private static final String UNKNOWN_HASH = + "fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210"; + // same length as a hash, but made of characters that would still escape the manifest directory private static final String HASH_LENGTH_PATH = "../../../user/foo/bar/" + "a".repeat(42); @TempDir @@ -44,10 +46,12 @@ public class LocalOrHdfsImageTagToManifestPluginTest { private LocalOrHdfsImageTagToManifestPlugin createPlugin() throws IOException { Path hashFile = tempDir.resolve("image-tag-to-hash"); - Files.write(hashFile, ("busybox:latest:" + KNOWN_HASH + "\n").getBytes(StandardCharsets.UTF_8)); + Files.write(hashFile, ("busybox:latest:" + KNOWN_HASH + "\n") + .getBytes(StandardCharsets.UTF_8)); Map conf = new HashMap<>(); - conf.put("storm.oci.local.or.hdfs.image.tag.to.manifest.plugin.local.hash.file", hashFile.toString()); + conf.put("storm.oci.local.or.hdfs.image.tag.to.manifest.plugin.local.hash.file", hashFile + .toString()); conf.put(DaemonConfig.STORM_OCI_IMAGE_HDFS_TOPLEVEL_DIR, "/storm/oci"); LocalOrHdfsImageTagToManifestPlugin plugin = new LocalOrHdfsImageTagToManifestPlugin(); @@ -68,18 +72,24 @@ public void testUnmappedImageTagIsUsedAsHashWhenItLooksLikeOne() throws Exceptio @Test public void testUnmappedImageTagThatIsNotAHashIsRejected() throws Exception { LocalOrHdfsImageTagToManifestPlugin plugin = createPlugin(); - assertThrows(UncheckedIOException.class, () -> plugin.getHashFromImageTag("../../../user/foo/bar")); - assertThrows(UncheckedIOException.class, () -> plugin.getHashFromImageTag("busybox:unknown")); + assertThrows(UncheckedIOException.class, () -> plugin + .getHashFromImageTag("../../../user/foo/bar")); + assertThrows(UncheckedIOException.class, () -> plugin + .getHashFromImageTag("busybox:unknown")); assertThrows(UncheckedIOException.class, () -> plugin.getHashFromImageTag("..")); assertThrows(UncheckedIOException.class, () -> plugin.getHashFromImageTag("/etc/passwd")); - assertThrows(UncheckedIOException.class, () -> plugin.getHashFromImageTag(HASH_LENGTH_PATH)); - assertThrows(UncheckedIOException.class, () -> plugin.getHashFromImageTag(UNKNOWN_HASH + "a")); + assertThrows(UncheckedIOException.class, () -> plugin + .getHashFromImageTag(HASH_LENGTH_PATH)); + assertThrows(UncheckedIOException.class, () -> plugin.getHashFromImageTag(UNKNOWN_HASH + + "a")); } @Test public void testGetManifestFromImageTagRejectsUnmappedNonHashTag() throws Exception { LocalOrHdfsImageTagToManifestPlugin plugin = createPlugin(); - assertThrows(UncheckedIOException.class, () -> plugin.getManifestFromImageTag("../../../user/foo/bar")); - assertThrows(UncheckedIOException.class, () -> plugin.getManifestFromImageTag(HASH_LENGTH_PATH)); + assertThrows(UncheckedIOException.class, () -> plugin + .getManifestFromImageTag("../../../user/foo/bar")); + assertThrows(UncheckedIOException.class, () -> plugin + .getManifestFromImageTag(HASH_LENGTH_PATH)); } } diff --git a/external/storm-hdfs/pom.xml b/external/storm-hdfs/pom.xml index ca1207cfb56..565ddd0e620 100644 --- a/external/storm-hdfs/pom.xml +++ b/external/storm-hdfs/pom.xml @@ -145,6 +145,16 @@ 1 + + org.openrewrite.maven + rewrite-maven-plugin + + + + org.codehaus.gmavenplus + gmavenplus-plugin + + org.apache.maven.plugins maven-checkstyle-plugin diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/AbstractAvroSerializer.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/AbstractAvroSerializer.java index 04cef636714..9b23b19915a 100644 --- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/AbstractAvroSerializer.java +++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/AbstractAvroSerializer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -26,15 +32,18 @@ import org.apache.avro.io.DecoderFactory; import org.apache.avro.io.EncoderFactory; -//Generously adapted from: -//https://github.com/kijiproject/kiji-express/blob/master/kiji-express/src/main/scala/org/kiji/express/flow/framework/serialization +// Generously adapted from: +// https://github.com/kijiproject/kiji-express/blob/master/kiji-express/src/main/scala/org/kiji/express/flow/framework/serialization // /AvroSerializer.scala -//Which has as an ASL2.0 license +// Which has as an ASL2.0 license /** - * This abstract class can be extended to implement concrete classes capable of (de)serializing generic avro objects - * across a Topology. The methods in the AvroSchemaRegistry interface specify how schemas can be mapped to unique - * identifiers and vice versa. Implementations based on pre-defining schemas or utilizing an external schema registry + * This abstract class can be extended to implement concrete classes capable of (de)serializing + * generic avro objects + * across a Topology. The methods in the AvroSchemaRegistry interface specify how schemas can be + * mapped to unique + * identifiers and vice versa. Implementations based on pre-defining schemas or utilizing an + * external schema registry * are provided. */ public abstract class AbstractAvroSerializer extends Serializer implements AvroSchemaRegistry { @@ -57,7 +66,8 @@ public void write(Kryo kryo, Output output, GenericContainer record) { } @Override - public GenericContainer read(Kryo kryo, Input input, Class someClass) { + public GenericContainer read(Kryo kryo, Input input, + Class someClass) { Schema theSchema = this.getSchema(input.readString()); GenericDatumReader reader = new GenericDatumReader<>(theSchema); Decoder decoder = DecoderFactory diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/AvroSchemaRegistry.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/AvroSchemaRegistry.java index cca5099df45..23a41f8e13e 100644 --- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/AvroSchemaRegistry.java +++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/AvroSchemaRegistry.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/AvroUtils.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/AvroUtils.java index 13798bb24f5..35f5aa01374 100644 --- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/AvroUtils.java +++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/AvroUtils.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -17,9 +23,12 @@ public class AvroUtils { /** - * A helper method to extract avro serialization configurations from the topology configuration and register - * specific kryo serializers as necessary. A default serializer will be provided if none is specified in the - * configuration. "avro.serializer" should specify the complete class name of the serializer, e.g. + * A helper method to extract avro serialization configurations from the topology configuration + * and register + * specific kryo serializers as necessary. A default serializer will be provided if none is + * specified in the + * configuration. "avro.serializer" should specify the complete class name of the serializer, + * e.g. * "org.apache.stgorm.hdfs.avro.GenericAvroSerializer" * * @param conf The topology configuration diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/FixedAvroSerializer.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/FixedAvroSerializer.java index 94607b37cb5..f5f5f423b82 100644 --- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/FixedAvroSerializer.java +++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/FixedAvroSerializer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -24,8 +30,10 @@ import org.apache.commons.codec.binary.Base64; /** - * A class to help (de)serialize a pre-defined set of Avro schemas. Schemas should be listed, one per line, in a file - * called "FixedAvroSerializer.config", which must be part of the Storm topology jar file. Any schemas intended to be + * A class to help (de)serialize a pre-defined set of Avro schemas. Schemas should be listed, one + * per line, in a file + * called "FixedAvroSerializer.config", which must be part of the Storm topology jar file. Any + * schemas intended to be * used with this class **MUST** be defined in that file. */ public class FixedAvroSerializer extends AbstractAvroSerializer { @@ -35,7 +43,8 @@ public class FixedAvroSerializer extends AbstractAvroSerializer { final Map schema2fingerprintMap = new HashMap<>(); public FixedAvroSerializer() throws IOException, NoSuchAlgorithmException { - InputStream in = this.getClass().getClassLoader().getResourceAsStream("FixedAvroSerializer.config"); + InputStream in = this.getClass().getClassLoader() + .getResourceAsStream("FixedAvroSerializer.config"); BufferedReader reader = new BufferedReader(new InputStreamReader(in)); String line; diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/GenericAvroSerializer.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/GenericAvroSerializer.java index 6bb0e26b411..a1ca4eadb0a 100644 --- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/GenericAvroSerializer.java +++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/GenericAvroSerializer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -15,7 +21,8 @@ import org.apache.avro.Schema; /** - * A default implementation of the AvroSerializer that will just pass literal schemas back and forth. This should + * A default implementation of the AvroSerializer that will just pass literal schemas back and + * forth. This should * only be used if no other serializer will fit a use case. */ public class GenericAvroSerializer extends AbstractAvroSerializer { diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/AbstractHdfsBolt.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/AbstractHdfsBolt.java index ee820663350..d0c6201769e 100644 --- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/AbstractHdfsBolt.java +++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/AbstractHdfsBolt.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -45,7 +51,7 @@ public abstract class AbstractHdfsBolt extends BaseRichBolt { private static final Logger LOG = LoggerFactory.getLogger(AbstractHdfsBolt.class); private static final Integer DEFAULT_RETRY_COUNT = 3; /** - * Half of the default Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS + * Half of the default Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS. */ private static final int DEFAULT_TICK_TUPLE_INTERVAL_SECS = 15; private static final Integer DEFAULT_MAX_OPEN_FILES = 50; @@ -89,7 +95,8 @@ protected void rotateOutputFile(Writer writer) throws IOException { * Marked as final to prevent override. Subclasses should implement the doPrepare() method. */ @Override - public final void prepare(Map conf, TopologyContext topologyContext, OutputCollector collector) { + public final void prepare(Map conf, TopologyContext topologyContext, + OutputCollector collector) { this.writeLock = new Object(); if (this.syncPolicy == null) { throw new IllegalStateException("SyncPolicy must be specified."); @@ -146,7 +153,7 @@ public final void execute(Tuple tuple) { this.offset = writer.write(tuple); tupleBatch.add(tuple); } catch (IOException e) { - //If the write failed, try to sync anything already written + // If the write failed, try to sync anything already written LOG.info("Tuple failed to write, forcing a flush of existing data."); this.collector.reportError(e); forceSync = true; @@ -158,13 +165,15 @@ public final void execute(Tuple tuple) { int attempts = 0; boolean success = false; IOException lastException = null; - // Make every attempt to sync the data we have. If it can't be done then kill the bolt with + // Make every attempt to sync the data we have. If it can't be done then kill the + // bolt with // a runtime exception. The filesystem is presumably in a very bad state. while (success == false && attempts < fileRetryCount) { attempts += 1; try { syncAllWriters(); - LOG.debug("Data synced to filesystem. Ack'ing [{}] tuples", tupleBatch.size()); + LOG.debug("Data synced to filesystem. Ack'ing [{}] tuples", tupleBatch + .size()); for (Tuple t : tupleBatch) { this.collector.ack(t); } @@ -172,7 +181,8 @@ public final void execute(Tuple tuple) { syncPolicy.reset(); success = true; } catch (IOException e) { - LOG.warn("Data could not be synced to filesystem on attempt [{}]", attempts); + LOG.warn("Data could not be synced to filesystem on attempt [{}]", + attempts); this.collector.reportError(e); lastException = e; } @@ -186,7 +196,8 @@ public final void execute(Tuple tuple) { } tupleBatch.clear(); - throw new RuntimeException("Sync failed [" + attempts + "] times.", lastException); + throw new RuntimeException("Sync failed [" + attempts + "] times.", + lastException); } } @@ -210,7 +221,8 @@ private Writer getOrCreateWriter(String writerKey, Tuple tuple) throws IOExcepti /** * A tuple must be mapped to a writer based on two factors. - * - bolt specific logic that must separate tuples into different files in the same directory (see the avro bolt + * - bolt specific logic that must separate tuples into different files in the same directory + * (see the avro bolt * for an example of this) * - the directory the tuple will be partioned into */ @@ -226,18 +238,23 @@ void doRotationAndRemoveWriter(String writerKey, Writer writer) { } catch (IOException e) { this.collector.reportError(e); LOG.error("File could not be rotated"); - //At this point there is nothing to do. In all likelihood any filesystem operations will fail. - //The next tuple will almost certainly fail to write and/or sync, which force a rotation. That - //will give rotateAndReset() a chance to work which includes creating a fresh file handle. + // At this point there is nothing to do. In all likelihood any filesystem operations + // will fail. + // The next tuple will almost certainly fail to write and/or sync, which force a + // rotation. That + // will give rotateAndReset() a chance to work which includes creating a fresh file + // handle. } finally { - //rotateOutputFile(writer) has closed the writer. It's safe to remove the writer from the map here. + // rotateOutputFile(writer) has closed the writer. It's safe to remove the writer from + // the map here. writers.remove(writerKey); } } @Override public Map getComponentConfiguration() { - return TupleUtils.putTickFrequencyIntoComponentConfig(super.getComponentConfiguration(), tickTupleInterval); + return TupleUtils.putTickFrequencyIntoComponentConfig(super.getComponentConfiguration(), + tickTupleInterval); } @Override @@ -262,7 +279,7 @@ private void doRotationAndRemoveAllWriters() { LOG.warn("IOException during scheduled file rotation.", e); } } - //above for-loop has closed all the writers. It's safe to clear the map here. + // above for-loop has closed all the writers. It's safe to clear the map here. writers.clear(); } } @@ -300,7 +317,8 @@ protected Path getBasePathForNextFile(Tuple tuple) { this.fileNameFormat.getName(rotation, System.currentTimeMillis())); } - protected abstract void doPrepare(Map conf, TopologyContext topologyContext, OutputCollector collector) throws + protected abstract void doPrepare(Map conf, TopologyContext topologyContext, + OutputCollector collector) throws IOException; protected abstract String getWriterKey(Tuple tuple); @@ -320,8 +338,8 @@ static class WritersMap extends LinkedHashMap { @Override protected boolean removeEldestEntry(Map.Entry eldest) { if (this.size() > this.maxWriters) { - //The writer must be closed before removed from the map. - //If it failed, we might lose some data. + // The writer must be closed before removed from the map. + // If it failed, we might lose some data. try { eldest.getValue().close(); } catch (IOException e) { diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/AvroGenericRecordBolt.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/AvroGenericRecordBolt.java index f00df3a744c..dadfc1bcf66 100644 --- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/AvroGenericRecordBolt.java +++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/AvroGenericRecordBolt.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -82,14 +88,17 @@ public AvroGenericRecordBolt withPartitioner(Partitioner partitioner) { } @Override - protected void doPrepare(Map conf, TopologyContext topologyContext, OutputCollector collector) throws IOException { + protected void doPrepare(Map conf, TopologyContext topologyContext, + OutputCollector collector) throws IOException { LOG.info("Preparing AvroGenericRecord Bolt..."); this.fs = FileSystem.get(URI.create(this.fsUrl), hdfsConfig); } /** - * AvroGenericRecordBolt must override this method because messages with different schemas cannot be written to the - * same file. By treating the complete schema as the "key" AbstractHdfsBolt will associate a different writer for + * AvroGenericRecordBolt must override this method because messages with different schemas + * cannot be written to the + * same file. By treating the complete schema as the "key" AbstractHdfsBolt will associate a + * different writer for * every distinct schema. */ @Override @@ -101,6 +110,7 @@ protected String getWriterKey(Tuple tuple) { @Override protected AbstractHDFSWriter makeNewWriter(Path path, Tuple tuple) throws IOException { Schema recordSchema = ((GenericRecord) tuple.getValue(0)).getSchema(); - return new AvroGenericRecordHDFSWriter(this.rotationPolicy, path, this.fs.create(path), recordSchema); + return new AvroGenericRecordHDFSWriter(this.rotationPolicy, path, this.fs.create(path), + recordSchema); } } diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/HdfsBolt.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/HdfsBolt.java index 6677c7b9f74..133460ae695 100644 --- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/HdfsBolt.java +++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/HdfsBolt.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -94,7 +100,8 @@ public HdfsBolt withMaxOpenFiles(int maxOpenFiles) { } @Override - public void doPrepare(Map conf, TopologyContext topologyContext, OutputCollector collector) throws IOException { + public void doPrepare(Map conf, TopologyContext topologyContext, + OutputCollector collector) throws IOException { LOG.info("Preparing HDFS Bolt..."); this.fs = FileSystem.get(URI.create(this.fsUrl), hdfsConfig); } diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/SequenceFileBolt.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/SequenceFileBolt.java index 991a23cb905..f90ed88d62d 100644 --- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/SequenceFileBolt.java +++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/SequenceFileBolt.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -112,7 +118,8 @@ public SequenceFileBolt withMaxOpenFiles(int maxOpenFiles) { } @Override - public void doPrepare(Map conf, TopologyContext topologyContext, OutputCollector collector) throws IOException { + public void doPrepare(Map conf, TopologyContext topologyContext, + OutputCollector collector) throws IOException { LOG.info("Preparing Sequence File Bolt..."); if (this.format == null) { throw new IllegalStateException("SequenceFormat must be specified."); @@ -134,7 +141,8 @@ protected AbstractHDFSWriter makeNewWriter(Path path, Tuple tuple) throws IOExce SequenceFile.Writer.file(path), SequenceFile.Writer.keyClass(this.format.keyClass()), SequenceFile.Writer.valueClass(this.format.valueClass()), - SequenceFile.Writer.compression(this.compressionType, this.codecFactory.getCodecByName(this.compressionCodec)) + SequenceFile.Writer.compression(this.compressionType, this.codecFactory + .getCodecByName(this.compressionCodec)) ); return new SequenceFileWriter(this.rotationPolicy, path, writer, this.format); diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/Writer.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/Writer.java index c6b3fb3d249..e58f964c805 100644 --- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/Writer.java +++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/Writer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

http://www.apache.org/licenses/LICENSE-2.0 + * + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/DefaultFileNameFormat.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/DefaultFileNameFormat.java index 157929c3579..0ba0c635fcb 100644 --- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/DefaultFileNameFormat.java +++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/DefaultFileNameFormat.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -64,7 +70,8 @@ public void prepare(Map conf, TopologyContext topologyContext) { @Override public String getName(long rotation, long timeStamp) { - return this.prefix + this.componentId + "-" + this.taskId + "-" + rotation + "-" + timeStamp + this.extension; + return this.prefix + this.componentId + "-" + this.taskId + "-" + rotation + "-" + + timeStamp + this.extension; } @Override diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/DelimitedRecordFormat.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/DelimitedRecordFormat.java index f8cdad92923..07e09d6d792 100644 --- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/DelimitedRecordFormat.java +++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/DelimitedRecordFormat.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/FileNameFormat.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/FileNameFormat.java index 70210a934b0..9e79d6633f9 100644 --- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/FileNameFormat.java +++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/FileNameFormat.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -26,6 +32,7 @@ public interface FileNameFormat extends Serializable { /** * Returns the filename the HdfsBolt will create. + * * @param rotation the current file rotation number (incremented on every rotation) * @param timeStamp current time in milliseconds when the rotation occurs */ diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/RecordFormat.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/RecordFormat.java index 5102f381863..b759808121a 100644 --- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/RecordFormat.java +++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/RecordFormat.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/SimpleFileNameFormat.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/SimpleFileNameFormat.java index d80aaa8cd9f..19d6a160889 100644 --- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/SimpleFileNameFormat.java +++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/SimpleFileNameFormat.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -66,7 +72,7 @@ public SimpleFileNameFormat withPath(String path) { } /** - * support parameters:
+ * Support parameters:
* $TIME - current time. use withTimeFormat to format.
* $NUM - rotation number
* $HOST - local host name
@@ -81,7 +87,7 @@ public SimpleFileNameFormat withName(String name) { } public SimpleFileNameFormat withTimeFormat(String timeFormat) { - //check format + // check format try { new SimpleDateFormat(timeFormat); } catch (Exception e) { diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/rotation/FileRotationPolicy.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/rotation/FileRotationPolicy.java index 13229dd125d..0bd76fa56b1 100644 --- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/rotation/FileRotationPolicy.java +++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/rotation/FileRotationPolicy.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -35,7 +41,6 @@ public interface FileRotationPolicy extends Serializable { */ boolean mark(Tuple tuple, long offset); - /** * Called after the HdfsBolt rotates a file. */ diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/rotation/FileSizeRotationPolicy.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/rotation/FileSizeRotationPolicy.java index eca10d67101..aa05344398b 100644 --- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/rotation/FileSizeRotationPolicy.java +++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/rotation/FileSizeRotationPolicy.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/rotation/NoRotationPolicy.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/rotation/NoRotationPolicy.java index f25be14da9f..048ad2bc5bd 100644 --- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/rotation/NoRotationPolicy.java +++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/rotation/NoRotationPolicy.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/sync/CountSyncPolicy.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/sync/CountSyncPolicy.java index a048dc3002e..f87eae1d219 100644 --- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/sync/CountSyncPolicy.java +++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/sync/CountSyncPolicy.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/sync/SyncPolicy.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/sync/SyncPolicy.java index 12c673f3499..067549131f9 100644 --- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/sync/SyncPolicy.java +++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/sync/SyncPolicy.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -30,7 +36,6 @@ public interface SyncPolicy extends Serializable { */ boolean mark(Tuple tuple, long offset); - /** * Called after the HdfsBolt performs a sync. * diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/AbstractHDFSWriter.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/AbstractHDFSWriter.java index caf6b49f925..d326e0277ac 100644 --- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/AbstractHDFSWriter.java +++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/AbstractHDFSWriter.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -27,7 +33,8 @@ public abstract class AbstractHDFSWriter implements Writer { protected boolean needsRotation; public AbstractHDFSWriter(FileRotationPolicy policy, Path path) { - //This must be defensively copied, because a bolt probably has only one rotation policy object + // This must be defensively copied, because a bolt probably has only one rotation policy + // object this.rotationPolicy = policy.copy(); this.filePath = path; } diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/AvroGenericRecordHDFSWriter.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/AvroGenericRecordHDFSWriter.java index 713aa587a8b..24b2f741ebe 100644 --- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/AvroGenericRecordHDFSWriter.java +++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/AvroGenericRecordHDFSWriter.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -36,7 +42,8 @@ public class AvroGenericRecordHDFSWriter extends AbstractHDFSWriter { private Schema schema; private DataFileWriter avroWriter; - public AvroGenericRecordHDFSWriter(FileRotationPolicy policy, Path path, FSDataOutputStream stream, Schema schema) throws IOException { + public AvroGenericRecordHDFSWriter(FileRotationPolicy policy, Path path, + FSDataOutputStream stream, Schema schema) throws IOException { super(policy, path); this.out = stream; this.schema = schema; @@ -60,7 +67,8 @@ protected void doSync() throws IOException { LOG.debug("Attempting to sync all data to filesystem"); if (this.out instanceof HdfsDataOutputStream) { - ((HdfsDataOutputStream) this.out).hsync(EnumSet.of(HdfsDataOutputStream.SyncFlag.UPDATE_LENGTH)); + ((HdfsDataOutputStream) this.out).hsync(EnumSet + .of(HdfsDataOutputStream.SyncFlag.UPDATE_LENGTH)); } else { this.out.hsync(); } diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/HDFSWriter.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/HDFSWriter.java index 8b3dcd1bf2d..db3aaa53b2a 100644 --- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/HDFSWriter.java +++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/HDFSWriter.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -31,7 +37,8 @@ public class HDFSWriter extends AbstractHDFSWriter { private FSDataOutputStream out; private RecordFormat format; - public HDFSWriter(FileRotationPolicy policy, Path path, FSDataOutputStream out, RecordFormat format) { + public HDFSWriter(FileRotationPolicy policy, Path path, FSDataOutputStream out, + RecordFormat format) { super(policy, path); this.out = out; this.format = format; @@ -48,7 +55,8 @@ protected void doWrite(Tuple tuple) throws IOException { protected void doSync() throws IOException { LOG.info("Attempting to sync all data to filesystem"); if (this.out instanceof HdfsDataOutputStream) { - ((HdfsDataOutputStream) this.out).hsync(EnumSet.of(HdfsDataOutputStream.SyncFlag.UPDATE_LENGTH)); + ((HdfsDataOutputStream) this.out).hsync(EnumSet + .of(HdfsDataOutputStream.SyncFlag.UPDATE_LENGTH)); } else { this.out.hsync(); } diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/HdfsUtils.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/HdfsUtils.java index 462087eebc4..f7000cdb6d0 100644 --- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/HdfsUtils.java +++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/HdfsUtils.java @@ -1,12 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -25,9 +30,11 @@ import org.apache.hadoop.ipc.RemoteException; public class HdfsUtils { - /** list files sorted by modification time that have not been modified since 'olderThan'. if + /** + * List files sorted by modification time that have not been modified since 'olderThan'. if * 'olderThan' is <= 0 then the filtering is disabled */ - public static ArrayList listFilesByModificationTime(FileSystem fs, Path directory, long olderThan) + public static ArrayList listFilesByModificationTime(FileSystem fs, Path directory, + long olderThan) throws IOException { ArrayList fstats = new ArrayList<>(); diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/ModifTimeComparator.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/ModifTimeComparator.java index 47ebdfe304e..d9f3420e276 100644 --- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/ModifTimeComparator.java +++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/ModifTimeComparator.java @@ -1,12 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -15,7 +20,6 @@ import java.util.Comparator; import org.apache.hadoop.fs.FileStatus; - public class ModifTimeComparator implements Comparator { @Override diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/NullPartitioner.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/NullPartitioner.java index 3137f48b099..c7785c1454b 100644 --- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/NullPartitioner.java +++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/NullPartitioner.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -15,7 +21,8 @@ import org.apache.storm.tuple.Tuple; /** - * The NullPartitioner partitions every tuple to the empty string. In otherwords, no partition sub directories will + * The NullPartitioner partitions every tuple to the empty string. In otherwords, no partition sub + * directories will * be added to the path. */ public class NullPartitioner implements Partitioner { diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/Partitioner.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/Partitioner.java index 92f674733b1..de83f2b9f14 100644 --- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/Partitioner.java +++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/Partitioner.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -18,8 +24,10 @@ public interface Partitioner extends Serializable { /** - * Return a relative path that the tuple should be written to. For example, if an HdfsBolt were configured to write - * to /common/output and a partitioner returned "/foo" then the bolt should open a file in "/common/output/foo" + * Return a relative path that the tuple should be written to. For example, if an HdfsBolt were + * configured to write + * to /common/output and a partitioner returned "/foo" then the bolt should open a file in + * "/common/output/foo" * *

A best practice is to use Path.SEPARATOR instead of a literal "/" * diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/SequenceFileWriter.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/SequenceFileWriter.java index d0507b84ee5..e09da85774e 100644 --- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/SequenceFileWriter.java +++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/SequenceFileWriter.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -28,7 +34,8 @@ public class SequenceFileWriter extends AbstractHDFSWriter { private SequenceFile.Writer writer; private SequenceFormat format; - public SequenceFileWriter(FileRotationPolicy policy, Path path, SequenceFile.Writer writer, SequenceFormat format) { + public SequenceFileWriter(FileRotationPolicy policy, Path path, SequenceFile.Writer writer, + SequenceFormat format) { super(policy, path); this.writer = writer; this.format = format; diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/AbstractFileReader.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/AbstractFileReader.java index 614d2409510..44425a8e4be 100644 --- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/AbstractFileReader.java +++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/AbstractFileReader.java @@ -1,12 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -15,7 +20,6 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; - abstract class AbstractFileReader implements FileReader { private final Path file; @@ -35,7 +39,6 @@ public Path getFilePath() { return file; } - @Override public boolean equals(Object o) { if (this == o) { diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/Configs.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/Configs.java index 9c54a19b6d8..07a194d4715 100644 --- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/Configs.java +++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/Configs.java @@ -1,12 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -25,6 +30,7 @@ public class Configs implements Validated { /** * Required - chose the file type being consumed. + * * @deprecated please use {@link HdfsSpout#setReaderType(String)} */ @Deprecated @@ -35,6 +41,7 @@ public class Configs implements Validated { public static final String SEQ = "seq"; /** * Required - HDFS name node. + * * @deprecated please use {@link HdfsSpout#setHdfsUri(String)} */ @Deprecated @@ -42,6 +49,7 @@ public class Configs implements Validated { public static final String HDFS_URI = "hdfsspout.hdfs"; /** * Required - dir from which to read files. + * * @deprecated please use {@link HdfsSpout#setSourceDir(String)} */ @Deprecated @@ -49,6 +57,7 @@ public class Configs implements Validated { public static final String SOURCE_DIR = "hdfsspout.source.dir"; /** * Required - completed files will be moved here. + * * @deprecated please use {@link HdfsSpout#setArchiveDir(String)} */ @Deprecated @@ -56,6 +65,7 @@ public class Configs implements Validated { public static final String ARCHIVE_DIR = "hdfsspout.archive.dir"; /** * Required - unparsable files will be moved here. + * * @deprecated please use {@link HdfsSpout#setBadFilesDir(String)} */ @Deprecated @@ -63,6 +73,7 @@ public class Configs implements Validated { public static final String BAD_DIR = "hdfsspout.badfiles.dir"; /** * Directory in which lock files will be created. + * * @deprecated please use {@link HdfsSpout#setLockDir(String)} */ @Deprecated @@ -70,6 +81,7 @@ public class Configs implements Validated { public static final String LOCK_DIR = "hdfsspout.lock.dir"; /** * Commit after N records. 0 disables this. + * * @deprecated please use {@link HdfsSpout#setCommitFrequencyCount(int)} */ @Deprecated @@ -78,6 +90,7 @@ public class Configs implements Validated { public static final String COMMIT_FREQ_COUNT = "hdfsspout.commit.count"; /** * Commit after N secs. cannot be disabled. + * * @deprecated please use {@link HdfsSpout#setCommitFrequencySec(int)} */ @Deprecated @@ -86,6 +99,7 @@ public class Configs implements Validated { public static final String COMMIT_FREQ_SEC = "hdfsspout.commit.sec"; /** * Max outstanding. + * * @deprecated please use {@link HdfsSpout#setMaxOutstanding(int)} */ @Deprecated @@ -94,6 +108,7 @@ public class Configs implements Validated { public static final String MAX_OUTSTANDING = "hdfsspout.max.outstanding"; /** * Lock timeout. + * * @deprecated please use {@link HdfsSpout#setLockTimeoutSec(int)} */ @Deprecated @@ -101,7 +116,8 @@ public class Configs implements Validated { @IsPositiveNumber public static final String LOCK_TIMEOUT = "hdfsspout.lock.timeout.sec"; /** - * If clocks on machines in the Storm cluster are in sync inactivity duration after which locks are considered + * If clocks on machines in the Storm cluster are in sync inactivity duration after which locks + * are considered * candidates for being reassigned to another spout. * * @deprecated please use {@link HdfsSpout#setClocksInSync(boolean)} @@ -111,6 +127,7 @@ public class Configs implements Validated { public static final String CLOCKS_INSYNC = "hdfsspout.clocks.insync"; /** * Ignore suffix. + * * @deprecated please use {@link HdfsSpout#setIgnoreSuffix(String)} */ @Deprecated diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/DirLock.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/DirLock.java index 488531a040d..d58f07fa5f4 100644 --- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/DirLock.java +++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/DirLock.java @@ -1,12 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -39,10 +44,12 @@ private DirLock(FileSystem fs, Path lockFile) throws IOException { this.lockFile = lockFile; } - /** Get a lock on file if not already locked. + /** + * Get a lock on file if not already locked. * * @param dir the dir on which to get a lock - * @return The lock object if it the lock was acquired. Returns null if the dir is already locked. + * @return The lock object if it the lock was acquired. Returns null if the dir is already + * locked. * @throws IOException if there were errors */ public static DirLock tryLock(FileSystem fs, Path dir) throws IOException { @@ -55,7 +62,8 @@ public static DirLock tryLock(FileSystem fs, Path dir) throws IOException { ostream.close(); return new DirLock(fs, lockFile); } else { - LOG.debug("Thread ({}) cannot lock dir {} as its already locked.", threadInfo(), dir); + LOG.debug("Thread ({}) cannot lock dir {} as its already locked.", threadInfo(), + dir); return null; } } catch (IOException e) { @@ -74,7 +82,7 @@ private static String threadInfo() { } /** - * if the lock on the directory is stale, take ownership. + * If the lock on the directory is stale, take ownership. */ public static DirLock takeOwnershipIfStale(FileSystem fs, Path dirToLock, int lockTimeoutSec) { Path dirLockFile = getDirLockFile(dirToLock); @@ -103,7 +111,8 @@ private static DirLock takeOwnership(FileSystem fs, Path dirLockFile) throws IOE } // delete and recreate lock file - if (fs.delete(dirLockFile, false)) { // returns false if somebody else already deleted it (to take ownership) + if (fs.delete(dirLockFile, + false)) { // returns false if somebody else already deleted it (to take ownership) FSDataOutputStream ostream = HdfsUtils.tryCreateFile(fs, dirLockFile); if (ostream != null) { ostream.close(); diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/FileLock.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/FileLock.java index 7ff5aaaa29c..3d78c995b5b 100644 --- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/FileLock.java +++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/FileLock.java @@ -1,12 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -28,8 +33,10 @@ import org.slf4j.LoggerFactory; /** - * Facility to synchronize access to HDFS files. Thread gains exclusive access to a file by acquiring - * a FileLock object. The lock itself is represented as file on HDFS. Relies on atomic file creation. + * Facility to synchronize access to HDFS files. Thread gains exclusive access to a file by + * acquiring + * a FileLock object. The lock itself is represented as file on HDFS. Relies on atomic file + * creation. * Owning thread must heartbeat periodically on the lock to prevent the lock from being deemed as * stale (i.e. lock whose owning thread have died). */ @@ -42,7 +49,8 @@ public class FileLock { private final FSDataOutputStream lockFileStream; private LogEntry lastEntry; - private FileLock(FileSystem fs, Path lockFile, FSDataOutputStream lockFileStream, String spoutId) + private FileLock(FileSystem fs, Path lockFile, FSDataOutputStream lockFileStream, + String spoutId) throws IOException { this.fs = fs; this.lockFile = lockFile; @@ -62,7 +70,7 @@ private FileLock(FileSystem fs, Path lockFile, String spoutId, LogEntry entry) } /** - * returns lock on file or null if file is already locked. throws if unexpected problem + * Returns lock on file or null if file is already locked. throws if unexpected problem */ public static FileLock tryLock(FileSystem fs, Path fileToLock, Path lockDirPath, String spoutId) throws IOException { @@ -71,10 +79,12 @@ public static FileLock tryLock(FileSystem fs, Path fileToLock, Path lockDirPath, try { FSDataOutputStream ostream = HdfsUtils.tryCreateFile(fs, lockFile); if (ostream != null) { - LOG.debug("Acquired lock on file {}. LockFile= {}, Spout = {}", fileToLock, lockFile, spoutId); + LOG.debug("Acquired lock on file {}. LockFile= {}, Spout = {}", fileToLock, + lockFile, spoutId); return new FileLock(fs, lockFile, ostream, spoutId); } else { - LOG.debug("Cannot lock file {} as its already locked. Spout = {}", fileToLock, spoutId); + LOG.debug("Cannot lock file {} as its already locked. Spout = {}", fileToLock, + spoutId); return null; } } catch (IOException e) { @@ -84,7 +94,7 @@ public static FileLock tryLock(FileSystem fs, Path fileToLock, Path lockDirPath, } /** - * checks if lockFile is older than 'olderThan' UTC time by examining the modification time + * Checks if lockFile is older than 'olderThan' UTC time by examining the modification time * on file and (if necessary) the timestamp in last log entry in the file. If its stale, then * returns the last log entry, else returns null. * @@ -95,7 +105,7 @@ public static LogEntry getLastEntryIfStale(FileSystem fs, Path lockFile, long ol throws IOException { long modifiedTime = fs.getFileStatus(lockFile).getModificationTime(); if (modifiedTime <= olderThan) { // look - //Impt: HDFS timestamp may not reflect recent appends, so we double check the + // Impt: HDFS timestamp may not reflect recent appends, so we double check the // timestamp in last line of file to see when the last update was made LogEntry lastEntry = getLastEntry(fs, lockFile); if (lastEntry == null) { @@ -116,7 +126,7 @@ public static LogEntry getLastEntryIfStale(FileSystem fs, Path lockFile, long ol } /** - * returns the last log entry. + * Returns the last log entry. */ public static LogEntry getLastEntry(FileSystem fs, Path lockFile) throws IOException { @@ -131,6 +141,7 @@ public static LogEntry getLastEntry(FileSystem fs, Path lockFile) /** * Takes ownership of the lock file if possible. + * * @param lastEntry last entry in the lock file. this param is an optimization. * we dont scan the lock file again to find its last entry here since * its already been done once by the logic used to check if the lock @@ -139,13 +150,15 @@ public static LogEntry getLastEntry(FileSystem fs, Path lockFile) * @return null if lock File is not recoverable * @throws IOException if unable to acquire */ - public static FileLock takeOwnership(FileSystem fs, Path lockFile, LogEntry lastEntry, String spoutId) + public static FileLock takeOwnership(FileSystem fs, Path lockFile, LogEntry lastEntry, + String spoutId) throws IOException { try { if (fs instanceof DistributedFileSystem) { if (!((DistributedFileSystem) fs).recoverLease(lockFile)) { LOG.warn( - "Unable to recover lease on lock file {} right now. Cannot transfer ownership. Will need to try later. Spout = {}", + "Unable to recover lease on lock file {} right now. Cannot transfer " + + "ownership. Will need to try later. Spout = {}", lockFile, spoutId); return null; } @@ -153,14 +166,18 @@ public static FileLock takeOwnership(FileSystem fs, Path lockFile, LogEntry last return new FileLock(fs, lockFile, spoutId, lastEntry); } catch (IOException e) { if (e instanceof RemoteException - && ((RemoteException) e).unwrapRemoteException() instanceof AlreadyBeingCreatedException) { + && ((RemoteException) e) + .unwrapRemoteException() instanceof AlreadyBeingCreatedException) { LOG.warn( - "Lock file " + lockFile + "is currently open. Cannot transfer ownership now. Will need to try later. Spout= " + "Lock file " + lockFile + + "is currently open. Cannot transfer ownership now. Will need to try " + + "later. Spout= " + spoutId, e); return null; } else { // unexpected error - LOG.warn("Cannot transfer ownership now for lock file " + lockFile + ". Will need to try later. Spout =" + spoutId, e); + LOG.warn("Cannot transfer ownership now for lock file " + lockFile + + ". Will need to try later. Spout =" + spoutId, e); throw e; } } @@ -172,12 +189,14 @@ public static FileLock takeOwnership(FileSystem fs, Path lockFile, LogEntry last * Impt: Assumes access to lockFilesDir has been externally synchronized such that * only one thread accessing the same thread */ - public static FileLock acquireOldestExpiredLock(FileSystem fs, Path lockFilesDir, int locktimeoutSec, String spoutId) + public static FileLock acquireOldestExpiredLock(FileSystem fs, Path lockFilesDir, + int locktimeoutSec, String spoutId) throws IOException { // list files long now = System.currentTimeMillis(); long olderThan = now - (locktimeoutSec * 1000); - Collection listing = HdfsUtils.listFilesByModificationTime(fs, lockFilesDir, olderThan); + Collection listing = HdfsUtils.listFilesByModificationTime(fs, lockFilesDir, + olderThan); // locate expired lock files (if any). Try to take ownership (oldest lock first) for (Path file : listing) { @@ -206,12 +225,14 @@ public static FileLock acquireOldestExpiredLock(FileSystem fs, Path lockFilesDir * * @return a Pair<lock file path, last entry in lock file> .. if expired lock file found */ - public static HdfsUtils.Pair locateOldestExpiredLock(FileSystem fs, Path lockFilesDir, int locktimeoutSec) + public static HdfsUtils.Pair locateOldestExpiredLock(FileSystem fs, + Path lockFilesDir, int locktimeoutSec) throws IOException { // list files long now = System.currentTimeMillis(); long olderThan = now - (locktimeoutSec * 1000); - Collection listing = HdfsUtils.listFilesByModificationTime(fs, lockFilesDir, olderThan); + Collection listing = HdfsUtils.listFilesByModificationTime(fs, lockFilesDir, + olderThan); // locate oldest expired lock file (if any) and take ownership for (Path file : listing) { @@ -250,6 +271,7 @@ private void logProgress(String fileOffset, boolean prefixNewLine) /** * Release lock by deleting file. + * * @throws IOException if lock file could not be deleted */ public void release() throws IOException { diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/FileOffset.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/FileOffset.java index a8b354a8dcf..665825297fd 100644 --- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/FileOffset.java +++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/FileOffset.java @@ -1,12 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -25,7 +30,7 @@ interface FileOffset extends Comparable, Cloneable { /** - * tests if rhs == currOffset+1. + * Tests if rhs == currOffset+1. */ boolean isNextOffset(FileOffset rhs); diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/FileReader.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/FileReader.java index b6e08f4523b..b019aedde5f 100644 --- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/FileReader.java +++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/FileReader.java @@ -1,12 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/HdfsSpout.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/HdfsSpout.java index c7a2bb16138..619835b34f9 100644 --- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/HdfsSpout.java +++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/HdfsSpout.java @@ -1,12 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -44,7 +49,8 @@ public class HdfsSpout extends BaseRichSpout { private static final Logger LOG = LoggerFactory.getLogger(HdfsSpout.class); private final AtomicBoolean commitTimeElapsed = new AtomicBoolean(false); HashMap> inflight = new HashMap<>(); - LinkedBlockingQueue>> retryList = new LinkedBlockingQueue<>(); + LinkedBlockingQueue>> retryList = + new LinkedBlockingQueue<>(); HdfsUtils.Pair lastExpiredLock = null; // user configurable private String hdfsUri; // required @@ -63,7 +69,8 @@ public class HdfsSpout extends BaseRichSpout { private int maxOutstanding = Configs.DEFAULT_MAX_OUTSTANDING; private int lockTimeoutSec = Configs.DEFAULT_LOCK_TIMEOUT; private boolean clocksInSync = true; - private String inprogressSuffix = ".inprogress"; // not configurable to prevent change between topology restarts + private String inprogressSuffix = + ".inprogress"; // not configurable to prevent change between topology restarts private String ignoreSuffix = ".ignore"; private String outputStreamName = null; private ProgressTracker tracker = null; @@ -94,10 +101,12 @@ private static void releaseLockAndLog(FileLock fileLock, String spoutId) { try { if (fileLock != null) { fileLock.release(); - LOG.debug("Spout {} released FileLock. SpoutId = {}", fileLock.getLockFile(), spoutId); + LOG.debug("Spout {} released FileLock. SpoutId = {}", fileLock.getLockFile(), + spoutId); } } catch (IOException e) { - LOG.error("Unable to delete lock file : " + fileLock.getLockFile() + " SpoutId =" + spoutId, e); + LOG.error("Unable to delete lock file : " + fileLock.getLockFile() + " SpoutId =" + + spoutId, e); } } @@ -106,15 +115,18 @@ private static void validateOrMakeDir(FileSystem fs, Path dir, String dirDescrip if (fs.exists(dir)) { if (!fs.isDirectory(dir)) { LOG.error(dirDescription + " directory is a file, not a dir. " + dir); - throw new RuntimeException(dirDescription + " directory is a file, not a dir. " + dir); + throw new RuntimeException(dirDescription + " directory is a file, not a dir. " + + dir); } } else if (!fs.mkdirs(dir)) { LOG.error("Unable to create " + dirDescription + " directory " + dir); - throw new RuntimeException("Unable to create " + dirDescription + " directory " + dir); + throw new RuntimeException("Unable to create " + dirDescription + " directory " + + dir); } } catch (IOException e) { LOG.error("Unable to create " + dirDescription + " directory " + dir, e); - throw new RuntimeException("Unable to create " + dirDescription + " directory " + dir, e); + throw new RuntimeException("Unable to create " + dirDescription + " directory " + dir, + e); } } @@ -135,7 +147,8 @@ static void checkValidReader(String readerType) { throw new IllegalArgumentException(readerType + " not found in classpath.", e); } catch (NoSuchMethodException e) { LOG.error(readerType + " is missing the expected constructor for Readers.", e); - throw new IllegalArgumentException(readerType + " is missing the expected constuctor for Readers."); + throw new IllegalArgumentException(readerType + + " is missing the expected constuctor for Readers."); } } @@ -208,7 +221,8 @@ public HdfsSpout withOutputFields(String... fields) { } /** - * set key name under which HDFS options are placed. (similar to HDFS bolt). default key name is 'hdfs.config' + * Set key name under which HDFS options are placed. (similar to HDFS bolt). default key name is + * 'hdfs.config' */ public HdfsSpout withConfigKey(String configKey) { this.configKey = configKey; @@ -273,7 +287,8 @@ public void nextTuple() { if (tuple != null) { fileReadCompletely = false; ++tupleCounter; - MessageId msgId = new MessageId(tupleCounter, reader.getFilePath(), reader.getFileOffset()); + MessageId msgId = new MessageId(tupleCounter, reader.getFilePath(), reader + .getFileOffset()); emitData(tuple, msgId); if (!ackEnabled) { @@ -295,7 +310,8 @@ public void nextTuple() { // don't emit anything .. allow configured spout wait strategy to kick in return; } catch (ParseException e) { - LOG.error("Parsing error when processing at file location " + getFileProgress(reader) + LOG.error("Parsing error when processing at file location " + + getFileProgress(reader) + ". Skipping remainder of file.", e); markFileAsBad(reader.getFilePath()); // Note: We don't return from this method on ParseException to avoid triggering the @@ -353,14 +369,18 @@ private void markFileAsBad(Path file) { String originalName = new Path(fileNameMinusSuffix).getName(); Path newFile = new Path(badFilesDirPath + Path.SEPARATOR + originalName); - LOG.info("Moving bad file {} to {}. Processed it till offset {}. SpoutID= {}", originalName, newFile, tracker.getCommitPosition(), + LOG.info("Moving bad file {} to {}. Processed it till offset {}. SpoutID= {}", originalName, + newFile, tracker.getCommitPosition(), spoutId); try { - if (!hdfs.rename(file, newFile)) { // seems this can fail by returning false or throwing exception - throw new IOException("Move failed for bad file: " + file); // convert false ret value to exception + if (!hdfs.rename(file, + newFile)) { // seems this can fail by returning false or throwing exception + throw new IOException("Move failed for bad file: " + + file); // convert false ret value to exception } } catch (IOException e) { - LOG.warn("Error moving bad file: " + file + " to destination " + newFile + " SpoutId =" + spoutId, e); + LOG.warn("Error moving bad file: " + file + " to destination " + newFile + " SpoutId =" + + spoutId, e); } closeReaderAndResetTrackers(); } @@ -390,7 +410,8 @@ protected void emitData(List tuple, MessageId id) { @SuppressWarnings("deprecation") @Override - public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { + public void open(Map conf, TopologyContext context, + SpoutOutputCollector collector) { LOG.info("Opening HDFS Spout"); this.conf = conf; this.commitTimer = new Timer(context.getThisTaskId() + "-commit-timer", true); @@ -418,7 +439,8 @@ public void open(Map conf, TopologyContext context, SpoutOutputC if (map != null) { for (String keyName : map.keySet()) { LOG.info("HDFS Config override : {} = {} ", keyName, - ConfigUtils.isCredentialKey(keyName) ? "*****" : String.valueOf(map.get(keyName))); + ConfigUtils.isCredentialKey(keyName) ? "*****" : String.valueOf(map + .get(keyName))); this.hdfsConfig.set(keyName, String.valueOf(map.get(keyName))); } try { @@ -510,7 +532,8 @@ public void open(Map conf, TopologyContext context, SpoutOutputC if (conf.get(Configs.COMMIT_FREQ_SEC) != null) { commitFrequencySec = Integer.parseInt(conf.get(Configs.COMMIT_FREQ_SEC).toString()); if (commitFrequencySec <= 0) { - throw new RuntimeException(Configs.COMMIT_FREQ_SEC + " setting must be greater than 0"); + throw new RuntimeException(Configs.COMMIT_FREQ_SEC + + " setting must be greater than 0"); } } @@ -571,7 +594,8 @@ public void fail(Object msgId) { LOG.trace("Fail received for msg id {} on spout {}", msgId, spoutId); super.fail(msgId); if (ackEnabled) { - HdfsUtils.Pair> item = HdfsUtils.Pair.of(msgId, inflight.remove(msgId)); + HdfsUtils.Pair> item = HdfsUtils.Pair.of(msgId, inflight + .remove(msgId)); retryList.add(item); } } @@ -581,15 +605,18 @@ private FileReader pickNextFile() { // 1) If there are any abandoned files, pick oldest one lock = getOldestExpiredLock(); if (lock != null) { - LOG.debug("Spout {} now took over ownership of abandoned FileLock {}", spoutId, lock.getLockFile()); + LOG.debug("Spout {} now took over ownership of abandoned FileLock {}", spoutId, lock + .getLockFile()); Path file = getFileForLockFile(lock.getLockFile(), sourceDirPath); String resumeFromOffset = lock.getLastLogEntry().fileOffset; LOG.info("Resuming processing of abandoned file : {}", file); return createFileReader(file, resumeFromOffset); } - // 2) If no abandoned files, then pick oldest file in sourceDirPath, lock it and rename it - Collection listing = HdfsUtils.listFilesByModificationTime(hdfs, sourceDirPath, 0); + // 2) If no abandoned files, then pick oldest file in sourceDirPath, lock it and rename + // it + Collection listing = HdfsUtils.listFilesByModificationTime(hdfs, sourceDirPath, + 0); for (Path file : listing) { if (file.getName().endsWith(inprogressSuffix)) { @@ -623,7 +650,8 @@ private FileReader pickNextFile() { } /** - * If clocks in sync, then acquires the oldest expired lock Else, on first call, just remembers the oldest expired lock, on next call + * If clocks in sync, then acquires the oldest expired lock Else, on first call, just remembers + * the oldest expired lock, on next call * check if the lock is updated. if not updated then acquires the lock * * @return a lock object @@ -634,10 +662,12 @@ private FileLock getOldestExpiredLock() throws IOException { if (dirlock == null) { dirlock = DirLock.takeOwnershipIfStale(hdfs, lockDirPath, lockTimeoutSec); if (dirlock == null) { - LOG.debug("Spout {} could not take over ownership of DirLock for {}", spoutId, lockDirPath); + LOG.debug("Spout {} could not take over ownership of DirLock for {}", spoutId, + lockDirPath); return null; } - LOG.debug("Spout {} now took over ownership of abandoned DirLock for {}", spoutId, lockDirPath); + LOG.debug("Spout {} now took over ownership of abandoned DirLock for {}", spoutId, + lockDirPath); } else { LOG.debug("Spout {} now owns DirLock for {}", spoutId, lockDirPath); } @@ -645,13 +675,16 @@ private FileLock getOldestExpiredLock() throws IOException { try { // 2 - if clocks are in sync then simply take ownership of the oldest expired lock if (clocksInSync) { - return FileLock.acquireOldestExpiredLock(hdfs, lockDirPath, lockTimeoutSec, spoutId); + return FileLock.acquireOldestExpiredLock(hdfs, lockDirPath, lockTimeoutSec, + spoutId); } // 3 - if clocks are not in sync .. if (lastExpiredLock == null) { - // just make a note of the oldest expired lock now and check if its still unmodified after lockTimeoutSec - lastExpiredLock = FileLock.locateOldestExpiredLock(hdfs, lockDirPath, lockTimeoutSec); + // just make a note of the oldest expired lock now and check if its still unmodified + // after lockTimeoutSec + lastExpiredLock = FileLock.locateOldestExpiredLock(hdfs, lockDirPath, + lockTimeoutSec); lastExpiredLockTime = System.currentTimeMillis(); return null; } @@ -663,7 +696,8 @@ private FileLock getOldestExpiredLock() throws IOException { // If lock file has expired, then own it FileLock.LogEntry lastEntry = FileLock.getLastEntry(hdfs, lastExpiredLock.getKey()); if (lastEntry.equals(lastExpiredLock.getValue())) { - FileLock result = FileLock.takeOwnership(hdfs, lastExpiredLock.getKey(), lastEntry, spoutId); + FileLock result = FileLock.takeOwnership(hdfs, lastExpiredLock.getKey(), lastEntry, + spoutId); lastExpiredLock = null; return result; } else { @@ -696,7 +730,8 @@ private FileReader createFileReader(Path file) } try { Class clsType = Class.forName(readerType); - Constructor constructor = clsType.getConstructor(FileSystem.class, Path.class, Map.class); + Constructor constructor = clsType.getConstructor(FileSystem.class, Path.class, + Map.class); return (FileReader) constructor.newInstance(this.hdfs, file, conf); } catch (Exception e) { LOG.error(e.getMessage(), e); @@ -721,7 +756,8 @@ private FileReader createFileReader(Path file, String offset) try { Class clsType = Class.forName(readerType); - Constructor constructor = clsType.getConstructor(FileSystem.class, Path.class, Map.class, String.class); + Constructor constructor = clsType.getConstructor(FileSystem.class, Path.class, + Map.class, String.class); return (FileReader) constructor.newInstance(this.hdfs, file, conf, offset); } catch (Exception e) { LOG.error(e.getMessage(), e); @@ -749,7 +785,8 @@ private Path renameToInProgressFile(Path file) } /** - * Returns the corresponding input file in the 'sourceDirPath' for the specified lock file. If no such file is found then returns null + * Returns the corresponding input file in the 'sourceDirPath' for the specified lock file. If + * no such file is found then returns null */ private Path getFileForLockFile(Path lockFile, Path sourceDirPath) throws IOException { diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/ParseException.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/ParseException.java index a7845ea24c2..6461e6b300a 100644 --- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/ParseException.java +++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/ParseException.java @@ -1,12 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/ProgressTracker.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/ProgressTracker.java index 93a9b093009..b185cd9277b 100644 --- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/ProgressTracker.java +++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/ProgressTracker.java @@ -1,12 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/SequenceFileReader.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/SequenceFileReader.java index cf9dc1767b6..6c2201de1ba 100644 --- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/SequenceFileReader.java +++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/SequenceFileReader.java @@ -1,12 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -35,16 +40,16 @@ public class SequenceFileReader private final SequenceFileReader.Offset offset; - private final KeyT key; private final ValueT value; - public SequenceFileReader(FileSystem fs, Path file, Map conf) throws IOException { super(fs, file); - int bufferSize = !conf.containsKey(BUFFER_SIZE) ? DEFAULT_BUFF_SIZE : Integer.parseInt(conf.get(BUFFER_SIZE).toString()); - this.reader = new SequenceFile.Reader(fs.getConf(), SequenceFile.Reader.file(file), SequenceFile.Reader.bufferSize(bufferSize)); + int bufferSize = !conf.containsKey(BUFFER_SIZE) ? DEFAULT_BUFF_SIZE : Integer.parseInt(conf + .get(BUFFER_SIZE).toString()); + this.reader = new SequenceFile.Reader(fs.getConf(), SequenceFile.Reader.file(file), + SequenceFile.Reader.bufferSize(bufferSize)); this.key = (KeyT) ReflectionUtils.newInstance(reader.getKeyClass(), fs.getConf()); this.value = (ValueT) ReflectionUtils.newInstance(reader.getValueClass(), fs.getConf()); this.offset = new SequenceFileReader.Offset(0, 0, 0); @@ -53,15 +58,18 @@ public SequenceFileReader(FileSystem fs, Path file, Map conf) public SequenceFileReader(FileSystem fs, Path file, Map conf, String offset) throws IOException { super(fs, file); - int bufferSize = !conf.containsKey(BUFFER_SIZE) ? DEFAULT_BUFF_SIZE : Integer.parseInt(conf.get(BUFFER_SIZE).toString()); + int bufferSize = !conf.containsKey(BUFFER_SIZE) ? DEFAULT_BUFF_SIZE : Integer.parseInt(conf + .get(BUFFER_SIZE).toString()); this.offset = new SequenceFileReader.Offset(offset); - this.reader = new SequenceFile.Reader(fs.getConf(), SequenceFile.Reader.file(file), SequenceFile.Reader.bufferSize(bufferSize)); + this.reader = new SequenceFile.Reader(fs.getConf(), SequenceFile.Reader.file(file), + SequenceFile.Reader.bufferSize(bufferSize)); this.key = (KeyT) ReflectionUtils.newInstance(reader.getKeyClass(), fs.getConf()); this.value = (ValueT) ReflectionUtils.newInstance(reader.getValueClass(), fs.getConf()); skipToOffset(this.reader, this.offset, this.key); } - private static void skipToOffset(SequenceFile.Reader reader, Offset offset, K key) throws IOException { + private static void skipToOffset(SequenceFile.Reader reader, Offset offset, + K key) throws IOException { reader.sync(offset.lastSyncPoint); for (int i = 0; i < offset.recordsSinceLastSync; ++i) { reader.next(key); @@ -93,7 +101,6 @@ public Offset getFileOffset() { return offset; } - public static class Offset implements FileOffset { public long lastSyncPoint; public long recordsSinceLastSync; @@ -138,7 +145,8 @@ public Offset(String offset) { } } catch (Exception e) { throw new IllegalArgumentException("'" + offset - + "' cannot be interpreted. It is not in expected format for SequenceFileReader." + + "' cannot be interpreted. It is not in expected format for " + + "SequenceFileReader." + " Format e.g. {sync=123:afterSync=345:record=67}"); } } @@ -206,8 +214,9 @@ void increment(boolean syncSeen, long newBytePosition) { @Override public Offset clone() { - return new Offset(lastSyncPoint, recordsSinceLastSync, currentRecord, currRecordEndOffset, prevRecordEndOffset); + return new Offset(lastSyncPoint, recordsSinceLastSync, currentRecord, + currRecordEndOffset, prevRecordEndOffset); } - } //class Offset -} //class + } // class Offset +} // class diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/TextFileReader.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/TextFileReader.java index 42924c91d18..1b96d3c2605 100644 --- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/TextFileReader.java +++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/TextFileReader.java @@ -1,12 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -39,19 +44,23 @@ public TextFileReader(FileSystem fs, Path file, Map conf) throws this(fs, file, conf, new TextFileReader.Offset(0, 0)); } - public TextFileReader(FileSystem fs, Path file, Map conf, String startOffset) throws IOException { + public TextFileReader(FileSystem fs, Path file, Map conf, + String startOffset) throws IOException { this(fs, file, conf, new TextFileReader.Offset(startOffset)); } - private TextFileReader(FileSystem fs, Path file, Map conf, TextFileReader.Offset startOffset) + private TextFileReader(FileSystem fs, Path file, Map conf, + TextFileReader.Offset startOffset) throws IOException { super(fs, file); offset = startOffset; FSDataInputStream in = fs.open(file); - String charSet = (conf == null || !conf.containsKey(CHARSET)) ? "UTF-8" : conf.get(CHARSET).toString(); + String charSet = (conf == null || !conf.containsKey(CHARSET)) ? "UTF-8" : conf.get(CHARSET) + .toString(); int buffSz = - (conf == null || !conf.containsKey(BUFFER_SIZE)) ? DEFAULT_BUFF_SIZE : Integer.parseInt(conf.get(BUFFER_SIZE).toString()); + (conf == null || !conf.containsKey(BUFFER_SIZE)) ? DEFAULT_BUFF_SIZE : Integer + .parseInt(conf.get(BUFFER_SIZE).toString()); reader = new BufferedReader(new InputStreamReader(in, charSet), buffSz); if (offset.charOffset > 0) { reader.skip(offset.charOffset); @@ -125,7 +134,8 @@ public Offset(String offset) { } } catch (Exception e) { throw new IllegalArgumentException("'" + offset - + "' cannot be interpreted. It is not in expected format for TextFileReader." + + "' cannot be interpreted. It is not in expected format for " + + "TextFileReader." + " Format e.g. {char=123:line=5}"); } } @@ -188,5 +198,5 @@ public int hashCode() { public Offset clone() { return new Offset(charOffset, lineNumber); } - } //class Offset + } // class Offset } diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/HdfsState.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/HdfsState.java index 118a113a204..a35ab4b1eda 100644 --- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/HdfsState.java +++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/HdfsState.java @@ -62,12 +62,12 @@ public class HdfsState implements State { private volatile TxnRecord lastSeenTxn; private Path indexFilePath; - HdfsState(Options options) { this.options = options; } - void prepare(Map conf, IMetricsContext metrics, int partitionIndex, int numPartitions) { + void prepare(Map conf, IMetricsContext metrics, int partitionIndex, + int numPartitions) { this.options.prepare(conf, partitionIndex, numPartitions); initLastTxn(conf, partitionIndex); } @@ -114,8 +114,10 @@ private TxnRecord getTxnRecord(Path indexFilePath) throws IOException { } private void initLastTxn(Map conf, int partition) { - // include partition id in the file name so that index for different partitions are independent. - String indexFileName = String.format(".index.%s.%d", conf.get(Config.TOPOLOGY_NAME), partition); + // include partition id in the file name so that index for different partitions are + // independent. + String indexFileName = String.format(".index.%s.%d", conf.get(Config.TOPOLOGY_NAME), + partition); this.indexFilePath = new Path(options.fileNameFormat.getPath(), indexFileName); try { this.lastSeenTxn = getTxnRecord(indexFilePath); @@ -132,7 +134,8 @@ private void updateIndex(long txId) { try (FSDataOutputStream out = this.options.fs.create(tmpPath, true); BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(out))) { - TxnRecord txnRecord = new TxnRecord(txId, options.currentFile.toString(), this.options.getCurrentOffset()); + TxnRecord txnRecord = new TxnRecord(txId, options.currentFile.toString(), this.options + .getCurrentOffset()); bw.write(txnRecord.toString()); bw.newLine(); bw.flush(); @@ -156,7 +159,8 @@ private void updateIndex(long txId) { @Override public void beginCommit(Long txId) { if (txId <= lastSeenTxn.txnid) { - LOG.info("txID {} is already processed, lastSeenTxn {}. Triggering recovery.", txId, lastSeenTxn); + LOG.info("txID {} is already processed, lastSeenTxn {}. Triggering recovery.", txId, + lastSeenTxn); long start = System.currentTimeMillis(); options.recover(lastSeenTxn.dataFilePath, lastSeenTxn.offset); LOG.info("Recovery took {} ms.", System.currentTimeMillis() - start); @@ -184,7 +188,7 @@ public void updateState(List tuples, TridentCollector tridentColle } /** - * for unit tests. + * For unit tests. */ void close() throws IOException { this.options.closeOutputFile(); @@ -208,7 +212,8 @@ public abstract static class Options implements Serializable { abstract void execute(List tuples) throws IOException; - abstract void doPrepare(Map conf, int partitionIndex, int numPartitions) throws IOException; + abstract void doPrepare(Map conf, int partitionIndex, + int numPartitions) throws IOException; abstract long getCurrentOffset() throws IOException; @@ -238,19 +243,23 @@ protected void rotateOutputFile() throws IOException { rotateOutputFile(true); } - void prepare(Map conf, int partitionIndex, int numPartitions) { if (this.rotationPolicy == null) { throw new IllegalStateException("RotationPolicy must be specified."); } else if (this.rotationPolicy instanceof FileSizeRotationPolicy) { long rotationBytes = ((FileSizeRotationPolicy) rotationPolicy).getMaxBytes(); LOG.warn("FileSizeRotationPolicy specified with {} bytes.", rotationBytes); - LOG.warn("Recovery will fail if data files cannot be copied within topology.message.timeout.secs."); - LOG.warn("Ensure that the data files does not grow too big with the FileSizeRotationPolicy."); + LOG.warn("Recovery will fail if data files cannot be copied within " + + "topology.message.timeout.secs."); + LOG.warn("Ensure that the data files does not grow too big with the " + + "FileSizeRotationPolicy."); } else if (this.rotationPolicy instanceof TimedRotationPolicy) { - LOG.warn("TimedRotationPolicy specified with interval {} ms.", ((TimedRotationPolicy) rotationPolicy).getInterval()); - LOG.warn("Recovery will fail if data files cannot be copied within topology.message.timeout.secs."); - LOG.warn("Ensure that the data files does not grow too big with the TimedRotationPolicy."); + LOG.warn("TimedRotationPolicy specified with interval {} ms.", + ((TimedRotationPolicy) rotationPolicy).getInterval()); + LOG.warn("Recovery will fail if data files cannot be copied within " + + "topology.message.timeout.secs."); + LOG.warn("Ensure that the data files does not grow too big with the " + + "TimedRotationPolicy."); } if (this.fsUrl == null) { throw new IllegalStateException("File system URL must be specified."); @@ -276,7 +285,8 @@ void prepare(Map conf, int partitionIndex, int numPartitions) { } /** - * Recovers nBytes from srcFile to the new file created by calling rotateOutputFile and then deletes the srcFile. + * Recovers nBytes from srcFile to the new file created by calling rotateOutputFile and then + * deletes the srcFile. */ private void recover(String srcFile, long numberOfBytes) { try { @@ -285,7 +295,8 @@ private void recover(String srcFile, long numberOfBytes) { this.rotationPolicy.reset(); if (numberOfBytes > 0) { doRecover(srcPath, numberOfBytes); - LOG.info("Recovered {} bytes from {} to {}", numberOfBytes, srcFile, currentFile); + LOG.info("Recovered {} bytes from {} to {}", numberOfBytes, srcFile, + currentFile); } else { LOG.info("Nothing to recover from {}", srcFile); } @@ -335,7 +346,7 @@ public HdfsFileOptions withRotationPolicy(FileRotationPolicy rotationPolicy) { *

Set the size of the buffer used for hdfs file copy in case of recovery. The default * value is 131072.

* - *

Note: The lower limit for the parameter is 4096, below which the + *

Note: The lower limit for the parameter is 4096, below which the * option is ignored.

* * @param sizeInBytes the buffer size in bytes @@ -353,7 +364,8 @@ public HdfsFileOptions addRotationAction(RotationAction action) { } @Override - void doPrepare(Map conf, int partitionIndex, int numPartitions) throws IOException { + void doPrepare(Map conf, int partitionIndex, + int numPartitions) throws IOException { LOG.info("Preparing HDFS File state..."); this.fs = FileSystem.get(URI.create(this.fsUrl), hdfsConfig); } @@ -371,7 +383,8 @@ public void doCommit(Long txId) throws IOException { this.rotationPolicy.reset(); } else { if (this.out instanceof HdfsDataOutputStream) { - ((HdfsDataOutputStream) this.out).hsync(EnumSet.of(HdfsDataOutputStream.SyncFlag.UPDATE_LENGTH)); + ((HdfsDataOutputStream) this.out).hsync(EnumSet + .of(HdfsDataOutputStream.SyncFlag.UPDATE_LENGTH)); } else { this.out.hsync(); } @@ -386,7 +399,8 @@ void doRecover(Path srcPath, long numberOfBytes) throws IOException { this.offset = numberOfBytes; } - private void copyBytes(FSDataInputStream is, FSDataOutputStream out, long bytesToCopy) throws IOException { + private void copyBytes(FSDataInputStream is, FSDataOutputStream out, + long bytesToCopy) throws IOException { byte[] buf = new byte[bufferSize]; int n; while ((n = is.read(buf)) != -1 && bytesToCopy > 0) { @@ -402,7 +416,8 @@ void closeOutputFile() throws IOException { @Override Path createOutputFile() throws IOException { - Path path = new Path(this.fileNameFormat.getPath(), this.fileNameFormat.getName(this.rotation, System.currentTimeMillis())); + Path path = new Path(this.fileNameFormat.getPath(), this.fileNameFormat + .getName(this.rotation, System.currentTimeMillis())); this.out = this.fs.create(path); return path; } @@ -465,7 +480,8 @@ public SequenceFileOptions addRotationAction(RotationAction action) { } @Override - void doPrepare(Map conf, int partitionIndex, int numPartitions) throws IOException { + void doPrepare(Map conf, int partitionIndex, + int numPartitions) throws IOException { LOG.info("Preparing Sequence File State..."); if (this.format == null) { throw new IllegalStateException("SequenceFormat must be specified."); @@ -490,7 +506,6 @@ public void doCommit(Long txId) throws IOException { } } - @Override void doRecover(Path srcPath, long numberOfBytes) throws Exception { SequenceFile.Reader reader = new SequenceFile.Reader(this.hdfsConfig, @@ -507,13 +522,15 @@ void doRecover(Path srcPath, long numberOfBytes) throws Exception { @Override Path createOutputFile() throws IOException { Path p = new Path(this.fsUrl + this.fileNameFormat.getPath(), - this.fileNameFormat.getName(this.rotation, System.currentTimeMillis())); + this.fileNameFormat.getName(this.rotation, System + .currentTimeMillis())); this.writer = SequenceFile.createWriter( this.hdfsConfig, SequenceFile.Writer.file(p), SequenceFile.Writer.keyClass(this.format.keyClass()), SequenceFile.Writer.valueClass(this.format.valueClass()), - SequenceFile.Writer.compression(this.compressionType, this.codecFactory.getCodecByName(this.compressionCodec)) + SequenceFile.Writer.compression(this.compressionType, this.codecFactory + .getCodecByName(this.compressionCodec)) ); return p; } diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/HdfsStateFactory.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/HdfsStateFactory.java index 568f8bc9310..2cb1e2bffa9 100644 --- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/HdfsStateFactory.java +++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/HdfsStateFactory.java @@ -37,7 +37,8 @@ public HdfsStateFactory withOptions(HdfsState.Options options) { } @Override - public State makeState(Map conf, IMetricsContext metrics, int partitionIndex, int numPartitions) { + public State makeState(Map conf, IMetricsContext metrics, int partitionIndex, + int numPartitions) { LOG.info("makeState(partitonIndex={}, numpartitions={}", partitionIndex, numPartitions); HdfsState state = new HdfsState(this.options); state.prepare(conf, metrics, partitionIndex, numPartitions); diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/HdfsUpdater.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/HdfsUpdater.java index a63bb40c0e8..3a5faa91740 100644 --- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/HdfsUpdater.java +++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/HdfsUpdater.java @@ -25,7 +25,8 @@ public class HdfsUpdater extends BaseStateUpdater { @Override - public void updateState(HdfsState state, List tuples, TridentCollector collector) { + public void updateState(HdfsState state, List tuples, + TridentCollector collector) { state.updateState(tuples, collector); } } diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/DefaultFileNameFormat.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/DefaultFileNameFormat.java index e48e1986700..a79f0d586e0 100644 --- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/DefaultFileNameFormat.java +++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/DefaultFileNameFormat.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -14,7 +20,6 @@ import java.util.Map; - /** * Creates file names with the following format: *

@@ -63,7 +68,8 @@ public void prepare(Map conf, int partitionIndex, int numPartiti
 
     @Override
     public String getName(long rotation, long timeStamp) {
-        return this.prefix + "-" + this.partitionIndex + "-" + rotation + "-" + timeStamp + this.extension;
+        return this.prefix + "-" + this.partitionIndex + "-" + rotation + "-" + timeStamp
+                + this.extension;
     }
 
     @Override
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/DefaultSequenceFormat.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/DefaultSequenceFormat.java
index f33c03083cb..afa6a76736a 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/DefaultSequenceFormat.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/DefaultSequenceFormat.java
@@ -39,7 +39,6 @@ public DefaultSequenceFormat(String keyField, String valueField) {
         this.valueField = valueField;
     }
 
-
     @Override
     public Class keyClass() {
         return LongWritable.class;
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/DelimitedRecordFormat.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/DelimitedRecordFormat.java
index c12b478f964..d81aeba8fd3 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/DelimitedRecordFormat.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/DelimitedRecordFormat.java
@@ -1,12 +1,18 @@
 /**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.  The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License.  You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
  *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * 

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/FileNameFormat.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/FileNameFormat.java index 0b7ac464fb3..5e11707f32e 100644 --- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/FileNameFormat.java +++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/FileNameFormat.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -24,6 +30,7 @@ public interface FileNameFormat extends Serializable { /** * Returns the filename the HdfsBolt will create. + * * @param rotation the current file rotation number (incremented on every rotation) * @param timeStamp current time in milliseconds when the rotation occurs */ diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/RecordFormat.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/RecordFormat.java index b2f2cc3dab3..a5d6d71c2be 100644 --- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/RecordFormat.java +++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/RecordFormat.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/SimpleFileNameFormat.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/SimpleFileNameFormat.java index 889c60b661e..a12cc257b22 100644 --- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/SimpleFileNameFormat.java +++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/SimpleFileNameFormat.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -62,7 +68,7 @@ public SimpleFileNameFormat withPath(String path) { } /** - * support parameters:
+ * Support parameters:
* $TIME - current time. use withTimeFormat to format.
* $NUM - rotation number
* $HOST - local host name
@@ -76,7 +82,7 @@ public SimpleFileNameFormat withName(String name) { } public SimpleFileNameFormat withTimeFormat(String timeFormat) { - //check format + // check format try { new SimpleDateFormat(timeFormat); } catch (Exception e) { diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/rotation/FileRotationPolicy.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/rotation/FileRotationPolicy.java index a2a593246f8..3eb9666d788 100644 --- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/rotation/FileRotationPolicy.java +++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/rotation/FileRotationPolicy.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/rotation/FileSizeRotationPolicy.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/rotation/FileSizeRotationPolicy.java index f9631b706e2..d3a1339e17f 100644 --- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/rotation/FileSizeRotationPolicy.java +++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/rotation/FileSizeRotationPolicy.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/rotation/NoRotationPolicy.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/rotation/NoRotationPolicy.java index f6fedb943a3..4fdabedaade 100644 --- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/rotation/NoRotationPolicy.java +++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/rotation/NoRotationPolicy.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/sync/CountSyncPolicy.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/sync/CountSyncPolicy.java index f98dbdf8a40..3d1ac65f992 100644 --- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/sync/CountSyncPolicy.java +++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/sync/CountSyncPolicy.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/sync/SyncPolicy.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/sync/SyncPolicy.java index ba397248907..9bee858c743 100644 --- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/sync/SyncPolicy.java +++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/sync/SyncPolicy.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -30,7 +36,6 @@ public interface SyncPolicy extends Serializable { */ boolean mark(TridentTuple tuple, long offset); - /** * Called after the HdfsBolt performs a sync. * diff --git a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/avro/TestFixedAvroSerializer.java b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/avro/TestFixedAvroSerializer.java index 0fd27fdab76..cb9abf3e1c2 100644 --- a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/avro/TestFixedAvroSerializer.java +++ b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/avro/TestFixedAvroSerializer.java @@ -1,34 +1,42 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.hdfs.avro; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; + import org.apache.avro.Schema; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotEquals; - public class TestFixedAvroSerializer { - //These should match FixedAvroSerializer.config in the test resources - private static final String schemaString1 = "{\"type\":\"record\"," + - "\"name\":\"stormtest1\"," + - "\"fields\":[{\"name\":\"foo1\",\"type\":\"string\"}," + - "{ \"name\":\"int1\", \"type\":\"int\" }]}"; - private static final String schemaString2 = "{\"type\":\"record\"," + - "\"name\":\"stormtest2\"," + - "\"fields\":[{\"name\":\"foobar1\",\"type\":\"string\"}," + - "{ \"name\":\"intint1\", \"type\":\"int\" }]}"; + // These should match FixedAvroSerializer.config in the test resources + private static final String schemaString1 = "{\"type\":\"record\"," + + "\"name\":\"stormtest1\"," + + "\"fields\":[{\"name\":\"foo1\",\"type\":\"string" + + "\"}," + + "{ \"name\":\"int1\", \"type\":\"int\" }]}"; + private static final String schemaString2 = "{\"type\":\"record\"," + + "\"name\":\"stormtest2\"," + + "\"fields\":[{\"name\":\"foobar1\",\"type\":\"st" + + "ri" + + "ng\"}," + + "{ \"name\":\"intint1\", \"type\":\"int\" }]}"; private static Schema schema1; private static Schema schema2; diff --git a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/avro/TestGenericAvroSerializer.java b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/avro/TestGenericAvroSerializer.java index 20435bc5126..d51093a331b 100644 --- a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/avro/TestGenericAvroSerializer.java +++ b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/avro/TestGenericAvroSerializer.java @@ -1,33 +1,41 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.hdfs.avro; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; + import org.apache.avro.Schema; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotEquals; - public class TestGenericAvroSerializer { - private static final String schemaString1 = "{\"type\":\"record\"," + - "\"name\":\"stormtest1\"," + - "\"fields\":[{\"name\":\"foo1\",\"type\":\"string\"}," + - "{ \"name\":\"int1\", \"type\":\"int\" }]}"; - private static final String schemaString2 = "{\"type\":\"record\"," + - "\"name\":\"stormtest2\"," + - "\"fields\":[{\"name\":\"foobar1\",\"type\":\"string\"}," + - "{ \"name\":\"intint1\", \"type\":\"int\" }]}"; + private static final String schemaString1 = "{\"type\":\"record\"," + + "\"name\":\"stormtest1\"," + + "\"fields\":[{\"name\":\"foo1\",\"type\":\"string" + + "\"}," + + "{ \"name\":\"int1\", \"type\":\"int\" }]}"; + private static final String schemaString2 = "{\"type\":\"record\"," + + "\"name\":\"stormtest2\"," + + "\"fields\":[{\"name\":\"foobar1\",\"type\":\"st" + + "ri" + + "ng\"}," + + "{ \"name\":\"intint1\", \"type\":\"int\" }]}"; private static Schema schema1; private static Schema schema2; diff --git a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/bolt/AvroGenericRecordBoltTest.java b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/bolt/AvroGenericRecordBoltTest.java index bcd31e47f7e..4c260377dfc 100644 --- a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/bolt/AvroGenericRecordBoltTest.java +++ b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/bolt/AvroGenericRecordBoltTest.java @@ -1,17 +1,24 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.hdfs.bolt; +import static org.junit.jupiter.api.Assertions.assertEquals; + import java.io.File; import java.io.FileOutputStream; import java.io.IOException; @@ -54,8 +61,6 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import static org.junit.jupiter.api.Assertions.assertEquals; - @ExtendWith(MockitoExtension.class) public class AvroGenericRecordBoltTest { @@ -67,7 +72,8 @@ public class AvroGenericRecordBoltTest { private static final String schemaV2 = "{\"type\":\"record\"," + "\"name\":\"myrecord\"," + "\"fields\":[{\"name\":\"foo1\",\"type\":\"string\"}," - + "{ \"name\":\"bar\", \"type\":\"string\", \"default\":\"baz\" }," + + "{ \"name\":\"bar\", \"type\":\"string\", " + + "\"default\":\"baz\" }," + "{ \"name\":\"int1\", \"type\":\"int\" }]}"; private static Schema schema1; private static Schema schema2; @@ -75,15 +81,16 @@ public class AvroGenericRecordBoltTest { private static Tuple tuple2; @RegisterExtension - public static final MiniDFSClusterExtension DFS_CLUSTER_EXTENSION = new MiniDFSClusterExtension(() -> { - Configuration conf = new Configuration(); - conf.set("fs.trash.interval", "10"); - conf.setBoolean("dfs.permissions", true); - File baseDir = new File("./target/hdfs/").getAbsoluteFile(); - FileUtil.fullyDelete(baseDir); - conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, baseDir.getAbsolutePath()); - return conf; - }); + public static final MiniDFSClusterExtension DFS_CLUSTER_EXTENSION = + new MiniDFSClusterExtension(() -> { + Configuration conf = new Configuration(); + conf.set("fs.trash.interval", "10"); + conf.setBoolean("dfs.permissions", true); + File baseDir = new File("./target/hdfs/").getAbsoluteFile(); + FileUtil.fullyDelete(baseDir); + conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, baseDir.getAbsolutePath()); + return conf; + }); @Mock private OutputCollector collector; @Mock @@ -113,14 +120,16 @@ public static void setupClass() { private static Tuple generateTestTuple(GenericRecord record) { TopologyBuilder builder = new TopologyBuilder(); GeneralTopologyContext topologyContext = - new GeneralTopologyContext(builder.createTopology(), new Config(), new HashMap(), new HashMap<>(), + new GeneralTopologyContext(builder.createTopology(), new Config(), new HashMap(), + new HashMap<>(), new HashMap<>(), "") { - @Override + @Override public Fields getComponentOutputFields(String componentId, String streamId) { - return new Fields("record"); - } - }; - return new TupleImpl(topologyContext, new Values(record), topologyContext.getComponentId(1), 1, ""); + return new Fields("record"); + } + }; + return new TupleImpl(topologyContext, new Values(record), topologyContext.getComponentId(1), + 1, ""); } @BeforeEach @@ -170,7 +179,7 @@ public void forwardSchemaChangeWorks() throws IOException { bolt.execute(tuple1); bolt.execute(tuple2); - //Schema change should have forced a rotation + // Schema change should have forced a rotation assertEquals(2, countNonZeroLengthFiles(testRoot)); verifyAllAvroFiles(testRoot); @@ -184,7 +193,7 @@ public void backwardSchemaChangeWorks() throws IOException { bolt.execute(tuple1); bolt.execute(tuple2); - //Schema changes should have forced file rotations + // Schema changes should have forced file rotations assertEquals(2, countNonZeroLengthFiles(testRoot)); verifyAllAvroFiles(testRoot); } @@ -203,12 +212,13 @@ public void schemaThrashing() throws IOException { bolt.execute(tuple1); bolt.execute(tuple2); - //Two distinct schema should result in only two files + // Two distinct schema should result in only two files assertEquals(2, countNonZeroLengthFiles(testRoot)); verifyAllAvroFiles(testRoot); } - private AvroGenericRecordBolt makeAvroBolt(String nameNodeAddr, int countSync, float rotationSizeMB, String schemaAsString) { + private AvroGenericRecordBolt makeAvroBolt(String nameNodeAddr, int countSync, + float rotationSizeMB, String schemaAsString) { SyncPolicy fieldsSyncPolicy = new CountSyncPolicy(countSync); @@ -249,7 +259,8 @@ private int countNonZeroLengthFiles(String path) throws IOException { private void fileIsGoodAvro(Path path) throws IOException { DatumReader datumReader = new GenericDatumReader<>(); - try (FSDataInputStream in = fs.open(path, 0); FileOutputStream out = new FileOutputStream("target/FOO.avro")) { + try (FSDataInputStream in = fs.open(path, + 0); FileOutputStream out = new FileOutputStream("target/FOO.avro")) { byte[] buffer = new byte[100]; int bytesRead; while ((bytesRead = in.read(buffer)) > 0) { @@ -259,7 +270,8 @@ private void fileIsGoodAvro(Path path) throws IOException { java.io.File file = new File("target/FOO.avro"); - try (DataFileReader dataFileReader = new DataFileReader<>(file, datumReader)) { + try (DataFileReader dataFileReader = new DataFileReader<>(file, + datumReader)) { GenericRecord user = null; while (dataFileReader.hasNext()) { user = dataFileReader.next(user); diff --git a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/bolt/TestHdfsBolt.java b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/bolt/TestHdfsBolt.java index 2a6faa0f4be..5cb1bd90757 100644 --- a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/bolt/TestHdfsBolt.java +++ b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/bolt/TestHdfsBolt.java @@ -1,21 +1,31 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.hdfs.bolt; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; + import java.io.File; import java.io.IOException; import java.util.HashMap; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -52,24 +62,19 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoInteractions; - - @ExtendWith(MockitoExtension.class) public class TestHdfsBolt { @RegisterExtension - public static final MiniDFSClusterExtension DFS_CLUSTER_EXTENSION = new MiniDFSClusterExtension(() -> { - Configuration conf = new Configuration(); - conf.set("fs.trash.interval", "10"); - conf.setBoolean("dfs.permissions", true); - File baseDir = new File("./target/hdfs/").getAbsoluteFile(); - FileUtil.fullyDelete(baseDir); - conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, baseDir.getAbsolutePath()); - return conf; - }); + public static final MiniDFSClusterExtension DFS_CLUSTER_EXTENSION = + new MiniDFSClusterExtension(() -> { + Configuration conf = new Configuration(); + conf.set("fs.trash.interval", "10"); + conf.setBoolean("dfs.permissions", true); + File baseDir = new File("./target/hdfs/").getAbsoluteFile(); + FileUtil.fullyDelete(baseDir); + conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, baseDir.getAbsolutePath()); + return conf; + }); private static final String testRoot = "/unittest"; Tuple tuple1 = generateTestTuple(1, "First Tuple", "SFO", "CA"); Tuple tuple2 = generateTestTuple(1, "Second Tuple", "SJO", "CA"); @@ -83,7 +88,8 @@ public class TestHdfsBolt { @BeforeEach public void setup() throws Exception { fs = DFS_CLUSTER_EXTENSION.getDfscluster().getFileSystem(); - hdfsURI = "hdfs://localhost:" + DFS_CLUSTER_EXTENSION.getDfscluster().getNameNodePort() + "/"; + hdfsURI = "hdfs://localhost:" + DFS_CLUSTER_EXTENSION.getDfscluster().getNameNodePort() + + "/"; } @AfterEach @@ -193,18 +199,18 @@ public void testTickTuples() throws IOException { bolt.execute(tuple1); - //Should not have flushed to file system yet + // Should not have flushed to file system yet assertEquals(0, countNonZeroLengthFiles(testRoot)); bolt.execute(MockTupleHelpers.mockTickTuple()); - //Tick should have flushed it + // Tick should have flushed it assertEquals(1, countNonZeroLengthFiles(testRoot)); } @Test public void testCleanupDoesNotThrowExceptionWhenRotationPolicyIsNotTimed() { - //STORM-3372: Rotation policy other than TimedRotationPolicy causes NPE on cleanup + // STORM-3372: Rotation policy other than TimedRotationPolicy causes NPE on cleanup FileRotationPolicy fieldsRotationPolicy = new FileSizeRotationPolicy(10_000, FileSizeRotationPolicy.Units.MB); HdfsBolt bolt = makeHdfsBolt(hdfsURI, 10, 10000f) @@ -239,7 +245,8 @@ private HdfsBolt makeHdfsBolt(String nameNodeAddr, int countSync, float rotation private Tuple generateTestTuple(Object id, Object msg, Object city, Object state) { TopologyBuilder builder = new TopologyBuilder(); - GeneralTopologyContext topologyContext = new GeneralTopologyContext(builder.createTopology(), + GeneralTopologyContext topologyContext = new GeneralTopologyContext(builder + .createTopology(), new Config(), new HashMap<>(), new HashMap<>(), new HashMap<>(), "") { @Override @@ -247,10 +254,12 @@ public Fields getComponentOutputFields(String componentId, String streamId) { return new Fields("id", "msg", "city", "state"); } }; - return new TupleImpl(topologyContext, new Values(id, msg, city, state), topologyContext.getComponentId(1), 1, ""); + return new TupleImpl(topologyContext, new Values(id, msg, city, state), topologyContext + .getComponentId(1), 1, ""); } - // Generally used to compare how files were actually written and compare to expectations based on total + // Generally used to compare how files were actually written and compare to expectations based + // on total // amount of data written and rotation policies private int countNonZeroLengthFiles(String path) throws IOException { Path p = new Path(path); diff --git a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/bolt/TestSequenceFileBolt.java b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/bolt/TestSequenceFileBolt.java index 3fe4be69335..7883258243d 100644 --- a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/bolt/TestSequenceFileBolt.java +++ b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/bolt/TestSequenceFileBolt.java @@ -1,17 +1,28 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.hdfs.bolt; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; + import java.io.File; import java.io.IOException; import java.util.HashMap; @@ -50,29 +61,25 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoInteractions; - @ExtendWith(MockitoExtension.class) public class TestSequenceFileBolt { private static final Logger LOG = LoggerFactory.getLogger(TestSequenceFileBolt.class); private static final String testRoot = "/unittest"; @RegisterExtension - public static final MiniDFSClusterExtension DFS_CLUSTER_EXTENSION = new MiniDFSClusterExtension(() -> { - Configuration conf = new Configuration(); - conf.set("fs.trash.interval", "10"); - conf.setBoolean("dfs.permissions", true); - File baseDir = new File("./target/hdfs/").getAbsoluteFile(); - FileUtil.fullyDelete(baseDir); - conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, baseDir.getAbsolutePath()); - return conf; - }); - - Tuple tuple1 = generateTestTuple(1l, "first tuple"); - Tuple tuple2 = generateTestTuple(2l, "second tuple"); + public static final MiniDFSClusterExtension DFS_CLUSTER_EXTENSION = + new MiniDFSClusterExtension(() -> { + Configuration conf = new Configuration(); + conf.set("fs.trash.interval", "10"); + conf.setBoolean("dfs.permissions", true); + File baseDir = new File("./target/hdfs/").getAbsoluteFile(); + FileUtil.fullyDelete(baseDir); + conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, baseDir.getAbsolutePath()); + return conf; + }); + + Tuple tuple1 = generateTestTuple(1L, "first tuple"); + Tuple tuple2 = generateTestTuple(2L, "second tuple"); private String hdfsURI; private DistributedFileSystem fs; @Mock @@ -83,7 +90,8 @@ public class TestSequenceFileBolt { @BeforeEach public void setup() throws Exception { fs = DFS_CLUSTER_EXTENSION.getDfscluster().getFileSystem(); - hdfsURI = "hdfs://localhost:" + DFS_CLUSTER_EXTENSION.getDfscluster().getNameNodePort() + "/"; + hdfsURI = "hdfs://localhost:" + DFS_CLUSTER_EXTENSION.getDfscluster().getNameNodePort() + + "/"; } @AfterEach @@ -152,7 +160,8 @@ private SequenceFileBolt makeSeqBolt(String nameNodeAddr, int countSync, float r private Tuple generateTestTuple(Long key, String value) { TopologyBuilder builder = new TopologyBuilder(); - GeneralTopologyContext topologyContext = new GeneralTopologyContext(builder.createTopology(), + GeneralTopologyContext topologyContext = new GeneralTopologyContext(builder + .createTopology(), new Config(), new HashMap<>(), new HashMap<>(), new HashMap<>(), "") { @Override @@ -160,10 +169,12 @@ public Fields getComponentOutputFields(String componentId, String streamId) { return new Fields("key", "value"); } }; - return new TupleImpl(topologyContext, new Values(key, value), topologyContext.getComponentId(1), 1, ""); + return new TupleImpl(topologyContext, new Values(key, value), topologyContext + .getComponentId(1), 1, ""); } - // Generally used to compare how files were actually written and compare to expectations based on total + // Generally used to compare how files were actually written and compare to expectations based + // on total // amount of data written and rotation policies private int countNonZeroLengthFiles(String path) throws IOException { Path p = new Path(path); diff --git a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/bolt/TestWritersMap.java b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/bolt/TestWritersMap.java index 1fabcee4e73..69a4fe23549 100644 --- a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/bolt/TestWritersMap.java +++ b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/bolt/TestWritersMap.java @@ -1,17 +1,26 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.hdfs.bolt; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + import org.apache.hadoop.fs.Path; import org.apache.storm.hdfs.bolt.rotation.FileRotationPolicy; import org.apache.storm.hdfs.bolt.rotation.FileSizeRotationPolicy; @@ -19,25 +28,25 @@ import org.apache.storm.tuple.Tuple; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - public class TestWritersMap { AbstractHdfsBolt.WritersMap map = new AbstractHdfsBolt.WritersMap(2, null); - AbstractHDFSWriterMock foo = new AbstractHDFSWriterMock(new FileSizeRotationPolicy(1, FileSizeRotationPolicy.Units.KB), null); - AbstractHDFSWriterMock bar = new AbstractHDFSWriterMock(new FileSizeRotationPolicy(1, FileSizeRotationPolicy.Units.KB), null); - AbstractHDFSWriterMock baz = new AbstractHDFSWriterMock(new FileSizeRotationPolicy(1, FileSizeRotationPolicy.Units.KB), null); + AbstractHDFSWriterMock foo = new AbstractHDFSWriterMock(new FileSizeRotationPolicy(1, + FileSizeRotationPolicy.Units.KB), null); + AbstractHDFSWriterMock bar = new AbstractHDFSWriterMock(new FileSizeRotationPolicy(1, + FileSizeRotationPolicy.Units.KB), null); + AbstractHDFSWriterMock baz = new AbstractHDFSWriterMock(new FileSizeRotationPolicy(1, + FileSizeRotationPolicy.Units.KB), null); @Test public void testLRUBehavior() { map.put("FOO", foo); map.put("BAR", bar); - //Access foo to make it most recently used + // Access foo to make it most recently used map.get("FOO"); - //Add an element and bar should drop out + // Add an element and bar should drop out map.put("BAZ", baz); assertTrue(map.keySet().contains("FOO")); diff --git a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/bolt/format/TestSimpleFileNameFormat.java b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/bolt/format/TestSimpleFileNameFormat.java index 4ac1257dca7..c5da49c448d 100644 --- a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/bolt/format/TestSimpleFileNameFormat.java +++ b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/bolt/format/TestSimpleFileNameFormat.java @@ -1,17 +1,26 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.hdfs.bolt.format; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + import java.net.UnknownHostException; import java.text.SimpleDateFormat; import java.util.HashMap; @@ -20,9 +29,6 @@ import org.apache.storm.utils.Utils; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; - public class TestSimpleFileNameFormat { @Test @@ -66,13 +72,15 @@ public void testTimeFormat() { Map topoConf = new HashMap(); SimpleFileNameFormat format = new SimpleFileNameFormat() .withTimeFormat("xyz"); - assertThrows(IllegalArgumentException.class, () -> format.prepare(null, createTopologyContext(topoConf))); + assertThrows(IllegalArgumentException.class, () -> format.prepare(null, + createTopologyContext(topoConf))); } private TopologyContext createTopologyContext(Map topoConf) { Map taskToComponent = new HashMap<>(); taskToComponent.put(7, "Xcom"); - return new TopologyContext(null, topoConf, taskToComponent, null, null, null, null, null, null, 7, 6703, null, null, null, null, + return new TopologyContext(null, topoConf, taskToComponent, null, null, null, null, null, + null, 7, 6703, null, null, null, null, null, null, null); } } diff --git a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/ConfigsTest.java b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/ConfigsTest.java index 56cc1694d3a..6539b041764 100644 --- a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/ConfigsTest.java +++ b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/ConfigsTest.java @@ -1,24 +1,29 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.hdfs.spout; +import static org.junit.jupiter.api.Assertions.fail; + import java.util.HashMap; import java.util.Map; import org.apache.storm.validation.ConfigValidation; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.fail; - public class ConfigsTest { public static void verifyBad(String key, Object value) { @@ -28,7 +33,7 @@ public static void verifyBad(String key, Object value) { ConfigValidation.validateFields(conf); fail("Expected " + key + " = " + value + " to throw Exception, but it didn't"); } catch (IllegalArgumentException e) { - //good + // good } } diff --git a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestDirLock.java b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestDirLock.java index 629249c4d84..e0ba778117b 100644 --- a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestDirLock.java +++ b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestDirLock.java @@ -1,17 +1,27 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.hdfs.spout; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + import java.io.IOException; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.FileSystem; @@ -23,17 +33,13 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; - public class TestDirLock { private static final int LOCK_EXPIRY_SEC = 1; private final Path locksDir = new Path("/tmp/lockdir"); @RegisterExtension - public static final MiniDFSClusterExtension DFS_CLUSTER_EXTENSION = new MiniDFSClusterExtension(); + public static final MiniDFSClusterExtension DFS_CLUSTER_EXTENSION = + new MiniDFSClusterExtension(); private FileSystem fs; private HdfsConfiguration conf = new HdfsConfiguration(); @@ -90,7 +96,8 @@ public void testConcurrentLocking() throws Exception { thread.interrupt(); thread.join(30_000); if (thread.isAlive()) { - throw new RuntimeException("Failed to stop threads within 30 seconds, threads may leak into other tests"); + throw new RuntimeException("Failed to stop threads within 30 seconds, " + + "threads may leak into other tests"); } } } @@ -121,7 +128,8 @@ public void testLockRecovery() throws Exception { Thread.sleep(LOCK_EXPIRY_SEC * 1000 + 500); // wait for lock to expire assertTrue(fs.exists(lock1.getLockFile())); - DirLock lock3 = DirLock.takeOwnershipIfStale(fs, locksDir, LOCK_EXPIRY_SEC); // should pass now + DirLock lock3 = DirLock.takeOwnershipIfStale(fs, locksDir, + LOCK_EXPIRY_SEC); // should pass now assertNotNull(lock3); assertTrue(fs.exists(lock3.getLockFile())); lock3.release(); diff --git a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestFileLock.java b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestFileLock.java index 54e7ab6f1e3..f7a9cb267ce 100644 --- a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestFileLock.java +++ b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestFileLock.java @@ -1,17 +1,28 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.hdfs.spout; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + import java.io.BufferedReader; import java.io.FileNotFoundException; import java.io.IOException; @@ -30,18 +41,13 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; - public class TestFileLock { private final Path filesDir = new Path("/tmp/filesdir"); private final Path locksDir = new Path("/tmp/locksdir"); @RegisterExtension - public static final MiniDFSClusterExtension DFS_CLUSTER_EXTENSION = new MiniDFSClusterExtension(); + public static final MiniDFSClusterExtension DFS_CLUSTER_EXTENSION = + new MiniDFSClusterExtension(); private FileSystem fs; private HdfsConfiguration conf = new HdfsConfiguration(); @@ -165,7 +171,8 @@ public void testConcurrentLocking() throws IOException, InterruptedException { thread.interrupt(); thread.join(30_000); if (thread.isAlive()) { - throw new RuntimeException("Failed to stop threads within 30 seconds, threads may leak into other tests"); + throw new RuntimeException("Failed to stop threads within 30 seconds, " + + "threads may leak into other tests"); } } } @@ -197,7 +204,8 @@ public void testStaleLockDetection_SingleLock() throws Exception { assertNotNull(lock1); assertTrue(fs.exists(lock1.getLockFile())); Thread.sleep(WAIT_MSEC); // wait for lock to expire - HdfsUtils.Pair expired = FileLock.locateOldestExpiredLock(fs, locksDir, LOCK_EXPIRY_SEC); + HdfsUtils.Pair expired = FileLock.locateOldestExpiredLock(fs, + locksDir, LOCK_EXPIRY_SEC); assertNotNull(expired); // heartbeat, ensure its no longer stale and read back the heartbeat data @@ -240,7 +248,8 @@ public void testStaleLockDetection_MultipleLocks() throws Exception { assertNotNull(lock3); try { - HdfsUtils.Pair expired = FileLock.locateOldestExpiredLock(fs, locksDir, LOCK_EXPIRY_SEC); + HdfsUtils.Pair expired = FileLock.locateOldestExpiredLock(fs, + locksDir, LOCK_EXPIRY_SEC); assertNull(expired); // 2) wait for all 3 locks to expire then heart beat on 2 locks and verify stale lock @@ -282,7 +291,8 @@ public void testLockRecovery() throws Exception { assertNotNull(lock3); try { - HdfsUtils.Pair expired = FileLock.locateOldestExpiredLock(fs, locksDir, LOCK_EXPIRY_SEC); + HdfsUtils.Pair expired = FileLock.locateOldestExpiredLock(fs, + locksDir, LOCK_EXPIRY_SEC); assertNull(expired); // 1) Simulate lock file lease expiring and getting closed by HDFS @@ -294,9 +304,11 @@ public void testLockRecovery() throws Exception { lock2.heartbeat("1"); // 3) Take ownership of stale lock - FileLock lock3b = FileLock.acquireOldestExpiredLock(fs, locksDir, LOCK_EXPIRY_SEC, "spout1"); + FileLock lock3b = FileLock.acquireOldestExpiredLock(fs, locksDir, LOCK_EXPIRY_SEC, + "spout1"); assertNotNull(lock3b); - assertEquals(Path.getPathWithoutSchemeAndAuthority(lock3b.getLockFile()), lock3.getLockFile(), "Expected lock3 file"); + assertEquals(Path.getPathWithoutSchemeAndAuthority(lock3b.getLockFile()), lock3 + .getLockFile(), "Expected lock3 file"); } finally { lock1.release(); lock2.release(); @@ -312,7 +324,7 @@ public void testLockRecovery() throws Exception { } /** - * return null if file not found + * Return null if file not found. */ private ArrayList readTextFile(Path file) throws IOException { try (FSDataInputStream os = fs.open(file)) { @@ -339,7 +351,8 @@ static class FileLockingThread extends Thread { private final Path locksDir; private final String spoutId; - public FileLockingThread(int thdNum, FileSystem fs, Path fileToLock, Path locksDir, String spoutId) { + public FileLockingThread(int thdNum, FileSystem fs, Path fileToLock, Path locksDir, + String spoutId) { this.thdNum = thdNum; this.fs = fs; this.fileToLock = fileToLock; diff --git a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestHdfsSemantics.java b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestHdfsSemantics.java index d17e8ac96b7..0f09c48b75a 100644 --- a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestHdfsSemantics.java +++ b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestHdfsSemantics.java @@ -1,12 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -14,6 +19,11 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.core.IsNull.notNullValue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; import java.io.IOException; import org.apache.hadoop.fs.CommonConfigurationKeys; @@ -30,18 +40,13 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; - public class TestHdfsSemantics { private final HdfsConfiguration conf = new HdfsConfiguration(); private final Path dir = new Path("/tmp/filesdir"); @RegisterExtension - public static final MiniDFSClusterExtension DFS_CLUSTER_EXTENSION = new MiniDFSClusterExtension(); + public static final MiniDFSClusterExtension DFS_CLUSTER_EXTENSION = + new MiniDFSClusterExtension(); private FileSystem fs; @BeforeEach @@ -107,7 +112,8 @@ public void testConcurrentDeletion() throws Exception { thread.interrupt(); thread.join(30_000); if (thread.isAlive()) { - throw new RuntimeException("Failed to stop threads within 30 seconds, threads may leak into other tests"); + throw new RuntimeException("Failed to stop threads within 30 seconds, " + + "threads may leak into other tests"); } } } @@ -116,7 +122,7 @@ public void testConcurrentDeletion() throws Exception { @Test public void testAppendSemantics() throws Exception { - //1 try to append to an open file + // 1 try to append to an open file Path file1 = new Path(dir.toString() + Path.SEPARATOR_CHAR + "file1"); try (FSDataOutputStream os1 = fs.create(file1, false)) { fs.append(file1); // should fail @@ -126,7 +132,7 @@ public void testAppendSemantics() throws Exception { assertEquals(AlreadyBeingCreatedException.class, e.unwrapRemoteException().getClass()); } - //2 try to append to a closed file + // 2 try to append to a closed file try (FSDataOutputStream os2 = fs.append(file1)) { assertThat(os2, notNullValue()); } @@ -134,7 +140,7 @@ public void testAppendSemantics() throws Exception { @Test public void testDoubleCreateSemantics() throws Exception { - //1 create an already existing open file w/o override flag + // 1 create an already existing open file w/o override flag Path file1 = new Path(dir.toString() + Path.SEPARATOR_CHAR + "file1"); try (FSDataOutputStream os1 = fs.create(file1, false)) { fs.create(file1, false); // should fail @@ -142,7 +148,7 @@ public void testDoubleCreateSemantics() throws Exception { } catch (RemoteException e) { assertEquals(AlreadyBeingCreatedException.class, e.unwrapRemoteException().getClass()); } - //2 close file and retry creation + // 2 close file and retry creation try { fs.create(file1, false); // should still fail fail("Create did not throw an exception"); @@ -150,7 +156,7 @@ public void testDoubleCreateSemantics() throws Exception { // expecting this exception } - //3 delete file and retry creation + // 3 delete file and retry creation fs.delete(file1, false); try (FSDataOutputStream os2 = fs.create(file1, false)) { assertNotNull(os2); diff --git a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestHdfsSpout.java b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestHdfsSpout.java index f6f72142339..e107764a17c 100644 --- a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestHdfsSpout.java +++ b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestHdfsSpout.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -14,7 +20,12 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.core.Is.is; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.BufferedReader; import java.io.File; @@ -40,8 +51,8 @@ import org.apache.hadoop.io.Writable; import org.apache.hadoop.util.ReflectionUtils; import org.apache.storm.Config; -import org.apache.storm.hdfs.common.HdfsUtils; import org.apache.storm.hdfs.common.HdfsUtils.Pair; +import org.apache.storm.hdfs.common.HdfsUtils; import org.apache.storm.hdfs.testing.MiniDFSClusterExtensionClassLevel; import org.apache.storm.spout.SpoutOutputCollector; import org.apache.storm.task.TopologyContext; @@ -57,7 +68,8 @@ public class TestHdfsSpout { private static final Configuration conf = new Configuration(); @RegisterExtension - public static final MiniDFSClusterExtensionClassLevel DFS_CLUSTER_EXTENSION = new MiniDFSClusterExtensionClassLevel(); + public static final MiniDFSClusterExtensionClassLevel DFS_CLUSTER_EXTENSION = + new MiniDFSClusterExtensionClassLevel(); private static DistributedFileSystem fs; @TempDir public File tempFolder; @@ -76,13 +88,15 @@ public static void teardownClass() throws IOException { fs.close(); } - private static T getField(HdfsSpout spout, String fieldName) throws NoSuchFieldException, IllegalAccessException { + private static T getField(HdfsSpout spout, + String fieldName) throws NoSuchFieldException, IllegalAccessException { Field readerFld = HdfsSpout.class.getDeclaredField(fieldName); readerFld.setAccessible(true); return (T) readerFld.get(spout); } - private static boolean getBoolField(HdfsSpout spout, String fieldName) throws NoSuchFieldException, IllegalAccessException { + private static boolean getBoolField(HdfsSpout spout, + String fieldName) throws NoSuchFieldException, IllegalAccessException { Field readerFld = HdfsSpout.class.getDeclaredField(fieldName); readerFld.setAccessible(true); return readerFld.getBoolean(spout); @@ -108,7 +122,8 @@ private static void createSeqFile(FileSystem fs, Path file, int rowCount) throws fs.delete(file, false); } - SequenceFile.Writer w = SequenceFile.createWriter(fs, conf, file, IntWritable.class, Text.class); + SequenceFile.Writer w = SequenceFile.createWriter(fs, conf, file, IntWritable.class, + Text.class); for (int i = 0; i < rowCount; i++) { w.append(new IntWritable(i), new Text("line " + i)); } @@ -145,7 +160,8 @@ public void testSimpleText_noACK() throws Exception { Path file2 = new Path(source.toString() + "/file2.txt"); createTextFile(file2, 5); - try (AutoCloseableHdfsSpout closeableSpout = makeSpout(Configs.TEXT, TextFileReader.defaultFields)) { + try (AutoCloseableHdfsSpout closeableSpout = makeSpout(Configs.TEXT, + TextFileReader.defaultFields)) { HdfsSpout spout = closeableSpout.spout; spout.setCommitFrequencyCount(1); spout.setCommitFrequencySec(1); @@ -169,7 +185,8 @@ public void testSimpleText_ACK() throws Exception { Path file2 = new Path(source.toString() + "/file2.txt"); createTextFile(file2, 5); - try (AutoCloseableHdfsSpout closeableSpout = makeSpout(Configs.TEXT, TextFileReader.defaultFields)) { + try (AutoCloseableHdfsSpout closeableSpout = makeSpout(Configs.TEXT, + TextFileReader.defaultFields)) { HdfsSpout spout = closeableSpout.spout; spout.setCommitFrequencyCount(1); spout.setCommitFrequencySec(1); @@ -195,13 +212,15 @@ public void testEmptySimpleText_ACK() throws Exception { Path file1 = new Path(source.toString() + "/file_empty.txt"); createTextFile(file1, 0); - //Ensure the second file has a later modified timestamp, as the spout should pick the first file first. + // Ensure the second file has a later modified timestamp, as the spout should pick the first + // file first. Thread.sleep(2); Path file2 = new Path(source.toString() + "/file.txt"); createTextFile(file2, 5); - try (AutoCloseableHdfsSpout closeableSpout = makeSpout(Configs.TEXT, TextFileReader.defaultFields)) { + try (AutoCloseableHdfsSpout closeableSpout = makeSpout(Configs.TEXT, + TextFileReader.defaultFields)) { HdfsSpout spout = closeableSpout.spout; spout.setCommitFrequencyCount(1); @@ -211,9 +230,9 @@ public void testEmptySimpleText_ACK() throws Exception { // Read once. Since the first file is empty, the spout should continue with file 2 runSpout(spout, "r6", "a0", "a1", "a2", "a3", "a4"); - //File 1 should be moved to archive + // File 1 should be moved to archive assertThat(fs.isFile(new Path(archive.toString() + "/file_empty.txt")), is(true)); - //File 2 should be read + // File 2 should be read Path arc2 = new Path(archive.toString() + "/file.txt"); checkCollectorOutput_txt((MockCollector) spout.getCollector(), arc2); } @@ -226,13 +245,15 @@ public void testResumeAbandoned_Text_NoAck() throws Exception { final Integer lockExpirySec = 1; - try (AutoCloseableHdfsSpout closeableSpout = makeSpout(Configs.TEXT, TextFileReader.defaultFields)) { + try (AutoCloseableHdfsSpout closeableSpout = makeSpout(Configs.TEXT, + TextFileReader.defaultFields)) { HdfsSpout spout = closeableSpout.spout; spout.setCommitFrequencyCount(1); spout.setCommitFrequencySec(1000); // effectively disable commits based on time spout.setLockTimeoutSec(lockExpirySec); - try (AutoCloseableHdfsSpout closeableSpout2 = makeSpout(Configs.TEXT, TextFileReader.defaultFields)) { + try (AutoCloseableHdfsSpout closeableSpout2 = makeSpout(Configs.TEXT, + TextFileReader.defaultFields)) { HdfsSpout spout2 = closeableSpout2.spout; spout2.setCommitFrequencyCount(1); spout2.setCommitFrequencySec(1000); // effectively disable commits based on time @@ -285,13 +306,15 @@ public void testResumeAbandoned_Seq_NoAck() throws Exception { final Integer lockExpirySec = 1; - try (AutoCloseableHdfsSpout closeableSpout = makeSpout(Configs.SEQ, SequenceFileReader.defaultFields)) { + try (AutoCloseableHdfsSpout closeableSpout = makeSpout(Configs.SEQ, + SequenceFileReader.defaultFields)) { HdfsSpout spout = closeableSpout.spout; spout.setCommitFrequencyCount(1); spout.setCommitFrequencySec(1000); // effectively disable commits based on time spout.setLockTimeoutSec(lockExpirySec); - try (AutoCloseableHdfsSpout closeableSpout2 = makeSpout(Configs.SEQ, SequenceFileReader.defaultFields)) { + try (AutoCloseableHdfsSpout closeableSpout2 = makeSpout(Configs.SEQ, + SequenceFileReader.defaultFields)) { HdfsSpout spout2 = closeableSpout2.spout; spout2.setCommitFrequencyCount(1); spout2.setCommitFrequencySec(1000); // effectively disable commits based on time @@ -336,7 +359,8 @@ public void testResumeAbandoned_Seq_NoAck() throws Exception { } } - private void checkCollectorOutput_txt(MockCollector collector, Path... txtFiles) throws IOException { + private void checkCollectorOutput_txt(MockCollector collector, + Path... txtFiles) throws IOException { ArrayList expected = new ArrayList<>(); for (Path txtFile : txtFiles) { List lines = getTextFileContents(fs, txtFile); @@ -363,7 +387,8 @@ private List getTextFileContents(FileSystem fs, Path txtFile) throws IOE return result; } - private void checkCollectorOutput_seq(MockCollector collector, Path... seqFiles) throws IOException { + private void checkCollectorOutput_seq(MockCollector collector, + Path... seqFiles) throws IOException { ArrayList expected = new ArrayList<>(); for (Path seqFile : seqFiles) { List lines = getSeqFileContents(fs, seqFile); @@ -377,10 +402,12 @@ private List getSeqFileContents(FileSystem fs, Path... seqFiles) throws for (Path seqFile : seqFiles) { Path file = new Path(fs.getUri().toString() + seqFile.toString()); - SequenceFile.Reader reader = new SequenceFile.Reader(conf, SequenceFile.Reader.file(file)); + SequenceFile.Reader reader = new SequenceFile.Reader(conf, SequenceFile.Reader + .file(file)); try { Writable key = (Writable) ReflectionUtils.newInstance(reader.getKeyClass(), conf); - Writable value = (Writable) ReflectionUtils.newInstance(reader.getValueClass(), conf); + Writable value = (Writable) ReflectionUtils.newInstance(reader.getValueClass(), + conf); while (reader.next(key, value)) { String keyValStr = Arrays.asList(key, value).toString(); result.add(keyValStr); @@ -388,7 +415,7 @@ private List getSeqFileContents(FileSystem fs, Path... seqFiles) throws } finally { reader.close(); } - }// for + } // for return result; } @@ -407,7 +434,8 @@ public void testMultipleFileConsumption_Ack() throws Exception { Path file1 = new Path(source.toString() + "/file1.txt"); createTextFile(file1, 5); - try (AutoCloseableHdfsSpout closeableSpout = makeSpout(Configs.TEXT, TextFileReader.defaultFields)) { + try (AutoCloseableHdfsSpout closeableSpout = makeSpout(Configs.TEXT, + TextFileReader.defaultFields)) { HdfsSpout spout = closeableSpout.spout; spout.setCommitFrequencyCount(1); spout.setCommitFrequencySec(1); @@ -434,7 +462,7 @@ public void testMultipleFileConsumption_Ack() throws Exception { assertNotNull(reader); assertTrue(getBoolField(spout, "fileReadCompletely")); - //ack rest + // ack rest runSpout(spout, "a3", "a4"); reader = getField(spout, "reader"); assertNull(reader); @@ -463,7 +491,7 @@ public void testMultipleFileConsumption_Ack() throws Exception { @Test public void testSimpleSequenceFile() throws Exception { - //1) create a couple files to consume + // 1) create a couple files to consume source = new Path("/tmp/hdfsspout/source"); fs.mkdirs(source); archive = new Path("/tmp/hdfsspout/archive"); @@ -475,7 +503,8 @@ public void testSimpleSequenceFile() throws Exception { Path file2 = new Path(source + "/file2.seq"); createSeqFile(fs, file2, 5); - try (AutoCloseableHdfsSpout closeableSpout = makeSpout(Configs.SEQ, SequenceFileReader.defaultFields)) { + try (AutoCloseableHdfsSpout closeableSpout = makeSpout(Configs.SEQ, + SequenceFileReader.defaultFields)) { HdfsSpout spout = closeableSpout.spout; Map conf = getCommonConfigs(); openSpout(spout, 0, conf); @@ -505,13 +534,15 @@ public void testReadFailures() throws Exception { // 2) run spout try ( - AutoCloseableHdfsSpout closeableSpout = makeSpout(MockTextFailingReader.class.getName(), MockTextFailingReader.defaultFields)) { + AutoCloseableHdfsSpout closeableSpout = makeSpout(MockTextFailingReader.class.getName(), + MockTextFailingReader.defaultFields)) { HdfsSpout spout = closeableSpout.spout; Map conf = getCommonConfigs(); openSpout(spout, 0, conf); List res = runSpout(spout, "r11"); - String[] expected = new String[]{ "[line 0]", "[line 1]", "[line 2]", "[line 0]", "[line 1]", "[line 2]" }; + String[] expected = + new String[]{ "[line 0]", "[line 1]", "[line 2]", "[line 0]", "[line 1]", "[line 2]" }; assertArrayEquals(expected, res.toArray()); // 3) make sure 6 lines (3 from each file) were read in all @@ -528,7 +559,8 @@ public void testLocking() throws Exception { createTextFile(file1, 10); // 0) config spout to log progress in lock file for each tuple - try (AutoCloseableHdfsSpout closeableSpout = makeSpout(Configs.TEXT, TextFileReader.defaultFields)) { + try (AutoCloseableHdfsSpout closeableSpout = makeSpout(Configs.TEXT, + TextFileReader.defaultFields)) { HdfsSpout spout = closeableSpout.spout; spout.setCommitFrequencyCount(1); spout.setCommitFrequencySec(1000); // effectively disable commits based on time @@ -578,7 +610,8 @@ public void testLockLoggingFreqCount() throws Exception { createTextFile(file1, 10); // 0) config spout to log progress in lock file for each tuple - try (AutoCloseableHdfsSpout closeableSpout = makeSpout(Configs.TEXT, TextFileReader.defaultFields)) { + try (AutoCloseableHdfsSpout closeableSpout = makeSpout(Configs.TEXT, + TextFileReader.defaultFields)) { HdfsSpout spout = closeableSpout.spout; spout.setCommitFrequencyCount(2); // 1 lock log entry every 2 tuples spout.setCommitFrequencySec(1000); // Effectively disable commits based on time @@ -607,7 +640,8 @@ public void testLockLoggingFreqSec() throws Exception { createTextFile(file1, 10); // 0) config spout to log progress in lock file for each tuple - try (AutoCloseableHdfsSpout closeableSpout = makeSpout(Configs.TEXT, TextFileReader.defaultFields)) { + try (AutoCloseableHdfsSpout closeableSpout = makeSpout(Configs.TEXT, + TextFileReader.defaultFields)) { HdfsSpout spout = closeableSpout.spout; spout.setCommitFrequencyCount(0); // disable it spout.setCommitFrequencySec(2); // log every 2 sec @@ -640,7 +674,8 @@ private Map getCommonConfigs() { private AutoCloseableHdfsSpout makeSpout(String readerType, String[] outputFields) { HdfsSpout spout = new HdfsSpout().withOutputFields(outputFields) .setReaderType(readerType) - .setHdfsUri(DFS_CLUSTER_EXTENSION.getDfscluster().getURI().toString()) + .setHdfsUri(DFS_CLUSTER_EXTENSION.getDfscluster().getURI() + .toString()) .setSourceDir(source.toString()) .setArchiveDir(archive.toString()) .setBadFilesDir(badfiles.toString()); @@ -656,8 +691,9 @@ private void openSpout(HdfsSpout spout, int spoutId, Map topoCon /** * Execute a sequence of calls on HdfsSpout. * - * @param cmds: set of commands to run, e.g. "r,r,r,r,a1,f2,...". The commands are: r[N] - receive() called N times aN - ack, item - * number: N fN - fail, item number: N + * @param cmds: set of commands to run, e.g. "r,r,r,r,a1,f2,...". The commands are: r[N] - + * receive() called N times aN - ack, item + * number: N fN - fail, item number: N */ private List runSpout(HdfsSpout spout, String... cmds) { MockCollector collector = (MockCollector) spout.getCollector(); @@ -706,7 +742,7 @@ public void close() throws Exception { } static class MockCollector extends SpoutOutputCollector { - //comma separated offsets + // comma separated offsets public ArrayList lines; public ArrayList>> items; @@ -752,7 +788,8 @@ static class MockTextFailingReader extends TextFileReader { public static final String[] defaultFields = { "line" }; int readAttempts = 0; - public MockTextFailingReader(FileSystem fs, Path file, Map conf) throws IOException { + public MockTextFailingReader(FileSystem fs, Path file, Map conf) throws IOException { super(fs, file, conf); } @@ -773,7 +810,8 @@ static class MockTopologyContext extends TopologyContext { private final int componentId; public MockTopologyContext(int componentId, Map topoConf) { - super(null, topoConf, null, null, null, null, null, null, null, 0, 0, null, null, null, null, null, null, null); + super(null, topoConf, null, null, null, null, null, null, null, 0, 0, null, null, null, + null, null, null, null); this.componentId = componentId; } diff --git a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestProgressTracker.java b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestProgressTracker.java index fbcd67449d9..c1715ba731a 100644 --- a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestProgressTracker.java +++ b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestProgressTracker.java @@ -1,17 +1,26 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.hdfs.spout; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + import java.io.File; import java.io.IOException; import org.apache.hadoop.conf.Configuration; @@ -22,8 +31,6 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -import static org.junit.jupiter.api.Assertions.*; - public class TestProgressTracker { @TempDir @@ -73,15 +80,15 @@ public void testBasic() throws Exception { assertEquals(pos2, pos2b); // read lines 3..7, don't ACK .. commit pos should remain same - assertNotNull(reader.next());//3 + assertNotNull(reader.next()); // 3 TextFileReader.Offset pos3 = reader.getFileOffset(); - assertNotNull(reader.next());//4 + assertNotNull(reader.next()); // 4 TextFileReader.Offset pos4 = reader.getFileOffset(); - assertNotNull(reader.next());//5 + assertNotNull(reader.next()); // 5 TextFileReader.Offset pos5 = reader.getFileOffset(); - assertNotNull(reader.next());//6 + assertNotNull(reader.next()); // 6 TextFileReader.Offset pos6 = reader.getFileOffset(); - assertNotNull(reader.next());//7 + assertNotNull(reader.next()); // 7 TextFileReader.Offset pos7 = reader.getFileOffset(); // now ack msg 5 and check diff --git a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/testing/MiniDFSClusterExtension.java b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/testing/MiniDFSClusterExtension.java index 93a0eab106b..4e2791ed4c5 100644 --- a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/testing/MiniDFSClusterExtension.java +++ b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/testing/MiniDFSClusterExtension.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -17,6 +17,10 @@ */ package org.apache.storm.hdfs.testing; +import static org.apache.hadoop.test.GenericTestUtils.DEFAULT_TEST_DATA_DIR; +import static org.apache.hadoop.test.GenericTestUtils.SYSPROP_TEST_DATA_DIR; +import static org.junit.jupiter.api.Assertions.assertTrue; + import java.io.File; import java.util.function.Supplier; import org.apache.hadoop.conf.Configuration; @@ -25,10 +29,6 @@ import org.junit.jupiter.api.extension.BeforeEachCallback; import org.junit.jupiter.api.extension.ExtensionContext; -import static org.apache.hadoop.test.GenericTestUtils.DEFAULT_TEST_DATA_DIR; -import static org.apache.hadoop.test.GenericTestUtils.SYSPROP_TEST_DATA_DIR; -import static org.junit.jupiter.api.Assertions.assertTrue; - public class MiniDFSClusterExtension implements BeforeEachCallback, AfterEachCallback { private static final String TEST_BUILD_DATA = "test.build.data"; @@ -73,6 +73,7 @@ public void afterEach(ExtensionContext arg0) throws Exception { * Get an uncreated directory for tests. * We use this method to get rid of getTestDir() in GenericTestUtils in Hadoop code * which uses assert from junit4. + * * @return the absolute directory for tests. Caller is expected to create it. */ public static File getTestDir(String subdir) { @@ -81,6 +82,7 @@ public static File getTestDir(String subdir) { /** * Get the (created) base directory for tests. + * * @return the absolute directory */ public static File getTestDir() { diff --git a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/testing/MiniDFSClusterExtensionClassLevel.java b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/testing/MiniDFSClusterExtensionClassLevel.java index d13208a7c4a..b0c992e01c6 100644 --- a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/testing/MiniDFSClusterExtensionClassLevel.java +++ b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/testing/MiniDFSClusterExtensionClassLevel.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -17,6 +17,8 @@ */ package org.apache.storm.hdfs.testing; +import static org.apache.storm.hdfs.testing.MiniDFSClusterExtension.getTestDir; + import java.io.File; import java.util.function.Supplier; import org.apache.hadoop.conf.Configuration; @@ -25,8 +27,6 @@ import org.junit.jupiter.api.extension.BeforeAllCallback; import org.junit.jupiter.api.extension.ExtensionContext; -import static org.apache.storm.hdfs.testing.MiniDFSClusterExtension.getTestDir; - public class MiniDFSClusterExtensionClassLevel implements BeforeAllCallback, AfterAllCallback { private static final String TEST_BUILD_DATA = "test.build.data"; diff --git a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/trident/HdfsStateTest.java b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/trident/HdfsStateTest.java index 8c02191b4b8..3649d6ef54f 100644 --- a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/trident/HdfsStateTest.java +++ b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/trident/HdfsStateTest.java @@ -1,17 +1,30 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.hdfs.trident; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + import java.io.File; import java.io.IOException; import java.nio.charset.Charset; @@ -35,17 +48,10 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - - public class HdfsStateTest { - private static final String TEST_OUT_DIR = Paths.get(System.getProperty("java.io.tmpdir"), "trident-unit-test").toString(); + private static final String TEST_OUT_DIR = Paths.get(System.getProperty("java.io.tmpdir"), + "trident-unit-test").toString(); private static final String FILE_NAME_PREFIX = "hdfs-data-"; private static final String TEST_TOPOLOGY_NAME = "test-topology"; @@ -58,7 +64,8 @@ private HdfsState createHdfsState() { RecordFormat recordFormat = new DelimitedRecordFormat().withFields(hdfsFields); - FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(5.0f, FileSizeRotationPolicy.Units.MB); + FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(5.0f, + FileSizeRotationPolicy.Units.MB); HdfsState.Options options = new HdfsState.HdfsFileOptions() .withFileNameFormat(fileNameFormat) @@ -107,7 +114,8 @@ public void testIndexFileCreation() { HdfsState state = createHdfsState(); state.beginCommit(1L); Collection files = FileUtils.listFiles(new File(TEST_OUT_DIR), null, false); - File hdfsIndexFile = Paths.get(TEST_OUT_DIR, INDEX_FILE_PREFIX + TEST_TOPOLOGY_NAME + ".0").toFile(); + File hdfsIndexFile = Paths.get(TEST_OUT_DIR, INDEX_FILE_PREFIX + TEST_TOPOLOGY_NAME + ".0") + .toFile(); assertTrue(files.contains(hdfsIndexFile)); } diff --git a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/trident/format/TestSimpleFileNameFormat.java b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/trident/format/TestSimpleFileNameFormat.java index cd40a278ffe..b30a13827b8 100644 --- a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/trident/format/TestSimpleFileNameFormat.java +++ b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/trident/format/TestSimpleFileNameFormat.java @@ -1,25 +1,31 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.hdfs.trident.format; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + import java.net.UnknownHostException; import java.text.SimpleDateFormat; import org.apache.storm.utils.Utils; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; - public class TestSimpleFileNameFormat { @Test @@ -59,7 +65,8 @@ public void testParameters() { @Test public void testTimeFormat() { - assertThrows(IllegalArgumentException.class, () -> {SimpleFileNameFormat format = new SimpleFileNameFormat() - .withTimeFormat("xyz");}); + assertThrows(IllegalArgumentException.class, () -> { + SimpleFileNameFormat format = new SimpleFileNameFormat() + .withTimeFormat("xyz"); }); } } diff --git a/external/storm-iceberg/src/main/java/org/apache/storm/iceberg/bolt/IcebergBolt.java b/external/storm-iceberg/src/main/java/org/apache/storm/iceberg/bolt/IcebergBolt.java index b72374b3b86..3258ceb6cf4 100644 --- a/external/storm-iceberg/src/main/java/org/apache/storm/iceberg/bolt/IcebergBolt.java +++ b/external/storm-iceberg/src/main/java/org/apache/storm/iceberg/bolt/IcebergBolt.java @@ -74,7 +74,8 @@ public IcebergBolt(IcebergOptions options) { } @Override - public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) { + public void prepare(Map topoConf, TopologyContext context, + OutputCollector collector) { this.collector = collector; this.pending = new ArrayList<>(); this.metrics = new IcebergMetrics(context); @@ -89,7 +90,8 @@ public void prepare(Map topoConf, TopologyContext context, Outpu // new. Those batches were never acked, so the source replays them. int abandoned = committer.recover(); if (abandoned > 0) { - LOG.info("Abandoned {} commit(s) left pending by an earlier run of global task id {} ({}/{})", + LOG.info("Abandoned {} commit(s) left pending by an earlier run of global task id {} " + + "({}/{})", abandoned, taskId, context.getThisComponentId(), context.getThisTaskIndex()); } } diff --git a/external/storm-iceberg/src/main/java/org/apache/storm/iceberg/bolt/IcebergCommitterBolt.java b/external/storm-iceberg/src/main/java/org/apache/storm/iceberg/bolt/IcebergCommitterBolt.java index 7a6317dafb8..a33a1f3d186 100644 --- a/external/storm-iceberg/src/main/java/org/apache/storm/iceberg/bolt/IcebergCommitterBolt.java +++ b/external/storm-iceberg/src/main/java/org/apache/storm/iceberg/bolt/IcebergCommitterBolt.java @@ -75,11 +75,14 @@ public IcebergCommitterBolt(IcebergOptions options) { } @Override - public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) { + public void prepare(Map topoConf, TopologyContext context, + OutputCollector collector) { int committerTasks = context.getComponentTasks(context.getThisComponentId()).size(); if (committerTasks > 1) { - LOG.warn("{} is running with a parallelism of {}: every task commits independently, so the " - + "table receives {} snapshots per interval instead of one. Set its parallelism to 1 " + LOG.warn("{} is running with a parallelism of {}: every task commits independently, " + + "so the " + + "table receives {} snapshots per interval instead of one. Set its " + + "parallelism to 1 " + "and feed it with a globalGrouping.", context.getThisComponentId(), committerTasks, committerTasks); } @@ -114,7 +117,8 @@ protected void process(Tuple tuple) { writer.refreshTable(); } catch (Exception e) { LOG.error("Failed refreshing table metadata for the descriptor from writer task {}, " - + "failing it for replay", tuple.getIntegerByField(IcebergWriterBolt.FIELD_WRITER_TASK_ID), e); + + "failing it for replay", tuple + .getIntegerByField(IcebergWriterBolt.FIELD_WRITER_TASK_ID), e); collector.fail(tuple); return; } @@ -151,7 +155,8 @@ private boolean shouldCommit() { } private long oldestPendingAgeMs() { - return sealed.isEmpty() ? 0L : TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - groupStartNanos); + return sealed.isEmpty() ? 0L : TimeUnit.NANOSECONDS.toMillis(System + .nanoTime() - groupStartNanos); } /** @@ -167,7 +172,8 @@ private void commitGroup() { try { committer.commit(dataFiles); } catch (Exception e) { - LOG.error("Failed committing {} data file(s) from {} writer batch(es), failing them for replay", + LOG.error("Failed committing {} data file(s) from {} writer batch(es), failing them " + + "for replay", dataFiles.size(), committing.size(), e); committing.forEach(collector::fail); return; diff --git a/external/storm-iceberg/src/main/java/org/apache/storm/iceberg/bolt/IcebergWriterBolt.java b/external/storm-iceberg/src/main/java/org/apache/storm/iceberg/bolt/IcebergWriterBolt.java index b0e1a3d7f78..d4ad826e618 100644 --- a/external/storm-iceberg/src/main/java/org/apache/storm/iceberg/bolt/IcebergWriterBolt.java +++ b/external/storm-iceberg/src/main/java/org/apache/storm/iceberg/bolt/IcebergWriterBolt.java @@ -77,7 +77,8 @@ public IcebergWriterBolt(IcebergOptions options) { } @Override - public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) { + public void prepare(Map topoConf, TopologyContext context, + OutputCollector collector) { this.collector = collector; this.pending = new ArrayList<>(); this.metrics = new IcebergMetrics(context); @@ -144,7 +145,8 @@ private void seal() { long startNanos = System.nanoTime(); try { dataFiles = writer.complete(); - descriptor = dataFiles.isEmpty() ? null : DataFileCodec.toJson(dataFiles, writer.table()); + descriptor = dataFiles.isEmpty() ? null : DataFileCodec.toJson(dataFiles, writer + .table()); } catch (Exception e) { LOG.error("Failed sealing {} tuple(s) for Iceberg, failing them for replay", sealing.size(), e); diff --git a/external/storm-iceberg/src/main/java/org/apache/storm/iceberg/common/CommitWal.java b/external/storm-iceberg/src/main/java/org/apache/storm/iceberg/common/CommitWal.java index 461fd004377..65c7407d88c 100644 --- a/external/storm-iceberg/src/main/java/org/apache/storm/iceberg/common/CommitWal.java +++ b/external/storm-iceberg/src/main/java/org/apache/storm/iceberg/common/CommitWal.java @@ -71,7 +71,8 @@ public final class CommitWal { * separates deployments that share one table; without it, each side's startup would clear the * other's entries. */ - public CommitWal(Table table, String namespace, String topologyName, String componentId, int taskIndex) { + public CommitWal(Table table, String namespace, String topologyName, String componentId, + int taskIndex) { this.table = table; this.io = table.io(); String location = table.location(); @@ -101,7 +102,8 @@ public WalEntry write(List dataFiles) { DataFileCodec.writeArray(json, dataFiles, table); json.writeEndObject(); } catch (IOException e) { - throw new UncheckedIOException("Failed writing Iceberg commit WAL entry " + location, e); + throw new UncheckedIOException("Failed writing Iceberg commit WAL entry " + location, + e); } return new WalEntry(commitId, location, createdAtMs); } @@ -119,7 +121,8 @@ public List listPending() { .forEach(fileInfo -> { String location = fileInfo.location(); if (location.endsWith(".json")) { - entries.add(new WalEntry(commitIdOf(location), location, createdAtMsOf(location))); + entries.add(new WalEntry(commitIdOf(location), location, + createdAtMsOf(location))); } }); } catch (UncheckedIOException e) { @@ -141,7 +144,8 @@ public List read(WalEntry entry) { JsonNode root = JsonUtil.mapper().readTree(in); return DataFileCodec.readArray(root.get(DATA_FILES), table.specs()); } catch (IOException e) { - throw new UncheckedIOException("Failed reading Iceberg commit WAL entry " + entry.location(), e); + throw new UncheckedIOException("Failed reading Iceberg commit WAL entry " + entry + .location(), e); } } diff --git a/external/storm-iceberg/src/main/java/org/apache/storm/iceberg/common/FieldNameRecordMapper.java b/external/storm-iceberg/src/main/java/org/apache/storm/iceberg/common/FieldNameRecordMapper.java index 3f85de3707b..ab9f367c986 100644 --- a/external/storm-iceberg/src/main/java/org/apache/storm/iceberg/common/FieldNameRecordMapper.java +++ b/external/storm-iceberg/src/main/java/org/apache/storm/iceberg/common/FieldNameRecordMapper.java @@ -47,7 +47,8 @@ public class FieldNameRecordMapper implements RecordMapper { public Record map(ITuple tuple, Schema schema) { GenericRecord record = GenericRecord.create(schema); for (Types.NestedField field : schema.columns()) { - Object value = tuple.contains(field.name()) ? tuple.getValueByField(field.name()) : null; + Object value = tuple.contains(field.name()) ? tuple.getValueByField(field + .name()) : null; if (value == null) { if (field.isRequired()) { throw new IllegalArgumentException( diff --git a/external/storm-iceberg/src/main/java/org/apache/storm/iceberg/common/IcebergOptions.java b/external/storm-iceberg/src/main/java/org/apache/storm/iceberg/common/IcebergOptions.java index 7eaecc72be7..e38b9ee6b89 100644 --- a/external/storm-iceberg/src/main/java/org/apache/storm/iceberg/common/IcebergOptions.java +++ b/external/storm-iceberg/src/main/java/org/apache/storm/iceberg/common/IcebergOptions.java @@ -289,7 +289,8 @@ public IcebergOptions build() { throw new IllegalStateException( "WAL namespace must be a single non-blank path segment."); } - if (commitIntervalBytes == null && commitIntervalMillis == null && commitIntervalRecords == null) { + if (commitIntervalBytes == null && commitIntervalMillis == null + && commitIntervalRecords == null) { // Without a threshold a batch would stay open until a tick tuple arrived, which // costs unbounded latency on a topology that configured none. this.commitIntervalRecords = DEFAULT_COMMIT_INTERVAL_RECORDS; diff --git a/external/storm-iceberg/src/main/java/org/apache/storm/iceberg/common/IcebergWriter.java b/external/storm-iceberg/src/main/java/org/apache/storm/iceberg/common/IcebergWriter.java index 27f524434b1..0bd7eccf6fc 100644 --- a/external/storm-iceberg/src/main/java/org/apache/storm/iceberg/common/IcebergWriter.java +++ b/external/storm-iceberg/src/main/java/org/apache/storm/iceberg/common/IcebergWriter.java @@ -123,7 +123,8 @@ public List complete() throws IOException { } /** - * Discard what has been written, when the batch never reached {@link #complete()}. Files already + * Discard what has been written, when the batch never reached {@link #complete()}. Files + * already * closed are deleted where possible and remain as orphans where not. */ public void abort() { @@ -132,9 +133,12 @@ public void abort() { } try { writer.abort(); - // BaseTaskWriter.abort() deletes the completed files through Tasks.throwFailureWhenFinished(), - // which propagates whatever FileIO threw — unchecked for HadoopFileIO and S3FileIO alike. - // Callers abort while failing a batch, so an escaping exception would leave those tuples + // BaseTaskWriter.abort() deletes the completed files through + // Tasks.throwFailureWhenFinished(), + // which propagates whatever FileIO threw — unchecked for HadoopFileIO and S3FileIO + // alike. + // Callers abort while failing a batch, so an escaping exception would leave those + // tuples // neither acked nor failed. } catch (IOException | RuntimeException e) { LOG.warn("Failed aborting Iceberg writer; uncommitted files may remain as orphans", e); @@ -182,9 +186,11 @@ private TaskWriter createWriter() { .format(format) .build(); if (spec.isUnpartitioned()) { - return new UnpartitionedWriter<>(spec, format, appenderFactory, fileFactory, table.io(), targetFileSize); + return new UnpartitionedWriter<>(spec, format, appenderFactory, fileFactory, table.io(), + targetFileSize); } - return new PartitionedRecordWriter(spec, format, appenderFactory, fileFactory, table.io(), targetFileSize, schema); + return new PartitionedRecordWriter(spec, format, appenderFactory, fileFactory, table.io(), + targetFileSize, schema); } @Override diff --git a/external/storm-iceberg/src/main/java/org/apache/storm/iceberg/common/PartitionedRecordWriter.java b/external/storm-iceberg/src/main/java/org/apache/storm/iceberg/common/PartitionedRecordWriter.java index 56e1fda8c8b..aa181c37698 100644 --- a/external/storm-iceberg/src/main/java/org/apache/storm/iceberg/common/PartitionedRecordWriter.java +++ b/external/storm-iceberg/src/main/java/org/apache/storm/iceberg/common/PartitionedRecordWriter.java @@ -37,7 +37,8 @@ class PartitionedRecordWriter extends PartitionedFanoutWriter { private final PartitionKey partitionKey; private final InternalRecordWrapper wrapper; - PartitionedRecordWriter(PartitionSpec spec, FileFormat format, FileAppenderFactory appenderFactory, + PartitionedRecordWriter(PartitionSpec spec, FileFormat format, + FileAppenderFactory appenderFactory, OutputFileFactory fileFactory, FileIO io, long targetFileSize, Schema schema) { super(spec, format, appenderFactory, fileFactory, io, targetFileSize); this.partitionKey = new PartitionKey(spec, schema); diff --git a/external/storm-iceberg/src/test/java/org/apache/storm/iceberg/bolt/IcebergCommitterBoltTest.java b/external/storm-iceberg/src/test/java/org/apache/storm/iceberg/bolt/IcebergCommitterBoltTest.java index 627683f2825..9b579dc395d 100644 --- a/external/storm-iceberg/src/test/java/org/apache/storm/iceberg/bolt/IcebergCommitterBoltTest.java +++ b/external/storm-iceberg/src/test/java/org/apache/storm/iceberg/bolt/IcebergCommitterBoltTest.java @@ -119,7 +119,8 @@ private DataFile dataFile(String name) { private Tuple descriptor(int writerTaskId, String fileName) { Tuple tuple = mock(Tuple.class); when(tuple.getSourceComponent()).thenReturn("iceberg-writer"); - when(tuple.getIntegerByField(IcebergWriterBolt.FIELD_WRITER_TASK_ID)).thenReturn(writerTaskId); + when(tuple.getIntegerByField(IcebergWriterBolt.FIELD_WRITER_TASK_ID)) + .thenReturn(writerTaskId); when(tuple.getStringByField(IcebergWriterBolt.FIELD_DATA_FILES)) .thenReturn(DataFileCodec.toJson(List.of(dataFile(fileName)), table)); return tuple; @@ -209,7 +210,8 @@ void aFailedCommitFailsEveryAccumulatedDescriptor() { bolt.execute(second); // The table disappears underneath the committer, so the append cannot land. A tick tuple // forces commitGroup() directly, without an intervening refresh that would instead fail - // just the tuple that triggered it (see anUnreadableDescriptorIsFailedWithoutPoisoningTheGroup). + // just the tuple that triggered it (see + // anUnreadableDescriptorIsFailedWithoutPoisoningTheGroup). verifyCatalog.dropTable(TABLE_ID, true); bolt.execute(tickTuple()); diff --git a/external/storm-iceberg/src/test/java/org/apache/storm/iceberg/common/CommitWalTest.java b/external/storm-iceberg/src/test/java/org/apache/storm/iceberg/common/CommitWalTest.java index 374c586cc80..2b87df58c2f 100644 --- a/external/storm-iceberg/src/test/java/org/apache/storm/iceberg/common/CommitWalTest.java +++ b/external/storm-iceberg/src/test/java/org/apache/storm/iceberg/common/CommitWalTest.java @@ -115,14 +115,16 @@ void pendingEntryReadsBackTheDataFilesItWasWritten() { void entriesAreIsolatedByComponentAndTaskIndex() { CommitWal mine = new CommitWal(table, null, "topo", "iceberg-committer", 0); CommitWal other = new CommitWal(table, null, "topo", "iceberg-writer", 0); - CommitWal sameComponentOtherIndex = new CommitWal(table, null, "topo", "iceberg-committer", 1); + CommitWal sameComponentOtherIndex = new CommitWal(table, null, "topo", "iceberg-committer", + 1); CommitWal.WalEntry entry = mine.write(List.of(dataFile("a.parquet", 1L))); assertEquals(1, mine.listPending().size()); assertEquals(entry.commitId(), mine.listPending().get(0).commitId()); assertEquals(List.of(), other.listPending(), "another component sees nothing"); - assertEquals(List.of(), sameComponentOtherIndex.listPending(), "another task index sees nothing"); + assertEquals(List.of(), sameComponentOtherIndex.listPending(), + "another task index sees nothing"); assertTrue(entry.location().contains("/iceberg-committer/0/"), "WAL path is scoped by component and task index"); } diff --git a/external/storm-iceberg/src/test/java/org/apache/storm/iceberg/common/DataFileCodecTest.java b/external/storm-iceberg/src/test/java/org/apache/storm/iceberg/common/DataFileCodecTest.java index 6e565351ba5..505b28cc2bc 100644 --- a/external/storm-iceberg/src/test/java/org/apache/storm/iceberg/common/DataFileCodecTest.java +++ b/external/storm-iceberg/src/test/java/org/apache/storm/iceberg/common/DataFileCodecTest.java @@ -84,7 +84,8 @@ void unpartitionedFilesRoundTripWithTheirCounts() { DataFile second = unpartitionedFile("b.parquet", 7L); List recovered = - DataFileCodec.fromJson(DataFileCodec.toJson(List.of(first, second), table), table.specs()); + DataFileCodec.fromJson(DataFileCodec.toJson(List.of(first, second), table), table + .specs()); assertEquals(2, recovered.size()); assertEquals(first.location(), recovered.get(0).location()); diff --git a/external/storm-iceberg/src/test/java/org/apache/storm/iceberg/common/FieldNameRecordMapperTest.java b/external/storm-iceberg/src/test/java/org/apache/storm/iceberg/common/FieldNameRecordMapperTest.java index 376f8368a72..82b641fcba4 100644 --- a/external/storm-iceberg/src/test/java/org/apache/storm/iceberg/common/FieldNameRecordMapperTest.java +++ b/external/storm-iceberg/src/test/java/org/apache/storm/iceberg/common/FieldNameRecordMapperTest.java @@ -118,7 +118,8 @@ void missingRequiredFieldThrows() { field(tuple, "id", 1L); // "name" (required) absent from the tuple - IllegalArgumentException e = assertThrows(IllegalArgumentException.class, () -> mapper.map(tuple, SCHEMA)); + IllegalArgumentException e = assertThrows(IllegalArgumentException.class, () -> mapper + .map(tuple, SCHEMA)); assertTrue(e.getMessage().contains("name")); } diff --git a/external/storm-iceberg/src/test/java/org/apache/storm/iceberg/common/IcebergCommitterTest.java b/external/storm-iceberg/src/test/java/org/apache/storm/iceberg/common/IcebergCommitterTest.java index 95032308b42..e97c68fc8f2 100644 --- a/external/storm-iceberg/src/test/java/org/apache/storm/iceberg/common/IcebergCommitterTest.java +++ b/external/storm-iceberg/src/test/java/org/apache/storm/iceberg/common/IcebergCommitterTest.java @@ -130,7 +130,8 @@ void theScanWindowStartsBeforeTheEntryByTheClockSkewAllowance() { // older than the entry — beyond what clock skew between hosts can explain — cannot be it. assertTrue(IcebergCommitter.withinScanWindow(entryCreatedAtMs + 1, entryCreatedAtMs)); assertTrue(IcebergCommitter.withinScanWindow(entryCreatedAtMs - slack, entryCreatedAtMs)); - assertFalse(IcebergCommitter.withinScanWindow(entryCreatedAtMs - slack - 1, entryCreatedAtMs)); + assertFalse(IcebergCommitter.withinScanWindow(entryCreatedAtMs - slack - 1, + entryCreatedAtMs)); } /** @@ -223,7 +224,8 @@ void aLandedCommitSurvivesAFailureToDeleteItsWalEntry() { CommitWal flakyWal = spy(wal); doThrow(new RuntimeIOException(new IOException("metadata store unavailable"))) .when(flakyWal).delete(any()); - IcebergCommitter committer = new IcebergCommitter(table, flakyWal, new IcebergMetrics(null)); + IcebergCommitter committer = new IcebergCommitter(table, flakyWal, + new IcebergMetrics(null)); committer.commit(List.of(dataFile("a.parquet"))); diff --git a/external/storm-iceberg/src/test/java/org/apache/storm/iceberg/common/IcebergOptionsTest.java b/external/storm-iceberg/src/test/java/org/apache/storm/iceberg/common/IcebergOptionsTest.java index 840533f53d1..9c7d2eaff3c 100644 --- a/external/storm-iceberg/src/test/java/org/apache/storm/iceberg/common/IcebergOptionsTest.java +++ b/external/storm-iceberg/src/test/java/org/apache/storm/iceberg/common/IcebergOptionsTest.java @@ -35,7 +35,8 @@ class IcebergOptionsTest { - private static final Map CATALOG_PROPS = Map.of("type", "hadoop", "warehouse", "file:///tmp/wh"); + private static final Map CATALOG_PROPS = Map.of("type", "hadoop", "warehouse", + "file:///tmp/wh"); private IcebergOptions.Builder validBuilder() { return new IcebergOptions.Builder() @@ -64,13 +65,15 @@ void rejectsMissingCatalogProperties() { @Test void rejectsMissingTable() { - IcebergOptions.Builder builder = new IcebergOptions.Builder().withCatalogProperties(CATALOG_PROPS); + IcebergOptions.Builder builder = new IcebergOptions.Builder() + .withCatalogProperties(CATALOG_PROPS); assertThrows(IllegalStateException.class, builder::build); } @Test void rejectsNonPositiveTargetFileSize() { - assertThrows(IllegalStateException.class, () -> validBuilder().withTargetFileSizeBytes(0).build()); + assertThrows(IllegalStateException.class, () -> validBuilder().withTargetFileSizeBytes(0) + .build()); } @Test @@ -122,9 +125,11 @@ void nonPositiveGroupCommitOptionsAreRejected() { @Test void aWalNamespaceMustBeOnePathSegment() { - assertEquals("staging", validBuilder().withWalNamespace("staging").build().getWalNamespace()); + assertEquals("staging", validBuilder().withWalNamespace("staging").build() + .getWalNamespace()); assertNull(validBuilder().build().getWalNamespace(), "unset unless asked for"); - // The namespace is interpolated into the WAL path, so a separator would silently reshape it. + // The namespace is interpolated into the WAL path, so a separator would silently reshape + // it. assertThrows(IllegalStateException.class, () -> validBuilder().withWalNamespace("a/b").build()); assertThrows(IllegalStateException.class, diff --git a/external/storm-iceberg/src/test/java/org/apache/storm/iceberg/common/IcebergWriterTest.java b/external/storm-iceberg/src/test/java/org/apache/storm/iceberg/common/IcebergWriterTest.java index 78448d89145..a9ac104fc1f 100644 --- a/external/storm-iceberg/src/test/java/org/apache/storm/iceberg/common/IcebergWriterTest.java +++ b/external/storm-iceberg/src/test/java/org/apache/storm/iceberg/common/IcebergWriterTest.java @@ -44,8 +44,8 @@ import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.data.IcebergGenerics; import org.apache.iceberg.data.Record; -import org.apache.iceberg.hadoop.HadoopCatalog; import org.apache.iceberg.exceptions.RuntimeIOException; +import org.apache.iceberg.hadoop.HadoopCatalog; import org.apache.iceberg.io.CloseableIterable; import org.apache.iceberg.io.FileIO; import org.apache.iceberg.io.InputFile; @@ -174,11 +174,15 @@ void bufferedBytesGrowWithWritesAndResetOnComplete() throws IOException { assertEquals(0L, writer.bufferedBytes(), "completing starts a fresh buffer"); } } + @Test void abortSwallowsAFileIoFailureSoTheBatchCanStillBeFailed() throws IOException { - // BaseTaskWriter.abort() deletes its completed files through Tasks.throwFailureWhenFinished(), - // and FileIO reports failures unchecked. Callers abort while failing a batch, so an escaping - // exception would leave those tuples neither acked nor failed, and only replayed on timeout. + // BaseTaskWriter.abort() deletes its completed files through + // Tasks.throwFailureWhenFinished(), + // and FileIO reports failures unchecked. Callers abort while failing a batch, so an + // escaping + // exception would leave those tuples neither acked nor failed, and only replayed on + // timeout. verifyCatalog.createTable(TABLE_ID, SCHEMA, PartitionSpec.unpartitioned()); Map catalogProps = new HashMap<>(); catalogProps.put(CatalogProperties.CATALOG_IMPL, FailingDeleteCatalog.class.getName()); diff --git a/external/storm-iceberg/src/test/java/org/apache/storm/iceberg/common/RecordingMetricsContext.java b/external/storm-iceberg/src/test/java/org/apache/storm/iceberg/common/RecordingMetricsContext.java index 4a391f1187a..a1df11b084b 100644 --- a/external/storm-iceberg/src/test/java/org/apache/storm/iceberg/common/RecordingMetricsContext.java +++ b/external/storm-iceberg/src/test/java/org/apache/storm/iceberg/common/RecordingMetricsContext.java @@ -103,7 +103,8 @@ public ReducedMetric registerMetric(String name, IReducer reducer, int timeBucke @Override @Deprecated - public CombinedMetric registerMetric(String name, ICombiner combiner, int timeBucketSizeInSecs) { + public CombinedMetric registerMetric(String name, ICombiner combiner, + int timeBucketSizeInSecs) { throw new UnsupportedOperationException("not used by IcebergState"); } } diff --git a/external/storm-jdbc/pom.xml b/external/storm-jdbc/pom.xml index 770754154c1..4816832ef0a 100644 --- a/external/storm-jdbc/pom.xml +++ b/external/storm-jdbc/pom.xml @@ -77,6 +77,16 @@ + + org.openrewrite.maven + rewrite-maven-plugin + + + + org.codehaus.gmavenplus + gmavenplus-plugin + + org.apache.maven.plugins maven-checkstyle-plugin diff --git a/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/bolt/AbstractJdbcBolt.java b/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/bolt/AbstractJdbcBolt.java index 5368cbefa10..ca68988dda1 100644 --- a/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/bolt/AbstractJdbcBolt.java +++ b/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/bolt/AbstractJdbcBolt.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -54,6 +60,7 @@ public abstract class AbstractJdbcBolt extends BaseTickTupleAwareRichBolt { /** * Constructor. *

+ * * @param connectionProviderParam database connection provider */ public AbstractJdbcBolt(final ConnectionProvider connectionProviderParam) { diff --git a/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/bolt/JdbcInsertBolt.java b/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/bolt/JdbcInsertBolt.java index 16317370871..463d45458e4 100644 --- a/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/bolt/JdbcInsertBolt.java +++ b/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/bolt/JdbcInsertBolt.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -45,7 +51,8 @@ public JdbcInsertBolt(ConnectionProvider connectionProvider, JdbcMapper jdbcMapp public JdbcInsertBolt withTableName(String tableName) { if (insertQuery != null) { - throw new IllegalArgumentException("You can not specify both insertQuery and tableName."); + throw new IllegalArgumentException("You can not specify both insertQuery and " + + "tableName."); } this.tableName = tableName; return this; @@ -53,7 +60,8 @@ public JdbcInsertBolt withTableName(String tableName) { public JdbcInsertBolt withInsertQuery(String insertQuery) { if (this.tableName != null) { - throw new IllegalArgumentException("You can not specify both insertQuery and tableName."); + throw new IllegalArgumentException("You can not specify both insertQuery and " + + "tableName."); } this.insertQuery = insertQuery; return this; @@ -65,10 +73,12 @@ public JdbcInsertBolt withQueryTimeoutSecs(int queryTimeoutSecs) { } @Override - public void prepare(Map map, TopologyContext topologyContext, OutputCollector collector) { + public void prepare(Map map, TopologyContext topologyContext, + OutputCollector collector) { super.prepare(map, topologyContext, collector); if (StringUtils.isBlank(tableName) && StringUtils.isBlank(insertQuery)) { - throw new IllegalArgumentException("You must supply either a tableName or an insert Query."); + throw new IllegalArgumentException("You must supply either a tableName or an insert " + + "Query."); } } diff --git a/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/bolt/JdbcLookupBolt.java b/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/bolt/JdbcLookupBolt.java index ae0610c8435..9989373879a 100644 --- a/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/bolt/JdbcLookupBolt.java +++ b/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/bolt/JdbcLookupBolt.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -33,7 +39,8 @@ public class JdbcLookupBolt extends AbstractJdbcBolt { private JdbcLookupMapper jdbcLookupMapper; - public JdbcLookupBolt(ConnectionProvider connectionProvider, String selectQuery, JdbcLookupMapper jdbcLookupMapper) { + public JdbcLookupBolt(ConnectionProvider connectionProvider, String selectQuery, + JdbcLookupMapper jdbcLookupMapper) { super(connectionProvider); Validate.notNull(selectQuery); diff --git a/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/common/Column.java b/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/common/Column.java index f5f08e12d1c..b75c68acaee 100644 --- a/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/common/Column.java +++ b/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/common/Column.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -15,8 +21,10 @@ import java.io.Serializable; /** - * A database table can be defined as a list of rows and each row can be defined as a list of columns where - * each column instance has a name, a value and a type. This class represents an instance of a column in a database + * A database table can be defined as a list of rows and each row can be defined as a list of + * columns where + * each column instance has a name, a value and a type. This class represents an instance of a + * column in a database * row. For example if we have the following table named user: *

  *  ____________________________
@@ -43,7 +51,8 @@ public class Column implements Serializable {
     private T val;
 
     /**
-     * The sql type(e.g. varchar, date, int) Ideally we would have an enum but java's jdbc API uses integer.
+     * The sql type(e.g. varchar, date, int) Ideally we would have an enum but java's jdbc API uses
+     * integer.
      * See {@link java.sql.Types}
      */
     private int sqlType;
diff --git a/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/common/ConnectionProvider.java b/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/common/ConnectionProvider.java
index 39b3ddfc74e..b6a51d044f4 100644
--- a/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/common/ConnectionProvider.java
+++ b/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/common/ConnectionProvider.java
@@ -1,12 +1,18 @@
 /**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.  The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License.  You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
  *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * 

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -20,18 +26,19 @@ */ public interface ConnectionProvider extends Serializable { /** - * method must be idempotent. + * Method must be idempotent. */ void prepare(); /** * Get connection. + * * @return a DB connection over which the queries can be executed. */ Connection getConnection(); /** - * called once when the system is shutting down, should be idempotent. + * Called once when the system is shutting down, should be idempotent. */ void cleanup(); } diff --git a/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/common/HikariCPConnectionProvider.java b/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/common/HikariCPConnectionProvider.java index ad1b80d0271..f08334bda71 100644 --- a/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/common/HikariCPConnectionProvider.java +++ b/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/common/HikariCPConnectionProvider.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/common/JdbcClient.java b/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/common/JdbcClient.java index 29cc85cb5d8..39a9a40ebe9 100644 --- a/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/common/JdbcClient.java +++ b/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/common/JdbcClient.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -75,7 +81,8 @@ public void executeInsertQuery(String query, List> columnLists) { int[] results = preparedStatement.executeBatch(); if (Arrays.asList(results).contains(Statement.EXECUTE_FAILED)) { connection.rollback(); - throw new RuntimeException("failed at least one sql statement in the batch, operation rolled back."); + throw new RuntimeException("failed at least one sql statement in the batch, " + + "operation rolled back."); } else { try { connection.commit(); @@ -96,12 +103,13 @@ public void executeInsertQuery(String query, List> columnLists) { private String constructInsertQuery(String tableName, List> columnLists) { StringBuilder sb = new StringBuilder(); sb.append("Insert into ").append(tableName).append(" ("); - Collection columnNames = Collections2.transform(columnLists.get(0), new Function() { - @Override + Collection columnNames = Collections2.transform(columnLists.get(0), + new Function() { + @Override public String apply(Column input) { - return input.getColumnName(); - } - }); + return input.getColumnName(); + } + }); String columns = Joiner.on(",").join(columnNames); sb.append(columns).append(") values ( "); @@ -132,29 +140,41 @@ public List> select(String sqlQuery, List queryParams) { int columnType = metaData.getColumnType(i); Class columnJavaType = Util.getJavaType(columnType); if (columnJavaType.equals(String.class)) { - row.add(new Column(columnLabel, resultSet.getString(columnLabel), columnType)); + row.add(new Column(columnLabel, resultSet + .getString(columnLabel), columnType)); } else if (columnJavaType.equals(Integer.class)) { - row.add(new Column(columnLabel, resultSet.getInt(columnLabel), columnType)); + row.add(new Column(columnLabel, resultSet + .getInt(columnLabel), columnType)); } else if (columnJavaType.equals(Double.class)) { - row.add(new Column(columnLabel, resultSet.getDouble(columnLabel), columnType)); + row.add(new Column(columnLabel, resultSet + .getDouble(columnLabel), columnType)); } else if (columnJavaType.equals(Float.class)) { - row.add(new Column(columnLabel, resultSet.getFloat(columnLabel), columnType)); + row.add(new Column(columnLabel, resultSet + .getFloat(columnLabel), columnType)); } else if (columnJavaType.equals(Short.class)) { - row.add(new Column(columnLabel, resultSet.getShort(columnLabel), columnType)); + row.add(new Column(columnLabel, resultSet + .getShort(columnLabel), columnType)); } else if (columnJavaType.equals(Boolean.class)) { - row.add(new Column(columnLabel, resultSet.getBoolean(columnLabel), columnType)); + row.add(new Column(columnLabel, resultSet + .getBoolean(columnLabel), columnType)); } else if (columnJavaType.equals(byte[].class)) { - row.add(new Column(columnLabel, resultSet.getBytes(columnLabel), columnType)); + row.add(new Column(columnLabel, resultSet + .getBytes(columnLabel), columnType)); } else if (columnJavaType.equals(Long.class)) { - row.add(new Column(columnLabel, resultSet.getLong(columnLabel), columnType)); + row.add(new Column(columnLabel, resultSet + .getLong(columnLabel), columnType)); } else if (columnJavaType.equals(Date.class)) { - row.add(new Column(columnLabel, resultSet.getDate(columnLabel), columnType)); + row.add(new Column(columnLabel, resultSet + .getDate(columnLabel), columnType)); } else if (columnJavaType.equals(Time.class)) { - row.add(new Column

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -64,7 +70,8 @@ public static Class getJavaType(int sqlType) { case Types.TIMESTAMP: return Timestamp.class; default: - throw new RuntimeException("We do not support tables with SqlType: " + getSqlTypeName(sqlType)); + throw new RuntimeException("We do not support tables with SqlType: " + + getSqlTypeName(sqlType)); } } } diff --git a/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/mapper/JdbcLookupMapper.java b/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/mapper/JdbcLookupMapper.java index 27368140f5e..15789a74bc2 100644 --- a/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/mapper/JdbcLookupMapper.java +++ b/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/mapper/JdbcLookupMapper.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -21,16 +27,19 @@ public interface JdbcLookupMapper extends JdbcMapper { /** - * Converts a DB row to a list of storm values that can be emitted. This is done to allow a single + * Converts a DB row to a list of storm values that can be emitted. This is done to allow a + * single * storm input tuple and a single DB row to result in multiple output values. + * * @param input the input tuple. * @param columns list of columns that represents a row - * @return a List of storm values that can be emitted. Each item in list is emitted as an output tuple. + * @return a List of storm values that can be emitted. Each item in list is emitted as an output + * tuple. */ List toTuple(ITuple input, List columns); /** - * declare what are the fields that this code will output. + * Declare what are the fields that this code will output. */ void declareOutputFields(OutputFieldsDeclarer declarer); } diff --git a/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/mapper/JdbcMapper.java b/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/mapper/JdbcMapper.java index b3f55007cf9..8d59e1bac4d 100644 --- a/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/mapper/JdbcMapper.java +++ b/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/mapper/JdbcMapper.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -20,6 +26,7 @@ public interface JdbcMapper extends Serializable { /** * Get columns. + * * @return list of columns that represents one row in a DB table. */ List getColumns(ITuple tuple); diff --git a/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/mapper/SimpleJdbcLookupMapper.java b/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/mapper/SimpleJdbcLookupMapper.java index f2987bc5bf1..d28642983c3 100644 --- a/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/mapper/SimpleJdbcLookupMapper.java +++ b/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/mapper/SimpleJdbcLookupMapper.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/mapper/SimpleJdbcMapper.java b/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/mapper/SimpleJdbcMapper.java index 55edd60d46a..67f2c1c4e6e 100644 --- a/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/mapper/SimpleJdbcMapper.java +++ b/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/mapper/SimpleJdbcMapper.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -84,7 +90,8 @@ public List getColumns(ITuple tuple) { Long value = tuple.getLongByField(columnName); columns.add(new Column(columnName, new Timestamp(value), columnSqlType)); } else { - throw new RuntimeException("Unsupported java type in tuple " + Util.getJavaType(columnSqlType)); + throw new RuntimeException("Unsupported java type in tuple " + Util + .getJavaType(columnSqlType)); } } return columns; diff --git a/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/trident/state/JdbcQuery.java b/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/trident/state/JdbcQuery.java index f6623a05adc..70364ec6ce0 100644 --- a/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/trident/state/JdbcQuery.java +++ b/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/trident/state/JdbcQuery.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -26,7 +32,8 @@ public List> batchRetrieve(JdbcState jdbcState, List } @Override - public void execute(TridentTuple tuples, List values, TridentCollector tridentCollector) { + public void execute(TridentTuple tuples, List values, + TridentCollector tridentCollector) { for (Values value : values) { tridentCollector.emit(value); } diff --git a/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/trident/state/JdbcState.java b/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/trident/state/JdbcState.java index 8c3d11a89b2..21eba33a21e 100644 --- a/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/trident/state/JdbcState.java +++ b/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/trident/state/JdbcState.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -40,7 +46,8 @@ public class JdbcState implements State { private JdbcClient jdbcClient; private Map map; - protected JdbcState(Map map, int partitionIndex, int numPartitions, Options options) { + protected JdbcState(Map map, int partitionIndex, int numPartitions, + Options options) { this.options = options; this.map = map; } @@ -51,12 +58,15 @@ protected void prepare() { if (StringUtils.isBlank(options.insertQuery) && StringUtils.isBlank(options.tableName) && StringUtils.isBlank(options.selectQuery)) { - throw new IllegalArgumentException("If you are trying to insert into DB you must supply either insertQuery " - + "or tableName. If you are attempting to user a query state you must supply a select query."); + throw new IllegalArgumentException("If you are trying to insert into DB you must " + + "supply either insertQuery " + + "or tableName. If you are attempting to user a query state you must supply " + + "a select query."); } if (options.queryTimeoutSecs == null) { - options.queryTimeoutSecs = Integer.parseInt(map.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS).toString()); + options.queryTimeoutSecs = Integer.parseInt(map + .get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS).toString()); } this.jdbcClient = new JdbcClient(options.connectionProvider, options.queryTimeoutSecs); @@ -86,7 +96,8 @@ public void updateState(List tuples, TridentCollector collector) { jdbcClient.executeInsertQuery(options.insertQuery, columnsLists); } } catch (Exception e) { - LOG.warn("Batch write failed but some requests might have succeeded. Triggering replay.", e); + LOG.warn("Batch write failed but some requests might have succeeded. Triggering " + + "replay.", e); throw new FailedException(e); } } diff --git a/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/trident/state/JdbcStateFactory.java b/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/trident/state/JdbcStateFactory.java index 7ed334ec271..539af25cc1e 100644 --- a/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/trident/state/JdbcStateFactory.java +++ b/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/trident/state/JdbcStateFactory.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -26,7 +32,8 @@ public JdbcStateFactory(JdbcState.Options options) { } @Override - public State makeState(Map map, IMetricsContext metricsContext, int partitionIndex, int numPartitions) { + public State makeState(Map map, IMetricsContext metricsContext, + int partitionIndex, int numPartitions) { JdbcState state = new JdbcState(map, partitionIndex, numPartitions, options); state.prepare(); return state; diff --git a/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/trident/state/JdbcUpdater.java b/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/trident/state/JdbcUpdater.java index 47c01ba471f..288d5aaa557 100644 --- a/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/trident/state/JdbcUpdater.java +++ b/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/trident/state/JdbcUpdater.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -20,7 +26,8 @@ public class JdbcUpdater extends BaseStateUpdater { @Override - public void updateState(JdbcState jdbcState, List tuples, TridentCollector collector) { + public void updateState(JdbcState jdbcState, List tuples, + TridentCollector collector) { jdbcState.updateState(tuples, collector); } } diff --git a/external/storm-jdbc/src/test/java/org/apache/storm/jdbc/bolt/JdbcInsertBoltTest.java b/external/storm-jdbc/src/test/java/org/apache/storm/jdbc/bolt/JdbcInsertBoltTest.java index d421b0b9872..da33bd0c3b1 100644 --- a/external/storm-jdbc/src/test/java/org/apache/storm/jdbc/bolt/JdbcInsertBoltTest.java +++ b/external/storm-jdbc/src/test/java/org/apache/storm/jdbc/bolt/JdbcInsertBoltTest.java @@ -1,17 +1,25 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.jdbc.bolt; +import static org.junit.jupiter.api.Assertions.assertThrows; + import com.google.common.collect.Lists; import java.util.HashMap; import org.apache.storm.jdbc.common.Column; @@ -21,8 +29,6 @@ import org.apache.storm.jdbc.mapper.SimpleJdbcMapper; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertThrows; - /** * Created by pbrahmbhatt on 10/29/15. */ diff --git a/external/storm-jdbc/src/test/java/org/apache/storm/jdbc/bolt/JdbcLookupBoltTest.java b/external/storm-jdbc/src/test/java/org/apache/storm/jdbc/bolt/JdbcLookupBoltTest.java index 3185f62f699..8e3064c119b 100644 --- a/external/storm-jdbc/src/test/java/org/apache/storm/jdbc/bolt/JdbcLookupBoltTest.java +++ b/external/storm-jdbc/src/test/java/org/apache/storm/jdbc/bolt/JdbcLookupBoltTest.java @@ -1,17 +1,25 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.jdbc.bolt; +import static org.junit.jupiter.api.Assertions.assertThrows; + import com.google.common.collect.Lists; import java.util.HashMap; import org.apache.storm.jdbc.common.Column; @@ -22,8 +30,6 @@ import org.apache.storm.tuple.Fields; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertThrows; - /** * Created by pbrahmbhatt on 10/29/15. */ @@ -32,15 +38,18 @@ public class JdbcLookupBoltTest { @Test public void testValidation() { ConnectionProvider provider = new HikariCPConnectionProvider(new HashMap<>()); - JdbcLookupMapper mapper = new SimpleJdbcLookupMapper(new Fields("test"), Lists.newArrayList(new Column("test", 0))); + JdbcLookupMapper mapper = new SimpleJdbcLookupMapper(new Fields("test"), Lists + .newArrayList(new Column("test", 0))); String selectQuery = "select * from dual"; expectNullPointerException(null, selectQuery, mapper); expectNullPointerException(provider, null, mapper); expectNullPointerException(provider, selectQuery, null); } - private void expectNullPointerException(ConnectionProvider provider, String selectQuery, JdbcLookupMapper mapper) { - assertThrows(NullPointerException.class, () -> new JdbcLookupBolt(provider, selectQuery, mapper)); + private void expectNullPointerException(ConnectionProvider provider, String selectQuery, + JdbcLookupMapper mapper) { + assertThrows(NullPointerException.class, () -> new JdbcLookupBolt(provider, selectQuery, + mapper)); } } diff --git a/external/storm-jdbc/src/test/java/org/apache/storm/jdbc/common/JdbcClientTest.java b/external/storm-jdbc/src/test/java/org/apache/storm/jdbc/common/JdbcClientTest.java index 0304c4c018b..5d4a5f542ec 100644 --- a/external/storm-jdbc/src/test/java/org/apache/storm/jdbc/common/JdbcClientTest.java +++ b/external/storm-jdbc/src/test/java/org/apache/storm/jdbc/common/JdbcClientTest.java @@ -1,17 +1,26 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.jdbc.common; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + import com.google.common.collect.Lists; import com.google.common.collect.Maps; import java.sql.Connection; @@ -24,9 +33,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; - public class JdbcClientTest { private static final String tableName = "user_details"; @@ -35,16 +41,18 @@ public class JdbcClientTest { @BeforeEach public void setup() { Map map = Maps.newHashMap(); - map.put("dataSourceClassName", "org.hsqldb.jdbc.JDBCDataSource");//com.mysql.jdbc.jdbc2.optional.MysqlDataSource - map.put("dataSource.url", "jdbc:hsqldb:mem:test");//jdbc:mysql://localhost/test - map.put("dataSource.user", "SA");//root - map.put("dataSource.password", "");//password + map.put("dataSourceClassName", + "org.hsqldb.jdbc.JDBCDataSource"); // com.mysql.jdbc.jdbc2.optional.MysqlDataSource + map.put("dataSource.url", "jdbc:hsqldb:mem:test"); // jdbc:mysql://localhost/test + map.put("dataSource.user", "SA"); // root + map.put("dataSource.password", ""); // password ConnectionProvider connectionProvider = new HikariCPConnectionProvider(map); connectionProvider.prepare(); int queryTimeoutSecs = 60; this.client = new JdbcClient(connectionProvider, queryTimeoutSecs); - client.executeSql("create table user_details (id integer, user_name varchar(100), created_timestamp TIMESTAMP)"); + client.executeSql("create table user_details (id integer, user_name varchar(100), " + + "created_timestamp TIMESTAMP)"); } @Test @@ -57,7 +65,8 @@ public void testInsertAndSelect() { client.insert(tableName, rows); List> selectedRows = - client.select("select * from user_details where id = ?", Lists.newArrayList(new Column("id", 1, Types.INTEGER))); + client.select("select * from user_details where id = ?", Lists + .newArrayList(new Column("id", 1, Types.INTEGER))); List> expectedRows = Lists.newArrayList(); expectedRows.add(row1); assertEquals(expectedRows, selectedRows); @@ -67,13 +76,15 @@ public void testInsertAndSelect() { moreRows.add(row3); client.executeInsertQuery("insert into user_details values(?,?,?)", moreRows); - selectedRows = client.select("select * from user_details where id = ?", Lists.newArrayList(new Column("id", 3, Types.INTEGER))); + selectedRows = client.select("select * from user_details where id = ?", Lists + .newArrayList(new Column("id", 3, Types.INTEGER))); expectedRows = Lists.newArrayList(); expectedRows.add(row3); assertEquals(expectedRows, selectedRows); - selectedRows = client.select("select * from user_details order by id", Lists.newArrayList()); + selectedRows = client.select("select * from user_details order by id", + Lists.newArrayList()); rows.add(row3); assertEquals(rows, selectedRows); client.executeSql("drop table " + tableName); @@ -98,7 +109,8 @@ private List createRow(int id, String name) { return Lists.newArrayList( new Column<>("ID", id, Types.INTEGER), new Column<>("USER_NAME", name, Types.VARCHAR), - new Column<>("CREATED_TIMESTAMP", new Timestamp(System.currentTimeMillis()), Types.TIMESTAMP)); + new Column<>("CREATED_TIMESTAMP", new Timestamp(System.currentTimeMillis()), + Types.TIMESTAMP)); } } diff --git a/external/storm-jdbc/src/test/java/org/apache/storm/jdbc/common/UtilTest.java b/external/storm-jdbc/src/test/java/org/apache/storm/jdbc/common/UtilTest.java index 19f578bd24f..e489b922751 100644 --- a/external/storm-jdbc/src/test/java/org/apache/storm/jdbc/common/UtilTest.java +++ b/external/storm-jdbc/src/test/java/org/apache/storm/jdbc/common/UtilTest.java @@ -1,26 +1,32 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.jdbc.common; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + import java.sql.Date; import java.sql.Time; import java.sql.Timestamp; import java.sql.Types; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; - public class UtilTest { @Test diff --git a/external/storm-jms/pom.xml b/external/storm-jms/pom.xml index 5aa3fc2abdc..dfd61417e01 100644 --- a/external/storm-jms/pom.xml +++ b/external/storm-jms/pom.xml @@ -80,6 +80,16 @@ -Xdoclint:none + + org.openrewrite.maven + rewrite-maven-plugin + + + + org.codehaus.gmavenplus + gmavenplus-plugin + + org.apache.maven.plugins maven-checkstyle-plugin diff --git a/external/storm-jms/src/main/java/org/apache/storm/jms/JmsMessageProducer.java b/external/storm-jms/src/main/java/org/apache/storm/jms/JmsMessageProducer.java index 8b9479d0f3b..1164bdab7f5 100644 --- a/external/storm-jms/src/main/java/org/apache/storm/jms/JmsMessageProducer.java +++ b/external/storm-jms/src/main/java/org/apache/storm/jms/JmsMessageProducer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -16,7 +22,6 @@ import jakarta.jms.Message; import jakarta.jms.Session; import java.io.Serializable; - import org.apache.storm.tuple.ITuple; /** diff --git a/external/storm-jms/src/main/java/org/apache/storm/jms/JmsProvider.java b/external/storm-jms/src/main/java/org/apache/storm/jms/JmsProvider.java index 34b8db39f0e..f34e1080a33 100644 --- a/external/storm-jms/src/main/java/org/apache/storm/jms/JmsProvider.java +++ b/external/storm-jms/src/main/java/org/apache/storm/jms/JmsProvider.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -14,7 +20,6 @@ import jakarta.jms.ConnectionFactory; import jakarta.jms.Destination; - import java.io.Serializable; /** diff --git a/external/storm-jms/src/main/java/org/apache/storm/jms/JmsTupleProducer.java b/external/storm-jms/src/main/java/org/apache/storm/jms/JmsTupleProducer.java index fa5a9c5d6f4..69cd83c624d 100644 --- a/external/storm-jms/src/main/java/org/apache/storm/jms/JmsTupleProducer.java +++ b/external/storm-jms/src/main/java/org/apache/storm/jms/JmsTupleProducer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/external/storm-jms/src/main/java/org/apache/storm/jms/bolt/JmsBolt.java b/external/storm-jms/src/main/java/org/apache/storm/jms/bolt/JmsBolt.java index 845ac1abb87..936f5148ee0 100644 --- a/external/storm-jms/src/main/java/org/apache/storm/jms/bolt/JmsBolt.java +++ b/external/storm-jms/src/main/java/org/apache/storm/jms/bolt/JmsBolt.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -66,11 +72,9 @@ public class JmsBolt extends BaseTickTupleAwareRichBolt { private boolean jmsTransactional = false; private int jmsAcknowledgeMode = Session.AUTO_ACKNOWLEDGE; - private JmsProvider jmsProvider; private JmsMessageProducer producer; - private OutputCollector collector; /** diff --git a/external/storm-jms/src/main/java/org/apache/storm/jms/spout/JmsMessageID.java b/external/storm-jms/src/main/java/org/apache/storm/jms/spout/JmsMessageID.java index aae3c4405e6..15fc084db48 100644 --- a/external/storm-jms/src/main/java/org/apache/storm/jms/spout/JmsMessageID.java +++ b/external/storm-jms/src/main/java/org/apache/storm/jms/spout/JmsMessageID.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/external/storm-jms/src/main/java/org/apache/storm/jms/spout/JmsSpout.java b/external/storm-jms/src/main/java/org/apache/storm/jms/spout/JmsSpout.java index dbd4782da55..3d3ff306047 100644 --- a/external/storm-jms/src/main/java/org/apache/storm/jms/spout/JmsSpout.java +++ b/external/storm-jms/src/main/java/org/apache/storm/jms/spout/JmsSpout.java @@ -25,12 +25,10 @@ import jakarta.jms.Message; import jakarta.jms.MessageConsumer; import jakarta.jms.Session; - import java.io.Serializable; import java.util.Collections; import java.util.HashMap; import java.util.Map; - import org.apache.storm.Config; import org.apache.storm.jms.JmsProvider; import org.apache.storm.jms.JmsTupleProducer; @@ -104,7 +102,6 @@ public class JmsSpout extends BaseRichSpout { */ private MessageConsumer consumer; - /** * If JMS provider supports ack-ing individual messages. */ @@ -340,7 +337,8 @@ public boolean isDistributed() { * Sets the "distributed" mode of this spout. * *

If true multiple instances of this spout may be - * created across the cluster (depending on the "parallelism_hint" in the topology configuration). + * created across the cluster (depending on the "parallelism_hint" in the topology + * configuration). * *

Setting this value to false essentially means this spout * will run as a singleton within the cluster ("parallelism_hint" will be ignored). diff --git a/external/storm-jms/src/main/java/org/apache/storm/jms/trident/JmsBatch.java b/external/storm-jms/src/main/java/org/apache/storm/jms/trident/JmsBatch.java index 5db46775c6c..c70673d4e65 100644 --- a/external/storm-jms/src/main/java/org/apache/storm/jms/trident/JmsBatch.java +++ b/external/storm-jms/src/main/java/org/apache/storm/jms/trident/JmsBatch.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/external/storm-jms/src/main/java/org/apache/storm/jms/trident/JmsState.java b/external/storm-jms/src/main/java/org/apache/storm/jms/trident/JmsState.java index 69c45508382..91b7b5cff78 100644 --- a/external/storm-jms/src/main/java/org/apache/storm/jms/trident/JmsState.java +++ b/external/storm-jms/src/main/java/org/apache/storm/jms/trident/JmsState.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -78,7 +84,8 @@ public void commit(Long someLong) { } } - public void updateState(List tuples, TridentCollector collector) throws JMSException { + public void updateState(List tuples, + TridentCollector collector) throws JMSException { try { for (TridentTuple tuple : tuples) { Message msg = this.options.msgProducer.toMessage(this.session, tuple); diff --git a/external/storm-jms/src/main/java/org/apache/storm/jms/trident/JmsStateFactory.java b/external/storm-jms/src/main/java/org/apache/storm/jms/trident/JmsStateFactory.java index 9c549a9e13e..59eeb6269a1 100644 --- a/external/storm-jms/src/main/java/org/apache/storm/jms/trident/JmsStateFactory.java +++ b/external/storm-jms/src/main/java/org/apache/storm/jms/trident/JmsStateFactory.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -26,7 +32,8 @@ public JmsStateFactory(JmsState.Options options) { } @Override - public State makeState(Map map, IMetricsContext metricsContext, int partitionIndex, int numPartitions) { + public State makeState(Map map, IMetricsContext metricsContext, + int partitionIndex, int numPartitions) { JmsState state = new JmsState(options); state.prepare(); return state; diff --git a/external/storm-jms/src/main/java/org/apache/storm/jms/trident/JmsUpdater.java b/external/storm-jms/src/main/java/org/apache/storm/jms/trident/JmsUpdater.java index ceb6975b0f3..c6bff725ccd 100644 --- a/external/storm-jms/src/main/java/org/apache/storm/jms/trident/JmsUpdater.java +++ b/external/storm-jms/src/main/java/org/apache/storm/jms/trident/JmsUpdater.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -22,7 +28,8 @@ public class JmsUpdater extends BaseStateUpdater { @Override - public void updateState(JmsState jmsState, List tuples, TridentCollector collector) { + public void updateState(JmsState jmsState, List tuples, + TridentCollector collector) { try { jmsState.updateState(tuples, collector); } catch (JMSException e) { diff --git a/external/storm-jms/src/main/java/org/apache/storm/jms/trident/TridentJmsSpout.java b/external/storm-jms/src/main/java/org/apache/storm/jms/trident/TridentJmsSpout.java index f5556c9019e..9fd996f420b 100644 --- a/external/storm-jms/src/main/java/org/apache/storm/jms/trident/TridentJmsSpout.java +++ b/external/storm-jms/src/main/java/org/apache/storm/jms/trident/TridentJmsSpout.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -65,7 +71,8 @@ public TridentJmsSpout() { } /** - * Return a friendly string for the given JMS acknowledge mode, or throw an IllegalArgumentException if + * Return a friendly string for the given JMS acknowledge mode, or throw an + * IllegalArgumentException if * the mode is not recognized. *

* Possible values: @@ -74,6 +81,7 @@ public TridentJmsSpout() { *

  • jakarta.jms.Session.CLIENT_ACKNOWLEDGE
  • *
  • jakarta.jms.Session.DUPS_OK_ACKNOWLEDGE
  • * + * * @param acknowledgeMode A valid JMS acknowledge mode * @return A friendly string describing the acknowledge mode * @throws IllegalArgumentException if the mode is not recognized @@ -88,12 +96,14 @@ private static String toDeliveryModeString(int acknowledgeMode) { return "DUPS_OK_ACKNOWLEDGE"; default: throw new IllegalArgumentException( - "Unknown JMS Acknowledge mode " + acknowledgeMode + " (See jakarta.jms.Session for valid values)"); + "Unknown JMS Acknowledge mode " + acknowledgeMode + + " (See jakarta.jms.Session for valid values)"); } } /** * Set the name for this spout, to improve log identification. + * * @param name The name to be used in log messages * @return This spout */ @@ -112,7 +122,8 @@ public TridentJmsSpout withJmsProvider(JmsProvider provider) { } /** - * Set the JmsTupleProducer implementation that will convert jakarta.jms.Message + * Set the JmsTupleProducer implementation that will convert + * jakarta.jms.Message * object to backtype.storm.tuple.Values objects to be emitted. * * @return This spout @@ -131,6 +142,7 @@ public TridentJmsSpout withTupleProducer(JmsTupleProducer tupleProducer) { *
  • jakarta.jms.Session.CLIENT_ACKNOWLEDGE
  • *
  • jakarta.jms.Session.DUPS_OK_ACKNOWLEDGE
  • * + * * @param jmsAcknowledgeMode The chosen acknowledge mode * @return This spout * @throws IllegalArgumentException if the mode is not recognized @@ -148,7 +160,8 @@ public ITridentSpout.BatchCoordinator getCoordinator( } @Override - public Emitter getEmitter(String txStateId, Map conf, TopologyContext context) { + public Emitter getEmitter(String txStateId, Map conf, + TopologyContext context) { return new JmsEmitter(name, jmsProvider, tupleProducer, jmsAcknowledgeMode, conf); } @@ -163,14 +176,16 @@ public Fields getOutputFields() { tupleProducer.declareOutputFields(fieldGetter); StreamInfo streamInfo = fieldGetter.getFieldsDeclaration().get(Utils.DEFAULT_STREAM_ID); if (streamInfo == null) { - throw new IllegalArgumentException("Jms Tuple producer has not declared output fields for the default stream"); + throw new IllegalArgumentException("Jms Tuple producer has not declared output fields " + + "for the default stream"); } return new Fields(streamInfo.get_output_fields()); } /** - * The JmsEmitter class listens for incoming messages and stores them in a blocking queue. On each invocation of emit, + * The JmsEmitter class listens for incoming messages and stores them in a blocking queue. On + * each invocation of emit, * the queued messages are emitted as a batch. * */ @@ -188,7 +203,8 @@ private class JmsEmitter implements Emitter, MessageListener { private final Logger log = LoggerFactory.getLogger(JmsEmitter.class); private long lastRotate; - JmsEmitter(String name, JmsProvider jmsProvider, JmsTupleProducer tupleProducer, int jmsAcknowledgeMode, + JmsEmitter(String name, JmsProvider jmsProvider, JmsTupleProducer tupleProducer, + int jmsAcknowledgeMode, Map conf) { if (jmsProvider == null) { throw new IllegalStateException("JMS provider has not been set."); @@ -201,7 +217,8 @@ private class JmsEmitter implements Emitter, MessageListener { this.name = name; batchMessageMap = new RotatingMap>(3); - rotateTimeMillis = 1000L * ((Number) conf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS)).intValue(); + rotateTimeMillis = 1000L * ((Number) conf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS)) + .intValue(); lastRotate = System.currentTimeMillis(); Number batchSize = (Number) conf.get(MAX_BATCH_SIZE_CONF); @@ -217,7 +234,8 @@ private class JmsEmitter implements Emitter, MessageListener { this.connection.start(); log.info( - "Created JmsEmitter with max batch size " + maxBatchSize + " rotate time " + rotateTimeMillis + "Created JmsEmitter with max batch size " + maxBatchSize + " rotate time " + + rotateTimeMillis + "ms and destination " + dest + " for " + name); } catch (Exception e) { @@ -235,7 +253,8 @@ public void success(TransactionAttempt tx) { if (messages != null) { if (!messages.isEmpty()) { - log.debug("Success for batch with transaction id " + tx.getTransactionId() + "/" + tx.getAttemptId() + " for " + name); + log.debug("Success for batch with transaction id " + tx.getTransactionId() + + "/" + tx.getAttemptId() + " for " + name); } for (Message msg : messages) { @@ -250,14 +269,18 @@ public void success(TransactionAttempt tx) { } } } else { - log.warn("No messages found in batch with transaction id " + tx.getTransactionId() + "/" + tx.getAttemptId()); + log.warn("No messages found in batch with transaction id " + tx.getTransactionId() + + "/" + tx.getAttemptId()); } } /** - * Fail a batch with the given transaction id. This is called when a batch is timed out, or a new batch with a - * matching transaction id is emitted. Note that the current implementation does nothing - i.e. it discards + * Fail a batch with the given transaction id. This is called when a batch is timed out, or + * a new batch with a + * matching transaction id is emitted. Note that the current implementation does nothing - + * i.e. it discards * messages that have been failed. + * * @param transactionId The transaction id of the failed batch * @param messages The list of messages to fail. */ @@ -302,7 +325,8 @@ public void emitBatch(TransactionAttempt tx, JmsBatch coordinatorMeta, } if (batchMessageMap.containsKey(tx.getTransactionId())) { - log.warn("FAILED duplicate batch with transaction id " + tx.getTransactionId() + "/" + tx.getAttemptId() + " for " + name); + log.warn("FAILED duplicate batch with transaction id " + tx.getTransactionId() + + "/" + tx.getAttemptId() + " for " + name); fail(tx.getTransactionId(), batchMessageMap.get(tx.getTransactionId())); } @@ -322,13 +346,15 @@ public void emitBatch(TransactionAttempt tx, JmsBatch coordinatorMeta, Values tuple = tupleProducer.toTuple(msg); collector.emit(tuple); } catch (JMSException e) { - log.warn("Failed to emit message, could not retrieve data for " + name + ": " + e); + log.warn("Failed to emit message, could not retrieve data for " + name + ": " + + e); } } if (!batchMessages.isEmpty()) { log.debug("Emitting batch with transaction id " + tx.getTransactionId() - + "/" + tx.getAttemptId() + " and size " + batchMessages.size() + " for " + name); + + "/" + tx.getAttemptId() + " and size " + batchMessages.size() + " for " + + name); } else { log.trace( "No items to acknowledge for batch with transaction id " + tx.getTransactionId() @@ -365,7 +391,8 @@ private class JmsBatchCoordinator implements BatchCoordinator { } @Override - public JmsBatch initializeTransaction(long txid, JmsBatch prevMetadata, JmsBatch curMetadata) { + public JmsBatch initializeTransaction(long txid, JmsBatch prevMetadata, + JmsBatch curMetadata) { log.debug("Initialise transaction " + txid + " for " + name); return null; } diff --git a/external/storm-jms/src/test/java/org/apache/storm/jms/spout/JmsSpoutTest.java b/external/storm-jms/src/test/java/org/apache/storm/jms/spout/JmsSpoutTest.java index 325c26e6cb0..5b86215634b 100644 --- a/external/storm-jms/src/test/java/org/apache/storm/jms/spout/JmsSpoutTest.java +++ b/external/storm-jms/src/test/java/org/apache/storm/jms/spout/JmsSpoutTest.java @@ -18,6 +18,8 @@ package org.apache.storm.jms.spout; +import static org.junit.jupiter.api.Assertions.assertTrue; + import jakarta.jms.ConnectionFactory; import jakarta.jms.Destination; import jakarta.jms.JMSException; @@ -25,6 +27,11 @@ import jakarta.jms.MessageProducer; import jakarta.jms.Session; import jakarta.jms.TextMessage; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.ObjectOutputStream; +import java.util.HashMap; +import java.util.Map; import org.apache.storm.Config; import org.apache.storm.jms.JmsProvider; import org.apache.storm.spout.SpoutOutputCollector; @@ -32,14 +39,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.ObjectOutputStream; -import java.util.HashMap; -import java.util.Map; - -import static org.junit.jupiter.api.Assertions.assertTrue; - public class JmsSpoutTest { private static final Logger LOG = LoggerFactory.getLogger(JmsSpoutTest.class); @@ -85,7 +84,8 @@ public void testSerializability() throws IOException { } /** - * Make sure that {@link JmsSpout#open} returns correctly regardless of the type of {@link Number} that is the value of {@link + * Make sure that {@link JmsSpout#open} returns correctly regardless of the type of {@link + * Number} that is the value of {@link * Config#TOPOLOGY_MESSAGE_TIMEOUT_SECS}. */ @Test diff --git a/external/storm-jms/src/test/java/org/apache/storm/jms/spout/MockJmsProvider.java b/external/storm-jms/src/test/java/org/apache/storm/jms/spout/MockJmsProvider.java index da26877ecbe..731fefceab0 100644 --- a/external/storm-jms/src/test/java/org/apache/storm/jms/spout/MockJmsProvider.java +++ b/external/storm-jms/src/test/java/org/apache/storm/jms/spout/MockJmsProvider.java @@ -20,12 +20,11 @@ import jakarta.jms.ConnectionFactory; import jakarta.jms.Destination; -import org.apache.activemq.ActiveMQConnectionFactory; -import org.apache.storm.jms.JmsProvider; - import javax.naming.Context; import javax.naming.InitialContext; import javax.naming.NamingException; +import org.apache.activemq.ActiveMQConnectionFactory; +import org.apache.storm.jms.JmsProvider; public class MockJmsProvider implements JmsProvider { private static final long serialVersionUID = 1L; @@ -34,7 +33,8 @@ public class MockJmsProvider implements JmsProvider { private Destination destination = null; public MockJmsProvider() throws NamingException { - this.connectionFactory = new ActiveMQConnectionFactory("vm://localhost?broker.persistent=false"); + this.connectionFactory = + new ActiveMQConnectionFactory("vm://localhost?broker.persistent=false"); Context jndiContext = new InitialContext(); this.destination = (Destination) jndiContext.lookup("dynamicQueues/FOO.BAR"); diff --git a/external/storm-jms/src/test/java/org/apache/storm/jms/spout/MockSpoutOutputCollector.java b/external/storm-jms/src/test/java/org/apache/storm/jms/spout/MockSpoutOutputCollector.java index 025194a40d9..c4c39eaf52a 100644 --- a/external/storm-jms/src/test/java/org/apache/storm/jms/spout/MockSpoutOutputCollector.java +++ b/external/storm-jms/src/test/java/org/apache/storm/jms/spout/MockSpoutOutputCollector.java @@ -18,10 +18,9 @@ package org.apache.storm.jms.spout; -import org.apache.storm.spout.ISpoutOutputCollector; - import java.util.ArrayList; import java.util.List; +import org.apache.storm.spout.ISpoutOutputCollector; public class MockSpoutOutputCollector implements ISpoutOutputCollector { boolean emitted = false; @@ -39,7 +38,7 @@ public void emitDirect(int taskId, String streamId, List tuple, Object m @Override public void flush() { - //NO-OP + // NO-OP } @Override diff --git a/external/storm-kafka-client/pom.xml b/external/storm-kafka-client/pom.xml index 368c3dce555..b47b75e3af6 100644 --- a/external/storm-kafka-client/pom.xml +++ b/external/storm-kafka-client/pom.xml @@ -173,6 +173,16 @@ + + org.openrewrite.maven + rewrite-maven-plugin + + + + org.codehaus.gmavenplus + gmavenplus-plugin + + org.apache.maven.plugins maven-checkstyle-plugin diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/bolt/KafkaBolt.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/bolt/KafkaBolt.java index 5929cafe8cd..89dea1a3c2d 100644 --- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/bolt/KafkaBolt.java +++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/bolt/KafkaBolt.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

    Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -39,7 +39,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; - /** * Bolt implementation that can send Tuple data to Kafka. *

    @@ -83,6 +82,7 @@ public KafkaBolt withTupleToKafkaMapper(TupleToKafkaMapper mapper) { /** * Set the messages to be published to a single topic. + * * @param topic the topic to publish to * @return this */ @@ -102,6 +102,7 @@ public KafkaBolt withProducerProperties(Properties producerProperties) { /** * Sets a user defined callback for use with the KafkaProducer. + * * @param producerCallback user defined callback * @return this */ @@ -111,18 +112,21 @@ public KafkaBolt withProducerCallback(PreparableCallback producerCallback) } @Override - public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) { + public void prepare(Map topoConf, TopologyContext context, + OutputCollector collector) { LOG.info("Preparing bolt with configuration {}", this); - //for backward compatibility. + // for backward compatibility. if (mapper == null) { - LOG.info("Mapper not specified. Setting default mapper to {}", FieldNameBasedTupleToKafkaMapper.class.getSimpleName()); + LOG.info("Mapper not specified. Setting default mapper to {}", + FieldNameBasedTupleToKafkaMapper.class.getSimpleName()); this.mapper = new FieldNameBasedTupleToKafkaMapper(); } - //for backward compatibility. + // for backward compatibility. if (topicSelector == null) { if (topoConf.containsKey(TOPIC)) { - LOG.info("TopicSelector not specified. Using [{}] for topic [{}] specified in bolt configuration,", + LOG.info("TopicSelector not specified. Using [{}] for topic [{}] specified in " + + "bolt configuration,", DefaultTopicSelector.class.getSimpleName(), topoConf.get(TOPIC)); this.topicSelector = new DefaultTopicSelector((String) topoConf.get(TOPIC)); } else { @@ -184,7 +188,8 @@ protected void process(final Tuple input) { } else if (providedCallback != null) { callback = providedCallback; } - Future result = producer.send(new ProducerRecord<>(topic, key, message), callback); + Future result = producer.send(new ProducerRecord<>(topic, key, + message), callback); if (!async) { try { result.get(); @@ -221,6 +226,7 @@ public void cleanup() { * the tuple as soon as it has handed the message off to the producer API * if false (the default) the message will be acked after it was successfully sent to kafka or * failed if it was not successfully sent. + * * @param fireAndForget whether the bolt should fire and forget */ public void setFireAndForget(boolean fireAndForget) { @@ -230,6 +236,7 @@ public void setFireAndForget(boolean fireAndForget) { /** * If set to true(the default) the bolt will not wait for the message * to be fully sent to Kafka before getting another tuple to send. + * * @param async true to have multiple tuples in flight to kafka, else false. */ public void setAsync(boolean async) { diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/bolt/PreparableCallback.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/bolt/PreparableCallback.java index 33814356876..0ed4888d8ba 100644 --- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/bolt/PreparableCallback.java +++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/bolt/PreparableCallback.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

    Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -20,7 +20,6 @@ import java.io.Serializable; import java.util.Map; - import org.apache.kafka.clients.producer.Callback; import org.apache.storm.task.TopologyContext; diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/bolt/mapper/FieldNameBasedTupleToKafkaMapper.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/bolt/mapper/FieldNameBasedTupleToKafkaMapper.java index 1a1b09717d9..ce9a83a0f60 100644 --- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/bolt/mapper/FieldNameBasedTupleToKafkaMapper.java +++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/bolt/mapper/FieldNameBasedTupleToKafkaMapper.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

    Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -38,7 +38,7 @@ public FieldNameBasedTupleToKafkaMapper(String boltKeyField, String boltMessageF @Override public K getKeyFromTuple(Tuple tuple) { - //for backward compatibility, we return null when key is not present. + // for backward compatibility, we return null when key is not present. return tuple.contains(boltKeyField) ? (K) tuple.getValueByField(boltKeyField) : null; } diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/bolt/mapper/TupleToKafkaMapper.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/bolt/mapper/TupleToKafkaMapper.java index 19d4da61fb5..b8e6ceeda23 100644 --- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/bolt/mapper/TupleToKafkaMapper.java +++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/bolt/mapper/TupleToKafkaMapper.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

    Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -23,6 +23,7 @@ /** * Interface defining a mapping from storm tuple to kafka key and message. + * * @param type of key. * @param type of value. */ diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/bolt/selector/DefaultTopicSelector.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/bolt/selector/DefaultTopicSelector.java index 4adc4663eed..825296deffe 100644 --- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/bolt/selector/DefaultTopicSelector.java +++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/bolt/selector/DefaultTopicSelector.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

    Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/bolt/selector/FieldIndexTopicSelector.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/bolt/selector/FieldIndexTopicSelector.java index e6c35ce1a66..6027682d709 100644 --- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/bolt/selector/FieldIndexTopicSelector.java +++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/bolt/selector/FieldIndexTopicSelector.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

    Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -35,8 +35,10 @@ public class FieldIndexTopicSelector implements KafkaTopicSelector { /** * Creates a new FieldIndexTopicSelector. + * * @param fieldIndex The index of the field containing the topic name - * @param defaultTopicName The default topic name if the topic name cannot be read from the tuple + * @param defaultTopicName The default topic name if the topic name cannot be read from the + * tuple */ public FieldIndexTopicSelector(int fieldIndex, String defaultTopicName) { this.fieldIndex = fieldIndex; @@ -51,7 +53,8 @@ public String getTopic(Tuple tuple) { if (fieldIndex < tuple.size()) { return tuple.getString(fieldIndex); } else { - LOG.warn("Field index {} is out of bounds. Using default topic {}", fieldIndex, defaultTopicName); + LOG.warn("Field index {} is out of bounds. Using default topic {}", fieldIndex, + defaultTopicName); return defaultTopicName; } } diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/bolt/selector/FieldNameTopicSelector.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/bolt/selector/FieldNameTopicSelector.java index c8fa74c2e2c..7e1bab6a21f 100644 --- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/bolt/selector/FieldNameTopicSelector.java +++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/bolt/selector/FieldNameTopicSelector.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

    Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -32,7 +32,6 @@ public class FieldNameTopicSelector implements KafkaTopicSelector { private final String fieldName; private final String defaultTopicName; - public FieldNameTopicSelector(String fieldName, String defaultTopicName) { this.fieldName = fieldName; this.defaultTopicName = defaultTopicName; diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/bolt/selector/KafkaTopicSelector.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/bolt/selector/KafkaTopicSelector.java index cd6dd2919f0..2764778ab7b 100644 --- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/bolt/selector/KafkaTopicSelector.java +++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/bolt/selector/KafkaTopicSelector.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

    Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/ByTopicRecordTranslator.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/ByTopicRecordTranslator.java index ef8203f0f01..fe2302bb7ef 100644 --- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/ByTopicRecordTranslator.java +++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/ByTopicRecordTranslator.java @@ -26,8 +26,10 @@ import org.apache.storm.tuple.Fields; /** - * Based off of a given Kafka topic a ConsumerRecord came from it will be translated to a Storm tuple + * Based off of a given Kafka topic a ConsumerRecord came from it will be translated to a Storm + * tuple * and emitted to a given stream. + * * @param the key of the incoming Records * @param the value of the incoming Records */ @@ -41,18 +43,22 @@ public class ByTopicRecordTranslator implements RecordTranslator { * Create a simple record translator that will use func to extract the fields of the tuple, * named by fields, and emit them to stream. This will handle all topics not explicitly set * elsewhere. + * * @param func extracts and turns them into a list of objects to be emitted * @param fields the names of the fields extracted * @param stream the stream to emit these fields on. */ - public ByTopicRecordTranslator(Func, List> func, Fields fields, String stream) { + public ByTopicRecordTranslator(Func, List> func, Fields fields, + String stream) { this(new SimpleRecordTranslator<>(func, fields, stream)); } /** * Create a simple record translator that will use func to extract the fields of the tuple, - * named by fields, and emit them to the default stream. This will handle all topics not explicitly set + * named by fields, and emit them to the default stream. This will handle all topics not + * explicitly set * elsewhere. + * * @param func extracts and turns them into a list of objects to be emitted * @param fields the names of the fields extracted */ @@ -62,17 +68,20 @@ public ByTopicRecordTranslator(Func, List> func, Fi /** * Create a record translator with the given default translator. + * * @param defaultTranslator a translator that will be used for all topics not explicitly set - * with one of the variants of {@link #forTopic(java.lang.String, org.apache.storm.kafka.spout.RecordTranslator) }. + * with one of the variants of {@link #forTopic(java.lang.String, + * org.apache.storm.kafka.spout.RecordTranslator) }. */ public ByTopicRecordTranslator(RecordTranslator defaultTranslator) { this.defaultTranslator = defaultTranslator; - //This shouldn't throw on a Check, because nothing is configured yet + // This shouldn't throw on a Check, because nothing is configured yet cacheNCheckFields(defaultTranslator); } /** * Configure a translator for a given topic with tuples to be emitted to the default stream. + * * @param topic the topic this should be used for * @param func extracts and turns them into a list of objects to be emitted * @param fields the names of the fields extracted @@ -81,12 +90,14 @@ public ByTopicRecordTranslator(RecordTranslator defaultTranslator) { * @throws IllegalArgumentException if the Fields for the stream this emits to do not match * any already configured Fields for the same stream */ - public ByTopicRecordTranslator forTopic(String topic, Func, List> func, Fields fields) { + public ByTopicRecordTranslator forTopic(String topic, Func, + List> func, Fields fields) { return forTopic(topic, new SimpleRecordTranslator<>(func, fields)); } /** * Configure a translator for a given topic. + * * @param topic the topic this should be used for * @param func extracts and turns them into a list of objects to be emitted * @param fields the names of the fields extracted @@ -103,6 +114,7 @@ public ByTopicRecordTranslator forTopic(String topic, Func translator) { @Override public List apply(ConsumerRecord record) { - RecordTranslator trans = topicToTranslator.getOrDefault(record.topic(), defaultTranslator); + RecordTranslator trans = topicToTranslator.getOrDefault(record.topic(), + defaultTranslator); return trans.apply(record); } diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/EmptyKafkaTupleListener.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/EmptyKafkaTupleListener.java index a9a5cc54897..9b37db6cbca 100644 --- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/EmptyKafkaTupleListener.java +++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/EmptyKafkaTupleListener.java @@ -16,7 +16,6 @@ * limitations under the License. */ - package org.apache.storm.kafka.spout; import java.util.Collection; @@ -28,22 +27,22 @@ public final class EmptyKafkaTupleListener implements KafkaTupleListener { @Override - public void open(Map conf, TopologyContext context) { } + public void open(Map conf, TopologyContext context) {} @Override - public void onEmit(List tuple, KafkaSpoutMessageId msgId) { } + public void onEmit(List tuple, KafkaSpoutMessageId msgId) {} @Override - public void onAck(KafkaSpoutMessageId msgId) { } + public void onAck(KafkaSpoutMessageId msgId) {} @Override - public void onPartitionsReassigned(Collection topicPartitions) { } + public void onPartitionsReassigned(Collection topicPartitions) {} @Override - public void onRetry(KafkaSpoutMessageId msgId) { } + public void onRetry(KafkaSpoutMessageId msgId) {} @Override - public void onMaxRetryReached(KafkaSpoutMessageId msgId) { } + public void onMaxRetryReached(KafkaSpoutMessageId msgId) {} @Override public String toString() { diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/FirstPollOffsetStrategy.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/FirstPollOffsetStrategy.java index 9184bc3c06c..f4d1743e74f 100644 --- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/FirstPollOffsetStrategy.java +++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/FirstPollOffsetStrategy.java @@ -17,35 +17,43 @@ package org.apache.storm.kafka.spout; /** - * Defines how the spout seeks the offset to be used in the first poll to Kafka upon topology deployment. By default this parameter is set + * Defines how the spout seeks the offset to be used in the first poll to Kafka upon topology + * deployment. By default this parameter is set * to UNCOMMITTED_EARLIEST. */ public enum FirstPollOffsetStrategy { /** - * The kafka spout polls records starting in the first offset of the partition, regardless of previous commits. This setting only takes + * The kafka spout polls records starting in the first offset of the partition, regardless of + * previous commits. This setting only takes * effect on topology deployment */ EARLIEST, /** - * The kafka spout polls records starting at the end of the partition, regardless of previous commits. This setting only takes effect on + * The kafka spout polls records starting at the end of the partition, regardless of previous + * commits. This setting only takes effect on * topology deployment */ LATEST, /** - * The kafka spout polls records starting at the earliest offset whose timestamp is greater than or equal to the given startTimestamp. - * This setting only takes effect on topology deployment. This option is currently available only for the Trident Spout + * The kafka spout polls records starting at the earliest offset whose timestamp is greater than + * or equal to the given startTimestamp. + * This setting only takes effect on topology deployment. This option is currently available + * only for the Trident Spout */ TIMESTAMP, /** - * The kafka spout polls records from the last committed offset, if any. If no offset has been committed it behaves as EARLIEST + * The kafka spout polls records from the last committed offset, if any. If no offset has been + * committed it behaves as EARLIEST */ UNCOMMITTED_EARLIEST, /** - * The kafka spout polls records from the last committed offset, if any. If no offset has been committed it behaves as LATEST + * The kafka spout polls records from the last committed offset, if any. If no offset has been + * committed it behaves as LATEST */ UNCOMMITTED_LATEST, /** - * The kafka spout polls records from the last committed offset, if any. If no offset has been committed it behaves as TIMESTAMP. + * The kafka spout polls records from the last committed offset, if any. If no offset has been + * committed it behaves as TIMESTAMP. * This option is currently available only for the Trident Spout */ UNCOMMITTED_TIMESTAMP; diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpout.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpout.java index 73582dd4c8a..bddf57803bd 100644 --- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpout.java +++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpout.java @@ -24,7 +24,6 @@ import static org.apache.storm.kafka.spout.FirstPollOffsetStrategy.UNCOMMITTED_LATEST; import com.google.common.annotations.VisibleForTesting; - import java.time.Duration; import java.util.ArrayList; import java.util.Collection; @@ -34,12 +33,11 @@ import java.util.Iterator; import java.util.LinkedList; import java.util.List; -import java.util.Map; import java.util.Map.Entry; +import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; - import org.apache.commons.lang3.Validate; import org.apache.kafka.clients.admin.Admin; import org.apache.kafka.clients.consumer.Consumer; @@ -70,7 +68,7 @@ public class KafkaSpout extends BaseRichSpout { private static final long serialVersionUID = 4151921085047987154L; - //Initial delay for the commit and assignment refresh timers + // Initial delay for the commit and assignment refresh timers public static final long TIMER_DELAY_MS = 500; private static final Logger LOG = LoggerFactory.getLogger(KafkaSpout.class); @@ -93,15 +91,19 @@ public class KafkaSpout extends BaseRichSpout { private transient KafkaTupleListener tupleListener; // timer == null only if the processing guarantee is at-most-once private transient Timer commitTimer; - // Initialization is only complete after the first call to KafkaSpoutConsumerRebalanceListener.onPartitionsAssigned() + // Initialization is only complete after the first call to + // KafkaSpoutConsumerRebalanceListener.onPartitionsAssigned() - // Tuples that were successfully acked/emitted. These tuples will be committed periodically when the commit timer expires, - // or after a consumer rebalance, or during close/deactivate. Always empty if processing guarantee is none or at-most-once. + // Tuples that were successfully acked/emitted. These tuples will be committed periodically when + // the commit timer expires, + // or after a consumer rebalance, or during close/deactivate. Always empty if processing + // guarantee is none or at-most-once. private transient Map offsetManagers; // Tuples that have been emitted but that are "on the wire", i.e. pending being acked or failed. // Always empty if processing guarantee is none or at-most-once private transient Set emitted; - // Records that have been polled and are queued to be emitted in the nextTuple() call. One record is emitted per nextTuple() + // Records that have been polled and are queued to be emitted in the nextTuple() call. One + // record is emitted per nextTuple() private transient Map>> waitingToEmit; // Triggers when an assignment should be refreshed private transient Timer refreshAssignmentTimer; @@ -115,14 +117,16 @@ public KafkaSpout(KafkaSpoutConfig kafkaSpoutConfig) { } @VisibleForTesting - KafkaSpout(KafkaSpoutConfig kafkaSpoutConfig, ClientFactory kafkaClientFactory, TopicAssigner topicAssigner) { + KafkaSpout(KafkaSpoutConfig kafkaSpoutConfig, ClientFactory kafkaClientFactory, + TopicAssigner topicAssigner) { this.kafkaClientFactory = kafkaClientFactory; this.topicAssigner = topicAssigner; this.kafkaSpoutConfig = kafkaSpoutConfig; } @Override - public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { + public void open(Map conf, TopologyContext context, + SpoutOutputCollector collector) { this.context = context; // Spout internals @@ -136,16 +140,21 @@ public void open(Map conf, TopologyContext context, SpoutOutputC tupleListener = kafkaSpoutConfig.getTupleListener(); - if (kafkaSpoutConfig.getProcessingGuarantee() != KafkaSpoutConfig.ProcessingGuarantee.AT_MOST_ONCE) { - // In at-most-once mode the offsets are committed after every poll, and not periodically as controlled by the timer - commitTimer = new Timer(TIMER_DELAY_MS, kafkaSpoutConfig.getOffsetsCommitPeriodMs(), TimeUnit.MILLISECONDS); + if (kafkaSpoutConfig + .getProcessingGuarantee() != KafkaSpoutConfig.ProcessingGuarantee.AT_MOST_ONCE) { + // In at-most-once mode the offsets are committed after every poll, and not periodically + // as controlled by the timer + commitTimer = new Timer(TIMER_DELAY_MS, kafkaSpoutConfig.getOffsetsCommitPeriodMs(), + TimeUnit.MILLISECONDS); } - refreshAssignmentTimer = new Timer(TIMER_DELAY_MS, kafkaSpoutConfig.getPartitionRefreshPeriodMs(), TimeUnit.MILLISECONDS); + refreshAssignmentTimer = new Timer(TIMER_DELAY_MS, kafkaSpoutConfig + .getPartitionRefreshPeriodMs(), TimeUnit.MILLISECONDS); offsetManagers = new HashMap<>(); emitted = new HashSet<>(); waitingToEmit = new HashMap<>(); - commitMetadataManager = new CommitMetadataManager(context, kafkaSpoutConfig.getProcessingGuarantee()); + commitMetadataManager = new CommitMetadataManager(context, kafkaSpoutConfig + .getProcessingGuarantee()); rebalanceListener = new KafkaSpoutConsumerRebalanceListener(); @@ -155,7 +164,8 @@ public void open(Map conf, TopologyContext context, SpoutOutputC tupleListener.open(conf, context); this.kafkaOffsetMetricManager - = new KafkaOffsetMetricManager<>(() -> Collections.unmodifiableMap(offsetManagers), () -> admin, context); + = new KafkaOffsetMetricManager<>(() -> Collections.unmodifiableMap(offsetManagers), + () -> admin, context); LOG.info("Kafka Spout opened with the following configuration: {}", kafkaSpoutConfig); } @@ -164,14 +174,16 @@ private boolean canRegisterMetrics() { try { KafkaConsumer.class.getDeclaredMethod("beginningOffsets", Collection.class); } catch (NoSuchMethodException e) { - LOG.warn("Minimum required kafka-clients library version to enable metrics is 0.10.1.0. Disabling spout metrics."); + LOG.warn("Minimum required kafka-clients library version to enable metrics is " + + "0.10.1.0. Disabling spout metrics."); return false; } return true; } private boolean isAtLeastOnceProcessing() { - return kafkaSpoutConfig.getProcessingGuarantee() == KafkaSpoutConfig.ProcessingGuarantee.AT_LEAST_ONCE; + return kafkaSpoutConfig + .getProcessingGuarantee() == KafkaSpoutConfig.ProcessingGuarantee.AT_LEAST_ONCE; } // =========== Consumer Rebalance Listener - On the same thread as the caller =========== @@ -193,8 +205,10 @@ public void onPartitionsRevoked(Collection partitions) { @Override public void onPartitionsAssigned(Collection partitions) { - LOG.info("Partitions reassignment. [task-ID={}, consumer-group={}, consumer={}, topic-partitions={}]", - context.getThisTaskId(), kafkaSpoutConfig.getConsumerGroupId(), consumer, partitions); + LOG.info("Partitions reassignment. [task-ID={}, consumer-group={}, consumer={}, " + + "topic-partitions={}]", + context.getThisTaskId(), kafkaSpoutConfig + .getConsumerGroupId(), consumer, partitions); initialize(partitions); tupleListener.onPartitionsReassigned(partitions); @@ -202,12 +216,14 @@ public void onPartitionsAssigned(Collection partitions) { private void initialize(Collection partitions) { if (isAtLeastOnceProcessing()) { - // remove offsetManagers for all partitions that are no longer assigned to this spout + // remove offsetManagers for all partitions that are no longer assigned to this + // spout offsetManagers.keySet().retainAll(partitions); retryService.retainAll(partitions); /* - * Emitted messages for partitions that are no longer assigned to this spout can't be acked and should not be retried, hence + * Emitted messages for partitions that are no longer assigned to this spout can't + * be acked and should not be retried, hence * remove them from emitted collection. */ emitted.removeIf(msgId -> !partitions.contains(msgId.getTopicPartition())); @@ -216,12 +232,15 @@ private void initialize(Collection partitions) { Set newPartitions = new HashSet<>(partitions); // If this partition was previously assigned to this spout, - // leave the acked offsets and consumer position as they were to resume where it left off + // leave the acked offsets and consumer position as they were to resume where it left + // off newPartitions.removeAll(previousAssignment); for (TopicPartition newTp : newPartitions) { - final Map committedOffset = consumer.committed(Collections.singleton(newTp)); + final Map committedOffset = consumer + .committed(Collections.singleton(newTp)); final long fetchOffset = doSeek(newTp, committedOffset.get(newTp)); - LOG.debug("Set consumer position to [{}] for topic-partition [{}] with [{}] and committed offset [{}]", + LOG.debug("Set consumer position to [{}] for topic-partition [{}] with [{}] and " + + "committed offset [{}]", fetchOffset, newTp, firstPollOffsetStrategy, committedOffset); if (isAtLeastOnceProcessing() && !offsetManagers.containsKey(newTp)) { offsetManagers.put(newTp, new OffsetManager(newTp, fetchOffset)); @@ -231,37 +250,45 @@ private void initialize(Collection partitions) { } /** - * Sets the cursor to the location dictated by the first poll strategy and returns the fetch offset. + * Sets the cursor to the location dictated by the first poll strategy and returns the fetch + * offset. */ private long doSeek(TopicPartition newTp, OffsetAndMetadata committedOffset) { LOG.trace("Seeking offset for topic-partition [{}] with [{}] and committed offset [{}]", newTp, firstPollOffsetStrategy, committedOffset); if (committedOffset != null) { - // offset was previously committed for this consumer group and topic-partition, either by this or another topology. + // offset was previously committed for this consumer group and topic-partition, + // either by this or another topology. if (commitMetadataManager.isOffsetCommittedByThisTopology(newTp, committedOffset, Collections.unmodifiableMap(offsetManagers))) { - // Another KafkaSpout instance (of this topology) already committed, therefore FirstPollOffsetStrategy does not apply. + // Another KafkaSpout instance (of this topology) already committed, therefore + // FirstPollOffsetStrategy does not apply. consumer.seek(newTp, committedOffset.offset()); } else { - // offset was not committed by this topology, therefore FirstPollOffsetStrategy applies + // offset was not committed by this topology, therefore FirstPollOffsetStrategy + // applies // (only when the topology is first deployed). if (firstPollOffsetStrategy.equals(EARLIEST)) { consumer.seekToBeginning(Collections.singleton(newTp)); } else if (firstPollOffsetStrategy.equals(LATEST)) { consumer.seekToEnd(Collections.singleton(newTp)); } else { - // Resume polling at the last committed offset, i.e. the first offset that is not marked as processed. + // Resume polling at the last committed offset, i.e. the first offset that + // is not marked as processed. consumer.seek(newTp, committedOffset.offset()); } } } else { - // no offset commits have ever been done for this consumer group and topic-partition, + // no offset commits have ever been done for this consumer group and + // topic-partition, // so start at the beginning or end depending on FirstPollOffsetStrategy - if (firstPollOffsetStrategy.equals(EARLIEST) || firstPollOffsetStrategy.equals(UNCOMMITTED_EARLIEST)) { + if (firstPollOffsetStrategy.equals(EARLIEST) || firstPollOffsetStrategy + .equals(UNCOMMITTED_EARLIEST)) { consumer.seekToBeginning(Collections.singleton(newTp)); - } else if (firstPollOffsetStrategy.equals(LATEST) || firstPollOffsetStrategy.equals(UNCOMMITTED_LATEST)) { + } else if (firstPollOffsetStrategy.equals(LATEST) || firstPollOffsetStrategy + .equals(UNCOMMITTED_LATEST)) { consumer.seekToEnd(Collections.singleton(newTp)); } } @@ -280,7 +307,8 @@ public void nextTuple() { if (commitTimer != null && commitTimer.isExpiredResetOnTrue()) { if (isAtLeastOnceProcessing()) { commitOffsetsForAckedTuples(); - } else if (kafkaSpoutConfig.getProcessingGuarantee() == ProcessingGuarantee.NO_GUARANTEE) { + } else if (kafkaSpoutConfig + .getProcessingGuarantee() == ProcessingGuarantee.NO_GUARANTEE) { Map offsetsToCommit = createFetchedOffsetsMetadata(consumer.assignment()); consumer.commitAsync(offsetsToCommit, null); @@ -304,8 +332,9 @@ public void nextTuple() { } private void throwKafkaConsumerInterruptedException() { - //Kafka throws their own type of exception when interrupted. - //Throw a new Java InterruptedException to ensure Storm can recognize the exception as a reaction to an interrupt. + // Kafka throws their own type of exception when interrupted. + // Throw a new Java InterruptedException to ensure Storm can recognize the exception as a + // reaction to an interrupt. throw new RuntimeException(new InterruptedException("Kafka consumer was interrupted")); } @@ -320,23 +349,27 @@ private PollablePartitionsInfo getPollablePartitionsInfo() { return new PollablePartitionsInfo(assignment, Collections.emptyMap()); } - Map earliestRetriableOffsets = retryService.earliestRetriableOffsets(); + Map earliestRetriableOffsets = retryService + .earliestRetriableOffsets(); Set pollablePartitions = new HashSet<>(); final int maxUncommittedOffsets = kafkaSpoutConfig.getMaxUncommittedOffsets(); for (TopicPartition tp : assignment) { OffsetManager offsetManager = offsetManagers.get(tp); int numUncommittedOffsets = offsetManager.getNumUncommittedOffsets(); if (numUncommittedOffsets < maxUncommittedOffsets) { - //Allow poll if the partition is not at the maxUncommittedOffsets limit + // Allow poll if the partition is not at the maxUncommittedOffsets limit pollablePartitions.add(tp); } else { - long offsetAtLimit = offsetManager.getNthUncommittedOffsetAfterCommittedOffset(maxUncommittedOffsets); + long offsetAtLimit = offsetManager + .getNthUncommittedOffsetAfterCommittedOffset(maxUncommittedOffsets); Long earliestRetriableOffset = earliestRetriableOffsets.get(tp); if (earliestRetriableOffset != null && earliestRetriableOffset <= offsetAtLimit) { - //Allow poll if there are retriable tuples within the maxUncommittedOffsets limit + // Allow poll if there are retriable tuples within the maxUncommittedOffsets + // limit pollablePartitions.add(tp); } else { - LOG.debug("Not polling on partition [{}]. It has [{}] uncommitted offsets, which exceeds the limit of [{}]. ", tp, + LOG.debug("Not polling on partition [{}]. It has [{}] uncommitted offsets, " + + "which exceeds the limit of [{}]. ", tp, numUncommittedOffsets, maxUncommittedOffsets); } } @@ -362,13 +395,15 @@ private ConsumerRecords pollKafkaBroker(PollablePartitionsInfo pollablePar pausedPartitions.removeIf(pollablePartitionsInfo.pollablePartitions::contains); try { consumer.pause(pausedPartitions); - final ConsumerRecords consumerRecords = consumer.poll(Duration.ofMillis(kafkaSpoutConfig.getPollTimeoutMs())); + final ConsumerRecords consumerRecords = consumer.poll(Duration + .ofMillis(kafkaSpoutConfig.getPollTimeoutMs())); ackRetriableOffsetsIfCompactedAway(pollablePartitionsInfo.pollableEarliestRetriableOffsets, consumerRecords); final int numPolledRecords = consumerRecords.count(); LOG.debug("Polled [{}] records from Kafka", numPolledRecords); - if (kafkaSpoutConfig.getProcessingGuarantee() == KafkaSpoutConfig.ProcessingGuarantee.AT_MOST_ONCE) { - //Commit polled records immediately to ensure delivery is at-most-once. + if (kafkaSpoutConfig + .getProcessingGuarantee() == KafkaSpoutConfig.ProcessingGuarantee.AT_MOST_ONCE) { + // Commit polled records immediately to ensure delivery is at-most-once. Map offsetsToCommit = createFetchedOffsetsMetadata(consumer.assignment()); consumer.commitSync(offsetsToCommit); @@ -380,14 +415,19 @@ private ConsumerRecords pollKafkaBroker(PollablePartitionsInfo pollablePar } } - private void doSeekRetriableTopicPartitions(Map pollableEarliestRetriableOffsets) { - for (Entry retriableTopicPartitionAndOffset : pollableEarliestRetriableOffsets.entrySet()) { - //Seek directly to the earliest retriable message for each retriable topic partition - consumer.seek(retriableTopicPartitionAndOffset.getKey(), retriableTopicPartitionAndOffset.getValue()); + private void doSeekRetriableTopicPartitions(Map pollableEarliestRetriableOffsets) { + for (Entry retriableTopicPartitionAndOffset : pollableEarliestRetriableOffsets + .entrySet()) { + // Seek directly to the earliest retriable message for each retriable topic partition + consumer.seek(retriableTopicPartitionAndOffset.getKey(), + retriableTopicPartitionAndOffset.getValue()); } } - private void ackRetriableOffsetsIfCompactedAway(Map earliestRetriableOffsets, + private void ackRetriableOffsetsIfCompactedAway(Map earliestRetriableOffsets, ConsumerRecords consumerRecords) { for (Entry entry : earliestRetriableOffsets.entrySet()) { TopicPartition tp = entry.getKey(); @@ -397,12 +437,15 @@ private void ackRetriableOffsetsIfCompactedAway(Map earlie long seekOffset = entry.getValue(); long earliestReceivedOffset = record.offset(); if (seekOffset < earliestReceivedOffset) { - //Since we asked for tuples starting at seekOffset, some retriable records must have been compacted away. - //Ack up to the first offset received if the record is not already acked or currently in the topology + // Since we asked for tuples starting at seekOffset, some retriable records must + // have been compacted away. + // Ack up to the first offset received if the record is not already acked or + // currently in the topology for (long i = seekOffset; i < earliestReceivedOffset; i++) { KafkaSpoutMessageId msgId = retryService.getMessageId(tp, i); if (!offsetManagers.get(tp).contains(msgId) && !emitted.contains(msgId)) { - LOG.debug("Record at offset [{}] appears to have been compacted away from topic [{}], marking as acked", i, tp); + LOG.debug("Record at offset [{}] appears to have been compacted away " + + "from topic [{}], marking as acked", i, tp); retryService.remove(msgId); emitted.add(msgId); ack(msgId); @@ -430,16 +473,19 @@ private void emitIfWaitingNotEmitted() { } /** - * Creates a tuple from the kafka record and emits it if it was never emitted or it is ready to be retried. + * Creates a tuple from the kafka record and emits it if it was never emitted or it is ready to + * be retried. * * @param record to be emitted - * @return true if tuple was emitted. False if tuple has been acked or has been emitted and is pending ack or fail + * @return true if tuple was emitted. False if tuple has been acked or has been emitted and is + * pending ack or fail */ private boolean emitOrRetryTuple(ConsumerRecord record) { final TopicPartition tp = new TopicPartition(record.topic(), record.partition()); final KafkaSpoutMessageId msgId = retryService.getMessageId(tp, record.offset()); - if (offsetManagers.containsKey(tp) && offsetManagers.get(tp).contains(msgId)) { // has been acked + if (offsetManagers.containsKey(tp) && offsetManagers.get(tp) + .contains(msgId)) { // has been acked LOG.trace("Tuple for record [{}] has already been acked. Skipping", record); } else if (emitted.contains(msgId)) { // has been emitted and it is pending ack or fail LOG.trace("Tuple for record [{}] has already been emitted. Skipping", record); @@ -447,14 +493,17 @@ private boolean emitOrRetryTuple(ConsumerRecord record) { final List tuple = kafkaSpoutConfig.getTranslator().apply(record); if (isEmitTuple(tuple)) { final boolean isScheduled = retryService.isScheduled(msgId); - // not scheduled <=> never failed (i.e. never emitted), or scheduled and ready to be retried + // not scheduled <=> never failed (i.e. never emitted), or scheduled and ready to be + // retried if (!isScheduled || retryService.isReady(msgId)) { - final String stream = tuple instanceof KafkaTuple ? ((KafkaTuple) tuple).getStream() : Utils.DEFAULT_STREAM_ID; + final String stream = tuple instanceof KafkaTuple ? ((KafkaTuple) tuple) + .getStream() : Utils.DEFAULT_STREAM_ID; if (!isAtLeastOnceProcessing()) { if (kafkaSpoutConfig.isTupleTrackingEnforced()) { collector.emit(stream, tuple, msgId); - LOG.trace("Emitted tuple [{}] for record [{}] with msgId [{}]", tuple, record, msgId); + LOG.trace("Emitted tuple [{}] for record [{}] with msgId [{}]", tuple, + record, msgId); } else { collector.emit(stream, tuple); LOG.trace("Emitted tuple [{}] for record [{}]", tuple, record); @@ -467,16 +516,19 @@ private boolean emitOrRetryTuple(ConsumerRecord record) { } collector.emit(stream, tuple, msgId); tupleListener.onEmit(tuple, msgId); - LOG.trace("Emitted tuple [{}] for record [{}] with msgId [{}]", tuple, record, msgId); + LOG.trace("Emitted tuple [{}] for record [{}] with msgId [{}]", tuple, + record, msgId); } return true; } } else { /* - * if a null tuple is not configured to be emitted, it should be marked as emitted and acked immediately to allow its offset + * if a null tuple is not configured to be emitted, it should be marked as emitted + * and acked immediately to allow its offset * to be commited to Kafka */ - LOG.debug("Not emitting null tuple for record [{}] as defined in configuration.", record); + LOG.debug("Not emitting null tuple for record [{}] as defined in configuration.", + record); if (isAtLeastOnceProcessing()) { msgId.setNullTuple(true); offsetManagers.get(tp).addToEmitMsgs(msgId.offset()); @@ -497,7 +549,8 @@ private boolean isEmitTuple(List tuple) { private Map createFetchedOffsetsMetadata(Set assignedPartitions) { Map offsetsToCommit = new HashMap<>(); for (TopicPartition tp : assignedPartitions) { - offsetsToCommit.put(tp, new OffsetAndMetadata(consumer.position(tp), commitMetadataManager.getCommitMetadata())); + offsetsToCommit.put(tp, new OffsetAndMetadata(consumer.position(tp), + commitMetadataManager.getCommitMetadata())); } return offsetsToCommit; } @@ -505,7 +558,8 @@ private Map createFetchedOffsetsMetadata(Set< private void commitOffsetsForAckedTuples() { final Map nextCommitOffsets = new HashMap<>(); for (Map.Entry tpOffset : offsetManagers.entrySet()) { - final OffsetAndMetadata nextCommitOffset = tpOffset.getValue().findNextCommitOffset(commitMetadataManager.getCommitMetadata()); + final OffsetAndMetadata nextCommitOffset = tpOffset.getValue() + .findNextCommitOffset(commitMetadataManager.getCommitMetadata()); if (nextCommitOffset != null) { nextCommitOffsets.put(tpOffset.getKey(), nextCommitOffset); } @@ -515,30 +569,39 @@ private void commitOffsetsForAckedTuples() { if (!nextCommitOffsets.isEmpty()) { consumer.commitSync(nextCommitOffsets); LOG.debug("Offsets successfully committed to Kafka [{}]", nextCommitOffsets); - // Instead of iterating again, it would be possible to commit and update the state for each TopicPartition - // in the prior loop, but the multiple network calls should be more expensive than iterating twice over a small loop - for (Map.Entry tpOffset : nextCommitOffsets.entrySet()) { - //Update the OffsetManager for each committed partition, and update numUncommittedOffsets + // Instead of iterating again, it would be possible to commit and update the state for + // each TopicPartition + // in the prior loop, but the multiple network calls should be more expensive than + // iterating twice over a small loop + for (Map.Entry tpOffset : nextCommitOffsets + .entrySet()) { + // Update the OffsetManager for each committed partition, and update + // numUncommittedOffsets final TopicPartition tp = tpOffset.getKey(); long position = consumer.position(tp); long committedOffset = tpOffset.getValue().offset(); if (position < committedOffset) { /* - * The position is behind the committed offset. This can happen in some cases, e.g. if a message failed, lots of (more - * than max.poll.records) later messages were acked, and the failed message then gets acked. The consumer may only be - * part way through "catching up" to where it was when it went back to retry the failed tuple. Skip the consumer forward + * The position is behind the committed offset. This can happen in some cases, + * e.g. if a message failed, lots of (more + * than max.poll.records) later messages were acked, and the failed message then + * gets acked. The consumer may only be + * part way through "catching up" to where it was when it went back to retry the + * failed tuple. Skip the consumer forward * to the committed offset. */ - LOG.debug("Consumer fell behind committed offset. Catching up. Position was [{}], skipping to [{}]", + LOG.debug("Consumer fell behind committed offset. Catching up. Position was " + + "[{}], skipping to [{}]", position, committedOffset); consumer.seek(tp, committedOffset); } /* - * In some cases the waitingToEmit list may contain tuples that have just been committed. Drop these. + * In some cases the waitingToEmit list may contain tuples that have just been + * committed. Drop these. */ List> waitingToEmitForTp = waitingToEmit.get(tp); if (waitingToEmitForTp != null) { - //Discard the pending records that are already committed + // Discard the pending records that are already committed waitingToEmit.put(tp, waitingToEmitForTp.stream() .filter(record -> record.offset() >= committedOffset) .collect(Collectors.toCollection(LinkedList::new))); @@ -546,7 +609,8 @@ private void commitOffsetsForAckedTuples() { final OffsetManager offsetManager = offsetManagers.get(tp); offsetManager.commit(tpOffset.getValue()); - LOG.debug("[{}] uncommitted offsets for partition [{}] after commit", offsetManager.getNumUncommittedOffsets(), tp); + LOG.debug("[{}] uncommitted offsets for partition [{}] after commit", offsetManager + .getNumUncommittedOffsets(), tp); } } else { LOG.trace("No offsets to commit. {}", this); @@ -565,7 +629,7 @@ public void ack(Object messageId) { final KafkaSpoutMessageId msgId = (KafkaSpoutMessageId) messageId; if (msgId.isNullTuple()) { - //a null tuple should be added to the ack list since by definition is a direct ack + // a null tuple should be added to the ack list since by definition is a direct ack offsetManagers.get(msgId.getTopicPartition()).addToAckMsgs(msgId); LOG.debug("Received direct ack for message [{}], associated with null tuple", msgId); tupleListener.onAck(msgId); @@ -573,12 +637,16 @@ public void ack(Object messageId) { } if (!emitted.contains(msgId)) { - LOG.debug("Received ack for message [{}], associated with tuple emitted for a ConsumerRecord that " - + "came from a topic-partition that this consumer group instance is no longer tracking " + LOG.debug("Received ack for message [{}], associated with tuple emitted for a " + + "ConsumerRecord that " + + "came from a topic-partition that this consumer group instance is no longer " + + "tracking " + "due to rebalance/partition reassignment. No action taken.", msgId); } else { - Validate.isTrue(!retryService.isScheduled(msgId), "The message id " + msgId + " is queued for retry while being acked." - + " This should never occur barring errors in the RetryService implementation or the spout code."); + Validate.isTrue(!retryService.isScheduled(msgId), "The message id " + msgId + + " is queued for retry while being acked." + + " This should never occur barring errors in the RetryService implementation " + + "or the spout code."); offsetManagers.get(msgId.getTopicPartition()).addToAckMsgs(msgId); emitted.remove(msgId); } @@ -599,15 +667,20 @@ public void fail(Object messageId) { + " Partitions may have been reassigned. Ignoring message [{}]", msgId); return; } - Validate.isTrue(!retryService.isScheduled(msgId), "The message id " + msgId + " is queued for retry while being failed." - + " This should never occur barring errors in the RetryService implementation or the spout code."); + Validate.isTrue(!retryService.isScheduled(msgId), "The message id " + msgId + + " is queued for retry while being failed." + + " This should never occur barring errors in the RetryService implementation or " + + "the spout code."); msgId.incrementNumFails(); if (!retryService.schedule(msgId)) { - LOG.debug("Reached maximum number of retries. Message [{}] being marked as acked.", msgId); - // this tuple should be removed from emitted only inside the ack() method. This is to ensure - // that the OffsetManager for that TopicPartition is updated and allows commit progression + LOG.debug("Reached maximum number of retries. Message [{}] being marked as acked.", + msgId); + // this tuple should be removed from emitted only inside the ack() method. This is to + // ensure + // that the OffsetManager for that TopicPartition is updated and allows commit + // progression tupleListener.onMaxRetryReached(msgId); ack(msgId); } else { @@ -627,12 +700,14 @@ public void activate() { } private void refreshAssignment() { - Set allPartitions = kafkaSpoutConfig.getTopicFilter().getAllSubscribedPartitions(consumer); + Set allPartitions = kafkaSpoutConfig.getTopicFilter() + .getAllSubscribedPartitions(consumer); List allPartitionsSorted = new ArrayList<>(allPartitions); Collections.sort(allPartitionsSorted, TopicPartitionComparator.INSTANCE); Set assignedPartitions = kafkaSpoutConfig.getTopicPartitioner() .getPartitionsForThisTask(allPartitionsSorted, context); - boolean partitionChanged = topicAssigner.assignPartitions(consumer, assignedPartitions, rebalanceListener); + boolean partitionChanged = topicAssigner.assignPartitions(consumer, assignedPartitions, + rebalanceListener); if (partitionChanged && canRegisterMetrics()) { LOG.info("Partitions assignments has changed, updating metrics."); kafkaOffsetMetricManager.registerMetricsForNewTopicPartitions(assignedPartitions); @@ -667,7 +742,7 @@ private void shutdown() { try { commitIfNecessary(); } finally { - //remove resources + // remove resources admin.close(); consumer.close(); } @@ -730,10 +805,11 @@ private String getTopicsString() { private static class PollablePartitionsInfo { private final Set pollablePartitions; - //The subset of earliest retriable offsets that are on pollable partitions + // The subset of earliest retriable offsets that are on pollable partitions private final Map pollableEarliestRetriableOffsets; - PollablePartitionsInfo(Set pollablePartitions, Map earliestRetriableOffsets) { + PollablePartitionsInfo(Set pollablePartitions, Map earliestRetriableOffsets) { this.pollablePartitions = pollablePartitions; this.pollableEarliestRetriableOffsets = earliestRetriableOffsets.entrySet().stream() .filter(entry -> pollablePartitions.contains(entry.getKey())) diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutConfig.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutConfig.java index d4e97897882..98b8240802d 100644 --- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutConfig.java +++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutConfig.java @@ -35,7 +35,8 @@ import org.slf4j.LoggerFactory; /** - * KafkaSpoutConfig defines the required configuration to connect a consumer to a consumer group, as well as the subscribing topics. + * KafkaSpoutConfig defines the required configuration to connect a consumer to a consumer group, as + * well as the subscribing topics. */ public class KafkaSpoutConfig extends CommonKafkaSpoutConfig { @@ -50,7 +51,8 @@ public class KafkaSpoutConfig extends CommonKafkaSpoutConfig { new KafkaSpoutRetryExponentialBackoff(TimeInterval.seconds(0), TimeInterval.milliSeconds(2), DEFAULT_MAX_RETRIES, TimeInterval.seconds(10)); - public static final ProcessingGuarantee DEFAULT_PROCESSING_GUARANTEE = ProcessingGuarantee.AT_LEAST_ONCE; + public static final ProcessingGuarantee DEFAULT_PROCESSING_GUARANTEE = + ProcessingGuarantee.AT_LEAST_ONCE; public static final KafkaTupleListener DEFAULT_TUPLE_LISTENER = new EmptyKafkaTupleListener(); public static final Logger LOG = LoggerFactory.getLogger(KafkaSpoutConfig.class); @@ -86,28 +88,38 @@ public KafkaSpoutConfig(Builder builder) { } /** - * This enum controls when the tuple with the {@link ConsumerRecord} for an offset is marked as processed, + * This enum controls when the tuple with the {@link ConsumerRecord} for an offset is marked as + * processed, * i.e. when the offset can be committed to Kafka. The default value is AT_LEAST_ONCE. - * The commit interval is controlled by {@link KafkaSpoutConfig#getOffsetsCommitPeriodMs() }, if the mode commits on an interval. - * NO_GUARANTEE may be removed in a later release without warning, we're still evaluating whether it makes sense to keep. + * The commit interval is controlled by {@link KafkaSpoutConfig#getOffsetsCommitPeriodMs() }, if + * the mode commits on an interval. + * NO_GUARANTEE may be removed in a later release without warning, we're still evaluating + * whether it makes sense to keep. */ @InterfaceStability.Unstable public enum ProcessingGuarantee { /** - * An offset is ready to commit only after the corresponding tuple has been processed and acked (at least once). If a tuple fails or - * times out it will be re-emitted, as controlled by the {@link KafkaSpoutRetryService}. Commits synchronously on the defined + * An offset is ready to commit only after the corresponding tuple has been processed and + * acked (at least once). If a tuple fails or + * times out it will be re-emitted, as controlled by the {@link KafkaSpoutRetryService}. + * Commits synchronously on the defined * interval. */ AT_LEAST_ONCE, /** - * Every offset will be synchronously committed to Kafka right after being polled but before being emitted to the downstream - * components of the topology. The commit interval is ignored. This mode guarantees that the offset is processed at most once by - * ensuring the spout won't retry tuples that fail or time out after the commit to Kafka has been done + * Every offset will be synchronously committed to Kafka right after being polled but before + * being emitted to the downstream + * components of the topology. The commit interval is ignored. This mode guarantees that the + * offset is processed at most once by + * ensuring the spout won't retry tuples that fail or time out after the commit to Kafka has + * been done */ AT_MOST_ONCE, /** - * The polled offsets are ready to commit immediately after being polled. The offsets are committed periodically, i.e. a message may - * be processed 0, 1 or more times. This behavior is similar to setting enable.auto.commit=true in the consumer, but allows the + * The polled offsets are ready to commit immediately after being polled. The offsets are + * committed periodically, i.e. a message may + * be processed 0, 1 or more times. This behavior is similar to setting + * enable.auto.commit=true in the consumer, but allows the * spout to control when commits occur. Commits asynchronously on the defined interval. */ NO_GUARANTEE, @@ -138,21 +150,27 @@ public Builder(String bootstrapServers, Pattern topics) { } /** - * Create a KafkaSpoutConfig builder with default property values and no key/value deserializers. + * Create a KafkaSpoutConfig builder with default property values and no key/value + * deserializers. * * @param bootstrapServers The bootstrap servers the consumer will use - * @param topicFilter The topic filter defining which topics and partitions the spout will read - * @param topicPartitioner The topic partitioner defining which topics and partitions are assinged to each spout task + * @param topicFilter The topic filter defining which topics and partitions the spout will + * read + * @param topicPartitioner The topic partitioner defining which topics and partitions are + * assinged to each spout task */ - public Builder(String bootstrapServers, TopicFilter topicFilter, ManualPartitioner topicPartitioner) { + public Builder(String bootstrapServers, TopicFilter topicFilter, + ManualPartitioner topicPartitioner) { super(bootstrapServers, topicFilter, topicPartitioner); } - //Spout Settings + // Spout Settings /** - * Specifies the period, in milliseconds, the offset commit task is periodically called. Default is 15s. + * Specifies the period, in milliseconds, the offset commit task is periodically called. + * Default is 15s. * - *

    This setting only has an effect if the configured {@link ProcessingGuarantee} is {@link ProcessingGuarantee#AT_LEAST_ONCE} or + *

    This setting only has an effect if the configured {@link ProcessingGuarantee} is + * {@link ProcessingGuarantee#AT_LEAST_ONCE} or * {@link ProcessingGuarantee#NO_GUARANTEE}. * * @param offsetCommitPeriodMs time in ms @@ -164,6 +182,7 @@ public Builder setOffsetCommitPeriodMs(long offsetCommitPeriodMs) { /** * Specifies the group id. + * * @param groupId the group id */ public Builder setGroupId(String groupId) { @@ -172,13 +191,17 @@ public Builder setGroupId(String groupId) { } /** - * Defines the max number of polled offsets (records) that can be pending commit, before another poll can take place. - * Once this limit is reached, no more offsets (records) can be polled until the next successful commit(s) sets the number - * of pending offsets below the threshold. The default is {@link #DEFAULT_MAX_UNCOMMITTED_OFFSETS}. + * Defines the max number of polled offsets (records) that can be pending commit, before + * another poll can take place. + * Once this limit is reached, no more offsets (records) can be polled until the next + * successful commit(s) sets the number + * of pending offsets below the threshold. The default is {@link + * #DEFAULT_MAX_UNCOMMITTED_OFFSETS}. * This limit is per partition and may in some cases be exceeded, * but each partition cannot exceed this limit by more than maxPollRecords - 1. * - *

    This setting only has an effect if the configured {@link ProcessingGuarantee} is {@link ProcessingGuarantee#AT_LEAST_ONCE}. + *

    This setting only has an effect if the configured {@link ProcessingGuarantee} is + * {@link ProcessingGuarantee#AT_LEAST_ONCE}. * * @param maxUncommittedOffsets max number of records that can be be pending commit */ @@ -190,7 +213,8 @@ public Builder setMaxUncommittedOffsets(int maxUncommittedOffsets) { /** * Sets the retry service for the spout to use. * - *

    This setting only has an effect if the configured {@link ProcessingGuarantee} is {@link ProcessingGuarantee#AT_LEAST_ONCE}. + *

    This setting only has an effect if the configured {@link ProcessingGuarantee} is + * {@link ProcessingGuarantee#AT_LEAST_ONCE}. * * @param retryService the new retry service * @return the builder (this). @@ -218,7 +242,8 @@ public Builder setTupleListener(KafkaTupleListener tupleListener) { } /** - * Specifies if the spout should emit null tuples to the component downstream, or rather not emit and directly ack them. By default + * Specifies if the spout should emit null tuples to the component downstream, or rather not + * emit and directly ack them. By default * this parameter is set to false, which means that null tuples are not emitted. * * @param emitNullTuples sets if null tuples should or not be emitted downstream @@ -229,7 +254,8 @@ public Builder setEmitNullTuples(boolean emitNullTuples) { } /** - * Specifies which processing guarantee the spout should offer. Refer to the documentation for {@link ProcessingGuarantee}. + * Specifies which processing guarantee the spout should offer. Refer to the documentation + * for {@link ProcessingGuarantee}. * * @param processingGuarantee The processing guarantee the spout should offer. */ @@ -239,12 +265,16 @@ public Builder setProcessingGuarantee(ProcessingGuarantee processingGuaran } /** - * Specifies whether the spout should require Storm to track emitted tuples when using a {@link ProcessingGuarantee} other than - * {@link ProcessingGuarantee#AT_LEAST_ONCE}. The spout will always track emitted tuples when offering at-least-once guarantees + * Specifies whether the spout should require Storm to track emitted tuples when using a + * {@link ProcessingGuarantee} other than + * {@link ProcessingGuarantee#AT_LEAST_ONCE}. The spout will always track emitted tuples + * when offering at-least-once guarantees * regardless of this setting. This setting is false by default. * - *

    Enabling tracking can be useful even in cases where reliability is not a concern, because it allows - * {@link Config#TOPOLOGY_MAX_SPOUT_PENDING} to have an effect, and enables some spout metrics (e.g. complete-latency) that would + *

    Enabling tracking can be useful even in cases where reliability is not a concern, + * because it allows + * {@link Config#TOPOLOGY_MAX_SPOUT_PENDING} to have an effect, and enables some spout + * metrics (e.g. complete-latency) that would * otherwise be disabled. * * @param tupleTrackingEnforced true if Storm should track emitted tuples, false otherwise @@ -256,6 +286,7 @@ public Builder setTupleTrackingEnforced(boolean tupleTrackingEnforced) { /** * The time period that metrics data in bucketed into. + * * @param metricsTimeBucketSizeInSecs time in seconds */ public Builder setMetricsTimeBucketSizeInSecs(int metricsTimeBucketSizeInSecs) { @@ -271,34 +302,46 @@ private Builder withStringDeserializers() { private Builder setKafkaPropsForProcessingGuarantee() { if (getKafkaProps().containsKey(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG)) { - throw new IllegalStateException("The KafkaConsumer " + ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG + throw new IllegalStateException("The KafkaConsumer " + + ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG + " setting is not supported." - + " You can configure similar behavior through KafkaSpoutConfig.Builder.setProcessingGuarantee"); + + " You can configure similar behavior through " + + "KafkaSpoutConfig.Builder.setProcessingGuarantee"); } - String autoOffsetResetPolicy = (String) getKafkaProps().get(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG); + String autoOffsetResetPolicy = (String) getKafkaProps() + .get(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG); if (processingGuarantee == ProcessingGuarantee.AT_LEAST_ONCE) { if (autoOffsetResetPolicy == null) { /* - * If the user wants to explicitly set an auto offset reset policy, we should respect it, but when the spout is - * configured for at-least-once processing we should default to seeking to the earliest offset in case there's an offset - * out of range error, rather than seeking to the latest (Kafka's default). This type of error will typically happen + * If the user wants to explicitly set an auto offset reset policy, we should + * respect it, but when the spout is + * configured for at-least-once processing we should default to seeking to the + * earliest offset in case there's an offset + * out of range error, rather than seeking to the latest (Kafka's default). This + * type of error will typically happen * when the consumer requests an offset that was deleted. */ - LOG.info("Setting Kafka consumer property '{}' to 'earliest' to ensure at-least-once processing", + LOG.info("Setting Kafka consumer property '{}' to 'earliest' to ensure " + + "at-least-once processing", ConsumerConfig.AUTO_OFFSET_RESET_CONFIG); setProp(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); - } else if (!autoOffsetResetPolicy.equals("earliest") && !autoOffsetResetPolicy.equals("none")) { - LOG.warn("Cannot guarantee at-least-once processing with auto.offset.reset.policy other than 'earliest' or 'none'." + } else if (!autoOffsetResetPolicy.equals("earliest") && !autoOffsetResetPolicy + .equals("none")) { + LOG.warn("Cannot guarantee at-least-once processing with " + + "auto.offset.reset.policy other than 'earliest' or 'none'." + " Some messages may be skipped."); } } else if (processingGuarantee == ProcessingGuarantee.AT_MOST_ONCE) { if (autoOffsetResetPolicy != null - && (!autoOffsetResetPolicy.equals("latest") && !autoOffsetResetPolicy.equals("none"))) { - LOG.warn("Cannot guarantee at-most-once processing with auto.offset.reset.policy other than 'latest' or 'none'." + && (!autoOffsetResetPolicy.equals("latest") && !autoOffsetResetPolicy + .equals("none"))) { + LOG.warn("Cannot guarantee at-most-once processing with " + + "auto.offset.reset.policy other than 'latest' or 'none'." + " Some messages may be processed more than once."); } } - LOG.info("Setting Kafka consumer property '{}' to 'false', because the spout does not support auto-commit", + LOG.info("Setting Kafka consumer property '{}' to 'false', because the spout does not " + + "support auto-commit", ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG); setProp(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false); return this; diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutMessageId.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutMessageId.java index ddf6391fc1e..13f0422082a 100644 --- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutMessageId.java +++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutMessageId.java @@ -27,7 +27,7 @@ public class KafkaSpoutMessageId implements Serializable { private final long offset; private int numFails = 0; /** - * false if the record was emitted using a form of collector.emit(...). true + * False if the record was emitted using a form of collector.emit(...). true * when skipping null tuples as configured by the user in KafkaSpoutConfig */ private boolean nullTuple; @@ -37,7 +37,8 @@ public KafkaSpoutMessageId(ConsumerRecord consumerRecord) { } public KafkaSpoutMessageId(ConsumerRecord consumerRecord, boolean nullTuple) { - this(new TopicPartition(consumerRecord.topic(), consumerRecord.partition()), consumerRecord.offset(), nullTuple); + this(new TopicPartition(consumerRecord.topic(), consumerRecord.partition()), consumerRecord + .offset(), nullTuple); } public KafkaSpoutMessageId(TopicPartition topicPart, long offset) { @@ -46,6 +47,7 @@ public KafkaSpoutMessageId(TopicPartition topicPart, long offset) { /** * Creates a new KafkaSpoutMessageId. + * * @param topicPart The topic partition this message belongs to * @param offset The offset of this message * @param nullTuple True if this message is being skipped as a null tuple diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutRetryExponentialBackoff.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutRetryExponentialBackoff.java index 8a2f54303e1..df244431e23 100644 --- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutRetryExponentialBackoff.java +++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutRetryExponentialBackoff.java @@ -35,22 +35,29 @@ import org.slf4j.LoggerFactory; /** - * Implementation of {@link KafkaSpoutRetryService} using the exponential backoff formula. The time of the nextRetry is set as follows: - * nextRetry = failCount == 1 ? currentTime + initialDelay : currentTime + delayPeriod*2^(failCount-1) where failCount = 1, 2, 3, ... + * Implementation of {@link KafkaSpoutRetryService} using the exponential backoff formula. The time + * of the nextRetry is set as follows: + * nextRetry = failCount == 1 ? currentTime + initialDelay : currentTime + + * delayPeriod*2^(failCount-1) where failCount = 1, 2, 3, ... * nextRetry = Min(nextRetry, currentTime + maxDelay) */ public class KafkaSpoutRetryExponentialBackoff implements KafkaSpoutRetryService { - private static final Logger LOG = LoggerFactory.getLogger(KafkaSpoutRetryExponentialBackoff.class); - private static final RetryEntryTimeStampComparator RETRY_ENTRY_TIME_STAMP_COMPARATOR = new RetryEntryTimeStampComparator(); + private static final Logger LOG = LoggerFactory + .getLogger(KafkaSpoutRetryExponentialBackoff.class); + private static final RetryEntryTimeStampComparator RETRY_ENTRY_TIME_STAMP_COMPARATOR = + new RetryEntryTimeStampComparator(); private final TimeInterval initialDelay; private final TimeInterval delayPeriod; private final TimeInterval maxDelay; private final int maxRetries; - //This class assumes that there is at most one retry schedule per message id in this set at a time. - private final Set retrySchedules = new TreeSet<>(RETRY_ENTRY_TIME_STAMP_COMPARATOR); - private final Set toRetryMsgs = new HashSet<>(); // Convenience data structure to speedup lookups + // This class assumes that there is at most one retry schedule per message id in this set at a + // time. + private final Set retrySchedules = + new TreeSet<>(RETRY_ENTRY_TIME_STAMP_COMPARATOR); + private final Set toRetryMsgs = + new HashSet<>(); // Convenience data structure to speedup lookups /** * Comparator ordering by timestamp. @@ -58,11 +65,12 @@ public class KafkaSpoutRetryExponentialBackoff implements KafkaSpoutRetryService private static class RetryEntryTimeStampComparator implements Serializable, Comparator { @Override public int compare(RetrySchedule entry1, RetrySchedule entry2) { - int result = Long.valueOf(entry1.nextRetryTimeNanos()).compareTo(entry2.nextRetryTimeNanos()); + int result = Long.valueOf(entry1.nextRetryTimeNanos()).compareTo(entry2 + .nextRetryTimeNanos()); if (result == 0) { - //TreeSet uses compareTo instead of equals() for the Set contract - //Ensure that we can save two retry schedules with the same timestamp + // TreeSet uses compareTo instead of equals() for the Set contract + // Ensure that we can save two retry schedules with the same timestamp result = entry1.hashCode() - entry2.hashCode(); } return result; @@ -112,6 +120,7 @@ public static class TimeInterval implements Serializable { /** * Creates a new TimeInterval. + * * @param length length of the time interval in the units specified by {@link TimeUnit} * @param timeUnit unit used to specify a time interval on which to specify a time unit */ @@ -151,20 +160,26 @@ public String toString() { } /** - * The time stamp of the next retry is scheduled according to the exponential backoff formula (geometric progression): - * nextRetry = failCount == 1 ? currentTime + initialDelay : currentTime + delayPeriod^(failCount-1), + * The time stamp of the next retry is scheduled according to the exponential backoff formula + * (geometric progression): + * nextRetry = failCount == 1 ? currentTime + initialDelay : currentTime + + * delayPeriod^(failCount-1), * where failCount = 1, 2, 3, ... nextRetry = Min(nextRetry, currentTime + maxDelay). *

    - * By specifying a value for maxRetries lower than Integer.MAX_VALUE, the user decides to sacrifice guarantee of delivery for the + * By specifying a value for maxRetries lower than Integer.MAX_VALUE, the user decides to + * sacrifice guarantee of delivery for the * previous polled records in favor of processing more records. * * @param initialDelay initial delay of the first retry - * @param delayPeriod the time interval that is the ratio of the exponential backoff formula (geometric progression) - * @param maxRetries maximum number of times a tuple is retried before being acked and scheduled for commit + * @param delayPeriod the time interval that is the ratio of the exponential backoff formula + * (geometric progression) + * @param maxRetries maximum number of times a tuple is retried before being acked and scheduled + * for commit * @param maxDelay maximum amount of time waiting before retrying * */ - public KafkaSpoutRetryExponentialBackoff(TimeInterval initialDelay, TimeInterval delayPeriod, int maxRetries, TimeInterval maxDelay) { + public KafkaSpoutRetryExponentialBackoff(TimeInterval initialDelay, TimeInterval delayPeriod, + int maxRetries, TimeInterval maxDelay) { this.initialDelay = initialDelay; this.delayPeriod = delayPeriod; this.maxRetries = maxRetries; @@ -179,13 +194,15 @@ public Map earliestRetriableOffsets() { for (RetrySchedule retrySchedule : retrySchedules) { if (retrySchedule.retry(currentTimeNanos)) { final KafkaSpoutMessageId msgId = retrySchedule.msgId; - final TopicPartition tpForMessage = new TopicPartition(msgId.topic(), msgId.partition()); + final TopicPartition tpForMessage = new TopicPartition(msgId.topic(), msgId + .partition()); tpToEarliestRetriableOffset.merge(tpForMessage, msgId.offset(), Math::min); } else { break; // Stop searching as soon as passed current time } } - LOG.debug("Topic partitions with entries ready to be retried [{}] ", tpToEarliestRetriableOffset); + LOG.debug("Topic partitions with entries ready to be retried [{}] ", + tpToEarliestRetriableOffset); return tpToEarliestRetriableOffset; } @@ -199,7 +216,7 @@ public boolean isReady(KafkaSpoutMessageId msgId) { if (retrySchedule.msgId.equals(msgId)) { retry = true; LOG.debug("Found entry to retry {}", retrySchedule); - break; //Stop searching if the message is known to be ready for retry + break; // Stop searching if the message is known to be ready for retry } } else { LOG.debug("Entry to retry not found {}", retrySchedule); @@ -220,12 +237,13 @@ public boolean remove(KafkaSpoutMessageId msgId) { boolean removed = false; if (isScheduled(msgId)) { toRetryMsgs.remove(msgId); - for (Iterator iterator = retrySchedules.iterator(); iterator.hasNext(); ) { + for (Iterator iterator = retrySchedules.iterator(); iterator + .hasNext(); ) { final RetrySchedule retrySchedule = iterator.next(); if (retrySchedule.msgId().equals(msgId)) { iterator.remove(); removed = true; - break; //There is at most one schedule per message id + break; // There is at most one schedule per message id } } } @@ -237,7 +255,8 @@ public boolean remove(KafkaSpoutMessageId msgId) { @Override public boolean retainAll(Collection topicPartitions) { boolean result = false; - for (Iterator rsIterator = retrySchedules.iterator(); rsIterator.hasNext(); ) { + for (Iterator rsIterator = retrySchedules.iterator(); rsIterator + .hasNext(); ) { final RetrySchedule retrySchedule = rsIterator.next(); final KafkaSpoutMessageId msgId = retrySchedule.msgId; final TopicPartition tpRetry = new TopicPartition(msgId.topic(), msgId.partition()); @@ -255,10 +274,11 @@ public boolean retainAll(Collection topicPartitions) { @Override public boolean schedule(KafkaSpoutMessageId msgId) { if (msgId.numFails() > maxRetries) { - LOG.debug("Not scheduling [{}] because reached maximum number of retries [{}].", msgId, maxRetries); + LOG.debug("Not scheduling [{}] because reached maximum number of retries [{}].", msgId, + maxRetries); return false; } else { - //Remove existing schedule for the message id + // Remove existing schedule for the message id remove(msgId); final RetrySchedule retrySchedule = new RetrySchedule(msgId, nextTime(msgId)); retrySchedules.add(retrySchedule); @@ -277,7 +297,7 @@ public int readyMessageCount() { if (retrySchedule.retry(currentTimeNanos)) { ++count; } else { - break; //Stop counting when past current time + break; // Stop counting when past current time } } return count; @@ -298,11 +318,13 @@ public KafkaSpoutMessageId getMessageId(TopicPartition tp, long offset) { // if value is greater than Long.MAX_VALUE it truncates to Long.MAX_VALUE private long nextTime(KafkaSpoutMessageId msgId) { - Validate.isTrue(msgId.numFails() > 0, "nextTime assumes the message has failed at least once"); + Validate.isTrue(msgId.numFails() > 0, + "nextTime assumes the message has failed at least once"); final long currentTimeNanos = Time.nanoTime(); final long nextTimeNanos = msgId.numFails() == 1 // numFails = 1, 2, 3, ... ? currentTimeNanos + initialDelay.lengthNanos - : currentTimeNanos + delayPeriod.lengthNanos * (long) (Math.pow(2, msgId.numFails() - 1)); + : currentTimeNanos + delayPeriod.lengthNanos * (long) (Math.pow(2, msgId + .numFails() - 1)); return Math.min(nextTimeNanos, currentTimeNanos + maxDelay.lengthNanos); } @@ -312,7 +334,7 @@ public String toString() { } private String toStringImpl() { - //This is here to avoid an overridable call in the constructor + // This is here to avoid an overridable call in the constructor return "KafkaSpoutRetryExponentialBackoff{" + "delay=" + initialDelay + ", ratio=" + delayPeriod diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutRetryService.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutRetryService.java index 66b551eca2f..1ff669395bf 100644 --- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutRetryService.java +++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutRetryService.java @@ -31,6 +31,7 @@ public interface KafkaSpoutRetryService extends Serializable { * Schedules this {@link KafkaSpoutMessageId} if not yet scheduled, or * updates retry time if it has already been scheduled. It may also indicate * that the message should not be retried, in which case the message will not be scheduled. + * * @param msgId message to schedule for retrial * @return true if the message will be retried, false otherwise */ @@ -38,6 +39,7 @@ public interface KafkaSpoutRetryService extends Serializable { /** * Removes a message from the list of messages scheduled for retrial. + * * @param msgId message to remove from retrial * @return true if the message was scheduled for retrial, false otherwise */ @@ -45,8 +47,10 @@ public interface KafkaSpoutRetryService extends Serializable { /** * Retains all the messages whose {@link TopicPartition} belongs to the specified {@code Collection}. - * All messages that come from a {@link TopicPartition} NOT existing in the collection will be removed. + * All messages that come from a {@link TopicPartition} NOT existing in the collection will be + * removed. * This method is useful to cleanup state following partition rebalance. + * * @param topicPartitions Collection of {@link TopicPartition} for which to keep messages * @return true if at least one message was removed, false otherwise */ @@ -54,6 +58,7 @@ public interface KafkaSpoutRetryService extends Serializable { /** * Gets the earliest retriable offsets. + * * @return The earliest retriable offset for each TopicPartition that has * offsets ready to be retried, i.e. for which a tuple has failed * and has retry time less than current time. @@ -63,6 +68,7 @@ public interface KafkaSpoutRetryService extends Serializable { /** * Checks if a specific failed {@link KafkaSpoutMessageId} is ready to be retried, * i.e is scheduled and has retry time that is less than current time. + * * @param msgId message to check for readiness * @return true if message is ready to be retried, false otherwise */ @@ -71,20 +77,24 @@ public interface KafkaSpoutRetryService extends Serializable { /** * Checks if a specific failed {@link KafkaSpoutMessageId} is scheduled to be retried. * The message may or may not be ready to be retried yet. + * * @param msgId message to check for scheduling status - * @return true if the message is scheduled to be retried, regardless of being or not ready to be retried. + * @return true if the message is scheduled to be retried, regardless of being or not ready to + * be retried. * Returns false is this message is not scheduled for retrial */ boolean isScheduled(KafkaSpoutMessageId msgId); /** * Get the number of messages ready for retry. + * * @return The number of messages that are ready for retry */ int readyMessageCount(); /** * Gets the {@link KafkaSpoutMessageId} for the record on the given topic partition and offset. + * * @param topicPartition The topic partition of the record * @param offset The offset of the record * @return The id the record was scheduled for retry with, diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaTuple.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaTuple.java index 2efb6a332c0..808f59b03ca 100644 --- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaTuple.java +++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaTuple.java @@ -38,6 +38,7 @@ public KafkaTuple(Object... vals) { /** * Sets the target stream of this Tuple. + * * @param stream The target stream * @return This */ diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaTupleListener.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaTupleListener.java index 10014831285..ed739d25b22 100644 --- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaTupleListener.java +++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaTupleListener.java @@ -25,13 +25,11 @@ import org.apache.kafka.common.TopicPartition; import org.apache.storm.task.TopologyContext; - /** * The KafkaTupleListener handles state changes of a kafka tuple inside a KafkaSpout. */ public interface KafkaTupleListener extends Serializable { - /** * Called during the initialization of the kafka spout. * @@ -42,8 +40,10 @@ public interface KafkaTupleListener extends Serializable { /** * Called when the tuple is emitted and auto commit is disabled. - * If kafka auto commit is enabled, the kafka consumer will periodically (depending on the commit interval) - * commit the offsets. Therefore, storm disables anchoring for tuples when auto commit is enabled and the spout will + * If kafka auto commit is enabled, the kafka consumer will periodically (depending on the + * commit interval) + * commit the offsets. Therefore, storm disables anchoring for tuples when auto commit is + * enabled and the spout will * not receive acks and fails for those tuples. * * @param tuple the storm tuple. @@ -61,7 +61,8 @@ public interface KafkaTupleListener extends Serializable { /** * Called when kafka partitions are rebalanced. * - * @param partitions The list of partitions that are now assigned to the consumer (may include partitions previously + * @param partitions The list of partitions that are now assigned to the consumer (may include + * partitions previously * assigned to the consumer) */ void onPartitionsReassigned(Collection partitions); diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/RecordTranslator.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/RecordTranslator.java index 0b48e48a20a..f99887d30e4 100644 --- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/RecordTranslator.java +++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/RecordTranslator.java @@ -33,6 +33,7 @@ public interface RecordTranslator extends Serializable, Func extends Serializable, Func extends Serializable, Func streams() { diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/SimpleRecordTranslator.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/SimpleRecordTranslator.java index a451afe1e2a..88003b5364e 100644 --- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/SimpleRecordTranslator.java +++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/SimpleRecordTranslator.java @@ -35,11 +35,13 @@ public SimpleRecordTranslator(Func, List> func, Fie /** * Creates a SimpleRecordTranslator. + * * @param func The mapping function responsible for translating a Kafka record to a Tuple * @param fields The fields tuples constructed by this translator will contain * @param stream The stream tuples constructed by this translator will target */ - public SimpleRecordTranslator(Func, List> func, Fields fields, String stream) { + public SimpleRecordTranslator(Func, List> func, Fields fields, + String stream) { this.func = func; this.fields = fields; this.stream = stream; diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/TopicPartitionComparator.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/TopicPartitionComparator.java index 91b7243a947..163b9feb4f3 100644 --- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/TopicPartitionComparator.java +++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/TopicPartitionComparator.java @@ -35,7 +35,7 @@ public class TopicPartitionComparator implements Comparator { * Private to make it a singleton. */ private TopicPartitionComparator() { - //Empty + // Empty } @Override diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/internal/ClientFactory.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/internal/ClientFactory.java index 2aafd3e8b01..1a8106d57db 100644 --- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/internal/ClientFactory.java +++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/internal/ClientFactory.java @@ -18,7 +18,6 @@ import java.io.Serializable; import java.util.Map; - import org.apache.kafka.clients.admin.Admin; import org.apache.kafka.clients.consumer.Consumer; diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/internal/ClientFactoryDefault.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/internal/ClientFactoryDefault.java index a470ce51442..2667160e47c 100644 --- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/internal/ClientFactoryDefault.java +++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/internal/ClientFactoryDefault.java @@ -17,7 +17,6 @@ package org.apache.storm.kafka.spout.internal; import java.util.Map; - import org.apache.kafka.clients.admin.Admin; import org.apache.kafka.clients.admin.KafkaAdminClient; import org.apache.kafka.clients.consumer.KafkaConsumer; diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/internal/CommitMetadataManager.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/internal/CommitMetadataManager.java index a63619c6e6f..9d3ec7fc6ad 100644 --- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/internal/CommitMetadataManager.java +++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/internal/CommitMetadataManager.java @@ -56,14 +56,16 @@ public CommitMetadataManager(TopologyContext context, ProcessingGuarantee proces } /** - * Checks if {@link OffsetAndMetadata} was committed by a {@link KafkaSpout} instance in this topology. + * Checks if {@link OffsetAndMetadata} was committed by a {@link KafkaSpout} instance in this + * topology. * * @param tp The topic partition the commit metadata belongs to. * @param committedOffset {@link OffsetAndMetadata} info committed to Kafka * @param offsetManagers The offset managers. * @return true if this topology committed this {@link OffsetAndMetadata}, false otherwise */ - public boolean isOffsetCommittedByThisTopology(TopicPartition tp, OffsetAndMetadata committedOffset, + public boolean isOffsetCommittedByThisTopology(TopicPartition tp, + OffsetAndMetadata committedOffset, Map offsetManagers) { try { if (processingGuarantee == ProcessingGuarantee.AT_LEAST_ONCE @@ -72,12 +74,15 @@ public boolean isOffsetCommittedByThisTopology(TopicPartition tp, OffsetAndMetad return true; } - final CommitMetadata committedMetadata = JSON_MAPPER.readValue(committedOffset.metadata(), CommitMetadata.class); + final CommitMetadata committedMetadata = JSON_MAPPER.readValue(committedOffset + .metadata(), CommitMetadata.class); return committedMetadata.getTopologyId().equals(context.getStormId()); } catch (IOException e) { LOG.warn("Failed to deserialize expected commit metadata [{}]." - + " This error is expected to occur once per partition, if the last commit to each partition" - + " was by an earlier version of the KafkaSpout, or by a process other than the KafkaSpout. " + + " This error is expected to occur once per partition, if the last commit to " + + "each partition" + + " was by an earlier version of the KafkaSpout, or by a process other than the " + + "KafkaSpout. " + "Defaulting to behavior compatible with earlier version", committedOffset); LOG.trace("", e); return false; diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/internal/CommonKafkaSpoutConfig.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/internal/CommonKafkaSpoutConfig.java index ad31f40f92f..e9f5c3532d1 100644 --- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/internal/CommonKafkaSpoutConfig.java +++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/internal/CommonKafkaSpoutConfig.java @@ -47,7 +47,8 @@ public abstract class CommonKafkaSpoutConfig implements Serializable { public static final long DEFAULT_PARTITION_REFRESH_PERIOD_MS = 2_000; // Earliest start public static final long DEFAULT_START_TS = 0L; - public static final FirstPollOffsetStrategy DEFAULT_FIRST_POLL_OFFSET_STRATEGY = FirstPollOffsetStrategy.UNCOMMITTED_EARLIEST; + public static final FirstPollOffsetStrategy DEFAULT_FIRST_POLL_OFFSET_STRATEGY = + FirstPollOffsetStrategy.UNCOMMITTED_EARLIEST; public static final Logger LOG = LoggerFactory.getLogger(CommonKafkaSpoutConfig.class); @@ -86,7 +87,8 @@ public abstract static class Builder> { private final ManualPartitioner topicPartitioner; private RecordTranslator translator; private long pollTimeoutMs = DEFAULT_POLL_TIMEOUT_MS; - private FirstPollOffsetStrategy firstPollOffsetStrategy = DEFAULT_FIRST_POLL_OFFSET_STRATEGY; + private FirstPollOffsetStrategy firstPollOffsetStrategy = + DEFAULT_FIRST_POLL_OFFSET_STRATEGY; private long partitionRefreshPeriodMs = DEFAULT_PARTITION_REFRESH_PERIOD_MS; private long startTimeStamp = DEFAULT_START_TS; @@ -99,17 +101,22 @@ public Builder(String bootstrapServers, Set topics) { } public Builder(String bootstrapServers, Pattern topics) { - this(bootstrapServers, new PatternTopicFilter(topics), new RoundRobinManualPartitioner()); + this(bootstrapServers, new PatternTopicFilter(topics), + new RoundRobinManualPartitioner()); } /** - * Create a KafkaSpoutConfig builder with default property values and no key/value deserializers. + * Create a KafkaSpoutConfig builder with default property values and no key/value + * deserializers. * * @param bootstrapServers The bootstrap servers the consumer will use - * @param topicFilter The topic filter defining which topics and partitions the spout will read - * @param topicPartitioner The topic partitioner defining which topics and partitions are assinged to each spout task + * @param topicFilter The topic filter defining which topics and partitions the spout will + * read + * @param topicPartitioner The topic partitioner defining which topics and partitions are + * assinged to each spout task */ - public Builder(String bootstrapServers, TopicFilter topicFilter, ManualPartitioner topicPartitioner) { + public Builder(String bootstrapServers, TopicFilter topicFilter, + ManualPartitioner topicPartitioner) { kafkaProps = new HashMap<>(); if (bootstrapServers == null || bootstrapServers.isEmpty()) { throw new IllegalArgumentException("bootstrap servers cannot be null"); @@ -144,15 +151,17 @@ public T setProp(Properties props) { if (key instanceof String) { kafkaProps.put((String) key, value); } else { - throw new IllegalArgumentException("Kafka Consumer property keys must be Strings"); + throw new IllegalArgumentException("Kafka Consumer property keys must be " + + "Strings"); } }); return (T) this; } - //Spout Settings + // Spout Settings /** - * Specifies the time, in milliseconds, spent waiting in poll if data is not available. Default is 200ms. + * Specifies the time, in milliseconds, spent waiting in poll if data is not available. + * Default is 200ms. * * @param pollTimeoutMs time in ms */ @@ -162,7 +171,8 @@ public T setPollTimeoutMs(long pollTimeoutMs) { } /** - * Sets the offset used by the Kafka spout in the first poll to Kafka broker upon process start. Please refer to to the + * Sets the offset used by the Kafka spout in the first poll to Kafka broker upon process + * start. Please refer to to the * documentation in {@link FirstPollOffsetStrategy} * * @param firstPollOffsetStrategy Offset used by Kafka spout first poll @@ -180,7 +190,8 @@ public T setRecordTranslator(RecordTranslator translator) { /** * Configure a translator with tuples to be emitted on the default stream. * - * @param func extracts and turns a Kafka ConsumerRecord into a list of objects to be emitted + * @param func extracts and turns a Kafka ConsumerRecord into a list of objects to be + * emitted * @param fields the names of the fields extracted * @return this to be able to chain configuration */ @@ -191,17 +202,20 @@ public T setRecordTranslator(Func, List> func, Fiel /** * Configure a translator with tuples to be emitted to a given stream. * - * @param func extracts and turns a Kafka ConsumerRecord into a list of objects to be emitted + * @param func extracts and turns a Kafka ConsumerRecord into a list of objects to be + * emitted * @param fields the names of the fields extracted * @param stream the stream to emit the tuples on * @return this to be able to chain configuration */ - public T setRecordTranslator(Func, List> func, Fields fields, String stream) { + public T setRecordTranslator(Func, List> func, Fields fields, + String stream) { return setRecordTranslator(new SimpleRecordTranslator<>(func, fields, stream)); } /** - * Sets partition refresh period in milliseconds. This is how often Kafka will be polled to check for new topics and/or new + * Sets partition refresh period in milliseconds. This is how often Kafka will be polled to + * check for new topics and/or new * partitions. * * @param partitionRefreshPeriodMs time in milliseconds @@ -213,7 +227,9 @@ public T setPartitionRefreshPeriodMs(long partitionRefreshPeriodMs) { } /** - * Specifies the startTimeStamp if the first poll strategy is TIMESTAMP or UNCOMMITTED_TIMESTAMP. + * Specifies the startTimeStamp if the first poll strategy is TIMESTAMP or + * UNCOMMITTED_TIMESTAMP. + * * @param startTimeStamp time in ms */ public T setStartTimeStamp(long startTimeStamp) { diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/internal/OffsetManager.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/internal/OffsetManager.java index e9554d7f81b..232b06c797f 100755 --- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/internal/OffsetManager.java +++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/internal/OffsetManager.java @@ -40,7 +40,8 @@ public class OffsetManager { private final NavigableSet emittedOffsets = new TreeSet<>(); // Acked messages sorted by ascending order of offset private final NavigableSet ackedMsgs = new TreeSet<>(OFFSET_COMPARATOR); - // Committed offset, i.e. the offset where processing will resume upon spout restart. Initially it is set to fetchOffset. + // Committed offset, i.e. the offset where processing will resume upon spout restart. Initially + // it is set to fetchOffset. private long committedOffset; // True if this OffsetManager has made at least one commit to Kafka private boolean committed; @@ -48,6 +49,7 @@ public class OffsetManager { /** * Creates a new OffsetManager. + * * @param tp The TopicPartition * @param initialFetchOffset The initial fetch offset for the given TopicPartition */ @@ -92,9 +94,11 @@ public long getNthUncommittedOffsetAfterCommittedOffset(int index) { * acked. This guarantees that all offsets smaller than the committedOffset * have been delivered, or that those offsets no longer exist in Kafka. *

    - * The returned offset points to the earliest uncommitted offset, and matches the semantics of the KafkaConsumer.commitSync API. + * The returned offset points to the earliest uncommitted offset, and matches the semantics of + * the KafkaConsumer.commitSync API. * - * @param commitMetadata Metadata information to commit to Kafka. It is constant per KafkaSpout instance per topology + * @param commitMetadata Metadata information to commit to Kafka. It is constant per KafkaSpout + * instance per topology * @return the next OffsetAndMetadata to commit, or null if no offset is * ready to commit. */ @@ -127,7 +131,8 @@ public OffsetAndMetadata findNextCommitOffset(final String commitMetadata) { + " Missing offset: [{}], Processed: [{}]", nextCommitOffset, currOffset); final Long nextEmittedOffset = emittedOffsets.ceiling(nextCommitOffset); if (nextEmittedOffset != null && currOffset == nextEmittedOffset) { - LOG.debug("Found committable offset: [{}] after missing offset: [{}], skipping to the committable offset", + LOG.debug("Found committable offset: [{}] after missing offset: [{}], " + + "skipping to the committable offset", currOffset, nextCommitOffset); found = true; nextCommitOffset = currOffset + 1; @@ -138,9 +143,11 @@ public OffsetAndMetadata findNextCommitOffset(final String commitMetadata) { } } } else { - throw new IllegalStateException("The offset [" + currOffset + "] is below the current nextCommitOffset " + throw new IllegalStateException("The offset [" + currOffset + + "] is below the current nextCommitOffset " + "[" + nextCommitOffset + "] for [" + tp + "]." - + " This should not be possible, and likely indicates a bug in the spout's acking or emit logic."); + + " This should not be possible, and likely indicates a bug in the spout's " + + "acking or emit logic."); } } @@ -150,7 +157,8 @@ public OffsetAndMetadata findNextCommitOffset(final String commitMetadata) { LOG.debug("Topic-partition [{}] has offsets [{}-{}] ready to be committed." + " Processing will resume at offset [{}] upon spout restart", - tp, committedOffset, nextCommitOffsetAndMetadata.offset() - 1, nextCommitOffsetAndMetadata.offset()); + tp, committedOffset, nextCommitOffsetAndMetadata + .offset() - 1, nextCommitOffsetAndMetadata.offset()); } else { LOG.debug("Topic-partition [{}] has no offsets ready to be committed", tp); } @@ -164,7 +172,8 @@ public OffsetAndMetadata findNextCommitOffset(final String commitMetadata) { * {@link #findNextCommitOffset(String)} will return offsets greater than or equal to the * offset specified, if any. * - * @param committedOffsetAndMeta The committed offset. All lower offsets are expected to have been committed. + * @param committedOffsetAndMeta The committed offset. All lower offsets are expected to have + * been committed. * @return Number of offsets committed in this commit */ public long commit(OffsetAndMetadata committedOffsetAndMeta) { diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/internal/Timer.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/internal/Timer.java index 371cb82cde6..64acfc4612f 100644 --- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/internal/Timer.java +++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/internal/Timer.java @@ -29,9 +29,11 @@ public class Timer { private long start; /** - * Creates a class that mimics a single threaded timer that expires periodically. If a call to {@link + * Creates a class that mimics a single threaded timer that expires periodically. If a call to + * {@link * #isExpiredResetOnTrue()} occurs later than {@code period} since the timer was initiated or reset, this method returns - * true. Each time the method returns true the counter is reset. The timer starts with the specified time delay. + * true. Each time the method returns true the counter is reset. The timer starts with the + * specified time delay. * * @param delay the initial delay before the timer starts * @param period the period between calls {@link #isExpiredResetOnTrue()} @@ -60,7 +62,8 @@ public TimeUnit getTimeUnit() { /** * Checks if a call to this method occurs later than {@code period} since the timer was initiated or reset. If that is the - * case the method returns true, otherwise it returns false. Each time this method returns true, the counter is reset + * case the method returns true, otherwise it returns false. Each time this method returns true, + * the counter is reset * (re-initiated) and a new cycle will start. * * @return true if the time elapsed since the last call returning true is greater than {@code period}. Returns false diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/metrics2/KafkaOffsetMetricManager.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/metrics2/KafkaOffsetMetricManager.java index 64c42832cf3..1d4d89d323f 100644 --- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/metrics2/KafkaOffsetMetricManager.java +++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/metrics2/KafkaOffsetMetricManager.java @@ -6,10 +6,10 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - *

    - * http://www.apache.org/licenses/LICENSE-2.0 - *

    - * Unless required by applicable law or agreed to in writing, software + * + *

    http://www.apache.org/licenses/LICENSE-2.0 + * + *

    Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -29,8 +29,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; - - /** * This class is used to manage both the partition and topic level offset metrics. */ @@ -43,7 +41,8 @@ public class KafkaOffsetMetricManager { private Map topicMetricsMap; private Map topicPartitionMetricsMap; - public KafkaOffsetMetricManager(Supplier> offsetManagerSupplier, + public KafkaOffsetMetricManager(Supplier> offsetManagerSupplier, Supplier adminSupplier, TopologyContext topologyContext) { this.offsetManagerSupplier = offsetManagerSupplier; @@ -65,13 +64,15 @@ public void registerMetricsForNewTopicPartitions(Set newAssignme KafkaOffsetTopicMetrics topicMetrics = topicMetricsMap.get(topic); if (topicMetrics == null) { LOG.info("Registering metric for topic: {}", topic); - topicMetrics = new KafkaOffsetTopicMetrics(topic, offsetManagerSupplier, adminSupplier, newAssignment); + topicMetrics = new KafkaOffsetTopicMetrics(topic, offsetManagerSupplier, + adminSupplier, newAssignment); topicMetricsMap.put(topic, topicMetrics); topologyContext.registerMetricSet("kafkaOffset", topicMetrics); } KafkaOffsetPartitionMetrics topicPartitionMetricSet - = new KafkaOffsetPartitionMetrics<>(offsetManagerSupplier, adminSupplier, topicPartition); + = new KafkaOffsetPartitionMetrics<>(offsetManagerSupplier, adminSupplier, + topicPartition); topicPartitionMetricsMap.put(topicPartition, topicPartitionMetricSet); topologyContext.registerMetricSet("kafkaOffset", topicPartitionMetricSet); } diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/metrics2/KafkaOffsetPartitionMetrics.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/metrics2/KafkaOffsetPartitionMetrics.java index df5b0c8694e..23b7044aa5e 100644 --- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/metrics2/KafkaOffsetPartitionMetrics.java +++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/metrics2/KafkaOffsetPartitionMetrics.java @@ -6,10 +6,10 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - *

    - * http://www.apache.org/licenses/LICENSE-2.0 - *

    - * Unless required by applicable law or agreed to in writing, software + * + *

    http://www.apache.org/licenses/LICENSE-2.0 + * + *

    Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -34,17 +34,17 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; - - - /** * Partition level metrics. - *

    - * topicName/partition_{number}/earliestTimeOffset //gives beginning offset of the partition + * + *

    topicName/partition_{number}/earliestTimeOffset //gives beginning offset of the partition * topicName/partition_{number}/latestTimeOffset //gives end offset of the partition - * topicName/partition_{number}/latestEmittedOffset //gives latest emitted offset of the partition from the spout - * topicName/partition_{number}/latestCompletedOffset //gives latest committed offset of the partition from the spout - * topicName/partition_{number}/spoutLag // the delta between the latest Offset and latestCompletedOffset + * topicName/partition_{number}/latestEmittedOffset //gives latest emitted offset of the partition + * from the spout + * topicName/partition_{number}/latestCompletedOffset //gives latest committed offset of the + * partition from the spout + * topicName/partition_{number}/spoutLag // the delta between the latest Offset and + * latestCompletedOffset * topicName/partition_{number}/recordsInPartition // total number of records in the partition *

    */ @@ -55,7 +55,8 @@ public class KafkaOffsetPartitionMetrics implements MetricSet { private TopicPartition topicPartition; - public KafkaOffsetPartitionMetrics(Supplier> offsetManagerSupplier, + public KafkaOffsetPartitionMetrics(Supplier> offsetManagerSupplier, Supplier adminSupplier, TopicPartition topicPartition) { this.offsetManagerSupplier = offsetManagerSupplier; @@ -71,9 +72,11 @@ public Map getMetrics() { String metricPath = topicPartition.topic() + "/partition_" + topicPartition.partition(); Gauge spoutLagGauge = () -> { - Map endOffsets = getEndOffsets(Collections.singleton(topicPartition), adminSupplier); + Map endOffsets = getEndOffsets(Collections + .singleton(topicPartition), adminSupplier); if (endOffsets == null || endOffsets.isEmpty()) { - LOG.error("Failed to get endOffsets from Kafka for topic partitions: {}.", topicPartition); + LOG.error("Failed to get endOffsets from Kafka for topic partitions: {}.", + topicPartition); return 0L; } OffsetManager offsetManager = offsetManagerSupplier.get().get(topicPartition); @@ -81,18 +84,22 @@ public Map getMetrics() { }; Gauge earliestTimeOffsetGauge = () -> { - Map beginningOffsets = getBeginningOffsets(Collections.singleton(topicPartition), adminSupplier); + Map beginningOffsets = getBeginningOffsets(Collections + .singleton(topicPartition), adminSupplier); if (beginningOffsets == null || beginningOffsets.isEmpty()) { - LOG.error("Failed to get beginningOffsets from Kafka for topic partitions: {}.", topicPartition); + LOG.error("Failed to get beginningOffsets from Kafka for topic partitions: {}.", + topicPartition); return 0L; } return beginningOffsets.get(topicPartition); }; Gauge latestTimeOffsetGauge = () -> { - Map endOffsets = getEndOffsets(Collections.singleton(topicPartition), adminSupplier); + Map endOffsets = getEndOffsets(Collections + .singleton(topicPartition), adminSupplier); if (endOffsets == null || endOffsets.isEmpty()) { - LOG.error("Failed to get endOffsets from Kafka for topic partitions: {}.", topicPartition); + LOG.error("Failed to get endOffsets from Kafka for topic partitions: {}.", + topicPartition); return 0L; } return endOffsets.get(topicPartition); @@ -109,14 +116,18 @@ public Map getMetrics() { }; Gauge recordsInPartitionGauge = () -> { - Map endOffsets = getEndOffsets(Collections.singleton(topicPartition), adminSupplier); + Map endOffsets = getEndOffsets(Collections + .singleton(topicPartition), adminSupplier); if (endOffsets == null || endOffsets.isEmpty()) { - LOG.error("Failed to get endOffsets from Kafka for topic partitions: {}.", topicPartition); + LOG.error("Failed to get endOffsets from Kafka for topic partitions: {}.", + topicPartition); return 0L; } - Map beginningOffsets = getBeginningOffsets(Collections.singleton(topicPartition), adminSupplier); + Map beginningOffsets = getBeginningOffsets(Collections + .singleton(topicPartition), adminSupplier); if (beginningOffsets == null || beginningOffsets.isEmpty()) { - LOG.error("Failed to get beginningOffsets from Kafka for topic partitions: {}.", topicPartition); + LOG.error("Failed to get beginningOffsets from Kafka for topic partitions: {}.", + topicPartition); return 0L; } return endOffsets.get(topicPartition) - beginningOffsets.get(topicPartition); diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/metrics2/KafkaOffsetTopicMetrics.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/metrics2/KafkaOffsetTopicMetrics.java index 8bdcef37485..657a8235325 100644 --- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/metrics2/KafkaOffsetTopicMetrics.java +++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/metrics2/KafkaOffsetTopicMetrics.java @@ -6,10 +6,10 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - *

    - * http://www.apache.org/licenses/LICENSE-2.0 - *

    - * Unless required by applicable law or agreed to in writing, software + * + *

    http://www.apache.org/licenses/LICENSE-2.0 + * + *

    Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -24,29 +24,31 @@ import com.codahale.metrics.Gauge; import com.codahale.metrics.Metric; import com.codahale.metrics.MetricSet; - import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.Set; import java.util.function.Supplier; - import org.apache.kafka.clients.admin.Admin; import org.apache.kafka.common.TopicPartition; import org.apache.storm.kafka.spout.internal.OffsetManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - /** * Topic level metrics. - *

    - * topicName/totalEarliestTimeOffset //gives the total beginning offset of all the associated partitions of this spout - * topicName/totalLatestTimeOffset //gives the total end offset of all the associated partitions of this spout - * topicName/totalLatestEmittedOffset //gives the total latest emitted offset of all the associated partitions of this spout - * topicName/totalLatestCompletedOffset //gives the total latest committed offset of all the associated partitions of this spout + * + *

    topicName/totalEarliestTimeOffset //gives the total beginning offset of all the associated + * partitions of this spout + * topicName/totalLatestTimeOffset //gives the total end offset of all the associated partitions of + * this spout + * topicName/totalLatestEmittedOffset //gives the total latest emitted offset of all the associated + * partitions of this spout + * topicName/totalLatestCompletedOffset //gives the total latest committed offset of all the + * associated partitions of this spout * topicName/spoutLag // total spout lag of all the associated partitions of this spout - * topicName/totalRecordsInPartitions //total number of records in all the associated partitions of this spout + * topicName/totalRecordsInPartitions //total number of records in all the associated partitions of + * this spout *

    */ public class KafkaOffsetTopicMetrics implements MetricSet { @@ -78,9 +80,11 @@ public Map getMetrics() { for (TopicPartition topicPartition : assignment) { String topicOfPartition = topicPartition.topic(); if (topicOfPartition.equals(topic)) { - Map endOffsets = getEndOffsets(Collections.singleton(topicPartition), adminSupplier); + Map endOffsets = getEndOffsets(Collections + .singleton(topicPartition), adminSupplier); if (endOffsets == null || endOffsets.isEmpty()) { - LOG.error("Failed to get endOffsets from Kafka for topic partitions: {}.", topicPartition); + LOG.error("Failed to get endOffsets from Kafka for topic partitions: {}.", + topicPartition); return 0L; } // add value to topic level metric @@ -99,9 +103,11 @@ public Map getMetrics() { for (TopicPartition topicPartition : assignment) { String topicOfPartition = topicPartition.topic(); if (topicOfPartition.equals(topic)) { - Map beginningOffsets = getBeginningOffsets(Collections.singleton(topicPartition), adminSupplier); + Map beginningOffsets = getBeginningOffsets(Collections + .singleton(topicPartition), adminSupplier); if (beginningOffsets == null || beginningOffsets.isEmpty()) { - LOG.error("Failed to get beginningOffsets from Kafka for topic partitions: {}.", topicPartition); + LOG.error("Failed to get beginningOffsets from Kafka for topic " + + "partitions: {}.", topicPartition); return 0L; } // add value to topic level metric @@ -119,9 +125,11 @@ public Map getMetrics() { for (TopicPartition topicPartition : assignment) { String topicOfPartition = topicPartition.topic(); if (topicOfPartition.equals(topic)) { - Map endOffsets = getEndOffsets(Collections.singleton(topicPartition), adminSupplier); + Map endOffsets = getEndOffsets(Collections + .singleton(topicPartition), adminSupplier); if (endOffsets == null || endOffsets.isEmpty()) { - LOG.error("Failed to get endOffsets from Kafka for topic partitions: {}.", topicPartition); + LOG.error("Failed to get endOffsets from Kafka for topic partitions: {}.", + topicPartition); return 0L; } // add value to topic level metric @@ -168,18 +176,23 @@ public Map getMetrics() { for (TopicPartition topicPartition : assignment) { String topicOfPartition = topicPartition.topic(); if (topicOfPartition.equals(topic)) { - Map endOffsets = getEndOffsets(Collections.singleton(topicPartition), adminSupplier); + Map endOffsets = getEndOffsets(Collections + .singleton(topicPartition), adminSupplier); if (endOffsets == null || endOffsets.isEmpty()) { - LOG.error("Failed to get endOffsets from Kafka for topic partitions: {}.", topicPartition); + LOG.error("Failed to get endOffsets from Kafka for topic partitions: {}.", + topicPartition); return 0L; } - Map beginningOffsets = getBeginningOffsets(Collections.singleton(topicPartition), adminSupplier); + Map beginningOffsets = getBeginningOffsets(Collections + .singleton(topicPartition), adminSupplier); if (beginningOffsets == null || beginningOffsets.isEmpty()) { - LOG.error("Failed to get beginningOffsets from Kafka for topic partitions: {}.", topicPartition); + LOG.error("Failed to get beginningOffsets from Kafka for topic " + + "partitions: {}.", topicPartition); return 0L; } // add value to topic level metric - Long ret = endOffsets.get(topicPartition) - beginningOffsets.get(topicPartition); + Long ret = endOffsets.get(topicPartition) - beginningOffsets + .get(topicPartition); totalRecordsInPartitions += ret; } } diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/metrics2/KafkaOffsetUtil.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/metrics2/KafkaOffsetUtil.java index c7f9fabc470..71cb8f27389 100644 --- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/metrics2/KafkaOffsetUtil.java +++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/metrics2/KafkaOffsetUtil.java @@ -6,10 +6,10 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - *

    - * http://www.apache.org/licenses/LICENSE-2.0 - *

    - * Unless required by applicable law or agreed to in writing, software + * + *

    http://www.apache.org/licenses/LICENSE-2.0 + * + *

    Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -33,12 +33,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; - public class KafkaOffsetUtil { private static final Logger LOG = LoggerFactory.getLogger(KafkaOffsetUtil.class); - public static Map getBeginningOffsets(Set topicPartitions, Supplier adminSupplier) { + public static Map getBeginningOffsets(Set topicPartitions, + Supplier adminSupplier) { Admin admin = adminSupplier.get(); if (admin == null) { LOG.error("Kafka admin object is null, returning 0."); @@ -49,13 +49,15 @@ public static Map getBeginningOffsets(Set try { beginningOffsets = getOffsets(admin, topicPartitions, OffsetSpec.earliest()); } catch (RetriableException | ExecutionException | InterruptedException e) { - LOG.error("Failed to get offset from Kafka for topic partitions: {}.", topicPartitions, e); + LOG.error("Failed to get offset from Kafka for topic partitions: {}.", topicPartitions, + e); return Collections.EMPTY_MAP; } return beginningOffsets; } - public static Map getEndOffsets(Set topicPartitions, Supplier adminSupplier) { + public static Map getEndOffsets(Set topicPartitions, + Supplier adminSupplier) { Admin admin = adminSupplier.get(); if (admin == null) { LOG.error("Kafka admin object is null, returning 0."); @@ -66,13 +68,15 @@ public static Map getEndOffsets(Set topicP try { endOffsets = getOffsets(admin, topicPartitions, OffsetSpec.latest()); } catch (RetriableException | ExecutionException | InterruptedException e) { - LOG.error("Failed to get offset from Kafka for topic partitions: {}.", topicPartitions, e); + LOG.error("Failed to get offset from Kafka for topic partitions: {}.", topicPartitions, + e); return Collections.EMPTY_MAP; } return endOffsets; } - public static Map getOffsets(Admin admin, Set topicPartitions, OffsetSpec offsetSpec) + public static Map getOffsets(Admin admin, + Set topicPartitions, OffsetSpec offsetSpec) throws InterruptedException, ExecutionException { Map offsetSpecMap = new HashMap<>(); @@ -81,7 +85,8 @@ public static Map getOffsets(Admin admin, Set ret = new HashMap<>(); ListOffsetsResult listOffsetsResult = admin.listOffsets(offsetSpecMap); - KafkaFuture> all = listOffsetsResult.all(); + KafkaFuture> all = + listOffsetsResult.all(); Map topicPartitionListOffsetsResultInfoMap = all.get(); for (Map.Entry entry : topicPartitionListOffsetsResultInfoMap.entrySet()) { diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/subscription/ManualPartitioner.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/subscription/ManualPartitioner.java index b6f3060189d..253a9702012 100644 --- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/subscription/ManualPartitioner.java +++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/subscription/ManualPartitioner.java @@ -27,18 +27,23 @@ /** * A function used to assign partitions to this spout. * - *

    WARNING if this is not done correctly you can really mess things up, like not reading data in some partitions. - * The complete TopologyContext is passed in, but it is suggested that you use the index of the spout and the total + *

    WARNING if this is not done correctly you can really mess things up, like not reading data in + * some partitions. + * The complete TopologyContext is passed in, but it is suggested that you use the index of the + * spout and the total * number of spouts to avoid missing partitions or double assigning partitions. */ @FunctionalInterface public interface ManualPartitioner extends Serializable { /** - * Filter the list of all partitions handled by this set of spouts to get only the partitions assigned to this task. + * Filter the list of all partitions handled by this set of spouts to get only the partitions + * assigned to this task. + * * @param allPartitionsSorted all of the partitions that the set of spouts want to subscribe to * in a strict ordering that is consistent across tasks * @param context the context of the topology * @return the subset of the partitions that this spout task should handle. */ - Set getPartitionsForThisTask(List allPartitionsSorted, TopologyContext context); + Set getPartitionsForThisTask(List allPartitionsSorted, + TopologyContext context); } diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/subscription/NamedTopicFilter.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/subscription/NamedTopicFilter.java index 87257d889f8..0d6e68cc38c 100644 --- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/subscription/NamedTopicFilter.java +++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/subscription/NamedTopicFilter.java @@ -37,6 +37,7 @@ public class NamedTopicFilter implements TopicFilter { /** * Create filter based on a set of topic names. + * * @param topics The topic names the filter will pass. */ public NamedTopicFilter(Set topics) { @@ -45,6 +46,7 @@ public NamedTopicFilter(Set topics) { /** * Convenience constructor. + * * @param topics The topic names the filter will pass. */ public NamedTopicFilter(String... topics) { @@ -58,7 +60,8 @@ public Set getAllSubscribedPartitions(Consumer consumer) { List partitionInfoList = consumer.partitionsFor(topic); if (partitionInfoList != null) { for (PartitionInfo partitionInfo : partitionInfoList) { - allPartitions.add(new TopicPartition(partitionInfo.topic(), partitionInfo.partition())); + allPartitions.add(new TopicPartition(partitionInfo.topic(), partitionInfo + .partition())); } } else { LOG.warn("Topic {} not found, skipping addition of the topic", topic); diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/subscription/PatternTopicFilter.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/subscription/PatternTopicFilter.java index 9ba49dd4284..6a9e7e914be 100644 --- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/subscription/PatternTopicFilter.java +++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/subscription/PatternTopicFilter.java @@ -34,7 +34,8 @@ public class PatternTopicFilter implements TopicFilter { private final Set topics = new HashSet<>(); /** - * Creates filter based on a Pattern. Only topic names matching the Pattern are passed by the filter. + * Creates filter based on a Pattern. Only topic names matching the Pattern are passed by the + * filter. * * @param pattern The Pattern to use. */ @@ -49,7 +50,8 @@ public Set getAllSubscribedPartitions(Consumer consumer) { for (Map.Entry> entry : consumer.listTopics().entrySet()) { if (pattern.matcher(entry.getKey()).matches()) { for (PartitionInfo partitionInfo : entry.getValue()) { - allPartitions.add(new TopicPartition(partitionInfo.topic(), partitionInfo.partition())); + allPartitions.add(new TopicPartition(partitionInfo.topic(), partitionInfo + .partition())); topics.add(partitionInfo.topic()); } } diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/subscription/RoundRobinManualPartitioner.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/subscription/RoundRobinManualPartitioner.java index ee2916a6763..2bef690fac1 100644 --- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/subscription/RoundRobinManualPartitioner.java +++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/subscription/RoundRobinManualPartitioner.java @@ -37,10 +37,12 @@ public class RoundRobinManualPartitioner implements ManualPartitioner { @Override - public Set getPartitionsForThisTask(List allPartitionsSorted, TopologyContext context) { + public Set getPartitionsForThisTask(List allPartitionsSorted, + TopologyContext context) { int thisTaskIndex = context.getThisTaskIndex(); int totalTaskCount = context.getComponentTasks(context.getThisComponentId()).size(); - Set myPartitions = new HashSet<>(allPartitionsSorted.size() / totalTaskCount + 1); + Set myPartitions = new HashSet<>(allPartitionsSorted + .size() / totalTaskCount + 1); for (int i = thisTaskIndex; i < allPartitionsSorted.size(); i += totalTaskCount) { myPartitions.add(allPartitionsSorted.get(i)); } diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/subscription/TopicAssigner.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/subscription/TopicAssigner.java index 2101dbf440c..0d1d59b8a7f 100644 --- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/subscription/TopicAssigner.java +++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/subscription/TopicAssigner.java @@ -31,12 +31,14 @@ public class TopicAssigner implements Serializable { /** * Assign partitions to the KafkaConsumer. + * * @param consumer The Kafka consumer to assign partitions to * @param newAssignment The partitions to assign. * @param listener The rebalance listener to call back on when the assignment changes * @return a boolean value indicating whether the partition assignment changed */ - public boolean assignPartitions(Consumer consumer, Set newAssignment, + public boolean assignPartitions(Consumer consumer, + Set newAssignment, ConsumerRebalanceListener listener) { Set currentAssignment = consumer.assignment(); if (!newAssignment.equals(currentAssignment)) { diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/subscription/TopicFilter.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/subscription/TopicFilter.java index 6c5941986d0..902d547f039 100644 --- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/subscription/TopicFilter.java +++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/subscription/TopicFilter.java @@ -25,6 +25,7 @@ public interface TopicFilter extends Serializable { /** * Get the Kafka TopicPartitions subscribed to by this set of spouts. + * * @param consumer The Kafka consumer to use to read the list of existing partitions * @return The Kafka partitions this set of spouts should subscribe to */ @@ -32,6 +33,7 @@ public interface TopicFilter extends Serializable { /** * Get the topics string. + * * @return A human-readable string representing the topics that pass the filter. */ String getTopicsString(); diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/trident/KafkaTridentOpaqueSpoutEmitter.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/trident/KafkaTridentOpaqueSpoutEmitter.java index fd68bdd9083..f0a9a195c75 100644 --- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/trident/KafkaTridentOpaqueSpoutEmitter.java +++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/trident/KafkaTridentOpaqueSpoutEmitter.java @@ -20,7 +20,6 @@ import java.util.List; import java.util.Map; import java.util.Set; - import org.apache.storm.trident.operation.TridentCollector; import org.apache.storm.trident.spout.IOpaquePartitionedTridentSpout; import org.apache.storm.trident.topology.TransactionAttempt; @@ -45,14 +44,14 @@ public Map> emitBatchNew(Tr return emitter.emitBatchNew(tx, collector, partitions, lastBatchMetaMap); } - @Override public void refreshPartitions(List partitionResponsibilities) { emitter.refreshPartitions(partitionResponsibilities); } @Override - public List getOrderedPartitions(List> allPartitionInfo) { + public List getOrderedPartitions(List> allPartitionInfo) { return emitter.getOrderedPartitions(allPartitionInfo); } diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/trident/KafkaTridentSpoutBatchMetadata.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/trident/KafkaTridentSpoutBatchMetadata.java index e20eb68eed5..54b7d927f82 100644 --- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/trident/KafkaTridentSpoutBatchMetadata.java +++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/trident/KafkaTridentSpoutBatchMetadata.java @@ -42,7 +42,7 @@ public class KafkaTridentSpoutBatchMetadata implements Serializable { private final long firstOffset; // last offset of this batch private final long lastOffset; - //The unique topology id for the topology that created this metadata + // The unique topology id for the topology that created this metadata private final String topologyId; /** diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/trident/KafkaTridentSpoutConfig.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/trident/KafkaTridentSpoutConfig.java index f62040f0301..00aade97585 100644 --- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/trident/KafkaTridentSpoutConfig.java +++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/trident/KafkaTridentSpoutConfig.java @@ -82,7 +82,8 @@ public Builder(String bootstrapServers, Pattern topics) { super(bootstrapServers, topics); } - public Builder(String bootstrapServers, TopicFilter topicFilter, ManualPartitioner topicPartitioner) { + public Builder(String bootstrapServers, TopicFilter topicFilter, + ManualPartitioner topicPartitioner) { super(bootstrapServers, topicFilter, topicPartitioner); } diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/trident/KafkaTridentSpoutCoordinator.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/trident/KafkaTridentSpoutCoordinator.java index 4789062bfd0..d30cb004a75 100644 --- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/trident/KafkaTridentSpoutCoordinator.java +++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/trident/KafkaTridentSpoutCoordinator.java @@ -38,7 +38,7 @@ public class KafkaTridentSpoutCoordinator implements IOpaquePartitionedTridentSpout.Coordinator>>, IPartitionedTridentSpout.Coordinator>>, Serializable { - //Initial delay for the assignment refresh timer + // Initial delay for the assignment refresh timer public static final long TIMER_DELAY_MS = 500; private static final Logger LOG = LoggerFactory.getLogger(KafkaTridentSpoutCoordinator.class); @@ -51,15 +51,18 @@ public class KafkaTridentSpoutCoordinator implements /** * Creates a new coordinator based on the given spout config. + * * @param kafkaSpoutConfig The spout config to use */ public KafkaTridentSpoutCoordinator(KafkaTridentSpoutConfig kafkaSpoutConfig) { this(kafkaSpoutConfig, new ClientFactoryDefault<>()); } - KafkaTridentSpoutCoordinator(KafkaTridentSpoutConfig kafkaSpoutConfig, ClientFactory clientFactory) { + KafkaTridentSpoutCoordinator(KafkaTridentSpoutConfig kafkaSpoutConfig, ClientFactory clientFactory) { this.kafkaSpoutConfig = kafkaSpoutConfig; - this.refreshAssignmentTimer = new Timer(TIMER_DELAY_MS, kafkaSpoutConfig.getPartitionRefreshPeriodMs(), TimeUnit.MILLISECONDS); + this.refreshAssignmentTimer = new Timer(TIMER_DELAY_MS, kafkaSpoutConfig + .getPartitionRefreshPeriodMs(), TimeUnit.MILLISECONDS); this.consumer = clientFactory.createConsumer(kafkaSpoutConfig.getKafkaProps()); LOG.debug("Created {}", this.toString()); } @@ -73,7 +76,8 @@ public boolean isReady(long txid) { @Override public List> getPartitionsForBatch() { if (refreshAssignmentTimer.isExpiredResetOnTrue() || partitionsForBatch == null) { - partitionsForBatch = kafkaSpoutConfig.getTopicFilter().getAllSubscribedPartitions(consumer); + partitionsForBatch = kafkaSpoutConfig.getTopicFilter() + .getAllSubscribedPartitions(consumer); } LOG.debug("TopicPartitions for batch {}", partitionsForBatch); return partitionsForBatch.stream() diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/trident/KafkaTridentSpoutEmitter.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/trident/KafkaTridentSpoutEmitter.java index 1c0a6ba41ad..453b1bb17e2 100644 --- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/trident/KafkaTridentSpoutEmitter.java +++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/trident/KafkaTridentSpoutEmitter.java @@ -37,7 +37,6 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; - import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.consumer.ConsumerRebalanceListener; import org.apache.kafka.clients.consumer.ConsumerRecord; @@ -66,7 +65,8 @@ public class KafkaTridentSpoutEmitter implements Serializable { private final KafkaTridentSpoutConfig kafkaSpoutConfig; private final TopicAssigner topicAssigner; - // The first seek offset for each topic partition, i.e. the offset this spout instance started processing at. + // The first seek offset for each topic partition, i.e. the offset this spout instance started + // processing at. private final Map tpToFirstSeekOffset = new HashMap<>(); private final long pollTimeoutMs; @@ -82,12 +82,14 @@ public class KafkaTridentSpoutEmitter implements Serializable { * @param kafkaSpoutConfig The kafka spout config * @param topologyContext The topology context */ - public KafkaTridentSpoutEmitter(KafkaTridentSpoutConfig kafkaSpoutConfig, TopologyContext topologyContext) { + public KafkaTridentSpoutEmitter(KafkaTridentSpoutConfig kafkaSpoutConfig, + TopologyContext topologyContext) { this(kafkaSpoutConfig, topologyContext, new ClientFactoryDefault<>(), new TopicAssigner()); } @VisibleForTesting - KafkaTridentSpoutEmitter(KafkaTridentSpoutConfig kafkaSpoutConfig, TopologyContext topologyContext, + KafkaTridentSpoutEmitter(KafkaTridentSpoutConfig kafkaSpoutConfig, + TopologyContext topologyContext, ClientFactory clientFactory, TopicAssigner topicAssigner) { this.kafkaSpoutConfig = kafkaSpoutConfig; this.consumer = clientFactory.createConsumer(kafkaSpoutConfig.getKafkaProps()); @@ -110,7 +112,8 @@ public void reEmitPartitionBatch(TransactionAttempt tx, TridentCollector collect throwIfEmittingForUnassignedPartition(currBatchTp); - KafkaTridentSpoutBatchMetadata currBatchMeta = KafkaTridentSpoutBatchMetadata.fromMap(currBatch); + KafkaTridentSpoutBatchMetadata currBatchMeta = KafkaTridentSpoutBatchMetadata + .fromMap(currBatch); Collection pausedTopicPartitions = Collections.emptySet(); if (!topologyContext.getStormId().equals(currBatchMeta.getTopologyId()) @@ -120,7 +123,8 @@ && isFirstPollOffsetStrategyIgnoringCommittedOffsets()) { return; } - LOG.debug("Re-emitting batch: [transaction= {}], [currBatchPartition = {}], [currBatchMetadata = {}], [collector = {}]", + LOG.debug("Re-emitting batch: [transaction= {}], [currBatchPartition = {}], " + + "[currBatchMetadata = {}], [collector = {}]", tx, currBatchPartition, currBatch, collector); try { @@ -130,9 +134,11 @@ && isFirstPollOffsetStrategyIgnoringCommittedOffsets()) { long seekOffset = currBatchMeta.getFirstOffset(); if (seekOffset < 0 && currBatchMeta.getFirstOffset() == currBatchMeta.getLastOffset()) { LOG.debug("Skipping re-emit of batch with negative starting offset." - + " The spout may set a negative starting offset for an empty batch that occurs at the start of a partition." + + " The spout may set a negative starting offset for an empty batch that " + + "occurs at the start of a partition." + " It is not expected that Trident will replay such an empty batch," - + " but this guard is here in case it tries to do so. See STORM-2990, STORM-3279 for context."); + + " but this guard is here in case it tries to do so. See STORM-2990, " + + "STORM-3279 for context."); return; } LOG.debug("Seeking to offset [{}] for topic partition [{}]", seekOffset, currBatchTp); @@ -146,9 +152,11 @@ && isFirstPollOffsetStrategyIgnoringCommittedOffsets()) { break; } if (record.offset() > currBatchMeta.getLastOffset()) { - throw new RuntimeException(String.format("Error when re-emitting batch. Overshot the end of the batch." + throw new RuntimeException(String + .format("Error when re-emitting batch. Overshot the end of the batch." + " The batch end offset was [{%d}], but received [{%d}]." - + " Ensure log compaction is disabled in Kafka, since it is incompatible with non-opaque transactional spouts.", + + " Ensure log compaction is disabled in Kafka, since it is incompatible " + + "with non-opaque transactional spouts.", currBatchMeta.getLastOffset(), record.offset())); } emitTuple(collector, record); @@ -157,7 +165,8 @@ && isFirstPollOffsetStrategyIgnoringCommittedOffsets()) { consumer.resume(pausedTopicPartitions); LOG.trace("Resumed topic-partitions {}", pausedTopicPartitions); } - LOG.debug("Re-emitted batch: [transaction = {}], [currBatchPartition = {}], [currBatchMetadata = {}], " + LOG.debug("Re-emitted batch: [transaction = {}], [currBatchPartition = {}], " + + "[currBatchMetadata = {}], " + "[collector = {}]", tx, currBatchPartition, currBatchMeta, collector); } @@ -168,10 +177,12 @@ public Map> emitBatchNew(Tr TridentCollector collector, Set partitions, Map> lastBatchMetaMap) { - LOG.debug("Processing batch: [transaction = {}], [currBatchPartitions = {}], [lastBatchMetadata = {}], [collector = {}]", + LOG.debug("Processing batch: [transaction = {}], [currBatchPartitions = {}], " + + "[lastBatchMetadata = {}], [collector = {}]", tx, partitions, lastBatchMetaMap, collector); - Map> partitionToBatchMeta = new HashMap<>(); + Map> partitionToBatchMeta = + new HashMap<>(); seekAllPartitions(partitions, lastBatchMetaMap); @@ -188,18 +199,21 @@ public Map> emitBatchNew(Tr records.get(records.size() - 1).offset(), topologyContext.getStormId()).toMap()); } else { - //Build new metadata based on the consumer position. - //We want the next emit to start at the current consumer position, - //so make a meta that indicates that position - 1 is the last emitted offset - //This helps us avoid cases like STORM-3279, and simplifies the seek logic. + // Build new metadata based on the consumer position. + // We want the next emit to start at the current consumer position, + // so make a meta that indicates that position - 1 is the last emitted offset + // This helps us avoid cases like STORM-3279, and simplifies the seek logic. long lastEmittedOffset = consumer.position(partition.getTopicPartition()) - 1; - partitionToBatchMeta.put(partition, new KafkaTridentSpoutBatchMetadata(lastEmittedOffset, lastEmittedOffset, + partitionToBatchMeta.put(partition, + new KafkaTridentSpoutBatchMetadata(lastEmittedOffset, lastEmittedOffset, topologyContext.getStormId()).toMap()); } } for (KafkaTridentSpoutTopicPartition kttp : partitionToBatchMeta.keySet()) { - LOG.debug("Emitted batch: [transaction = {}], [currBatchPartition = {}], [lastBatchMetadata = {}], " - + "[currBatchMetadata = {}], [collector = {}]", tx, kttp, lastBatchMetaMap.get(kttp), + LOG.debug("Emitted batch: [transaction = {}], [currBatchPartition = {}], " + + "[lastBatchMetadata = {}], " + + "[currBatchMetadata = {}], [collector = {}]", tx, kttp, lastBatchMetaMap + .get(kttp), partitionToBatchMeta.get(kttp), collector); } return partitionToBatchMeta; @@ -212,7 +226,8 @@ private void seekAllPartitions(Collection parti TopicPartition currentBatchTp = partition.getTopicPartition(); throwIfEmittingForUnassignedPartition(currentBatchTp); Map lastBatch = lastPartitionMetaMap.get(partition); - KafkaTridentSpoutBatchMetadata lastBatchMeta = lastBatch == null ? null : KafkaTridentSpoutBatchMetadata.fromMap(lastBatch); + KafkaTridentSpoutBatchMetadata lastBatchMeta = lastBatch == null + ? null : KafkaTridentSpoutBatchMetadata.fromMap(lastBatch); seek(currentBatchTp, lastBatchMeta); } } @@ -225,9 +240,11 @@ private boolean isFirstPollOffsetStrategyIgnoringCommittedOffsets() { private void throwIfEmittingForUnassignedPartition(TopicPartition currBatchTp) { final Set assignments = consumer.assignment(); if (!assignments.contains(currBatchTp)) { - throw new IllegalStateException("The spout is asked to emit tuples on a partition it is not assigned." + throw new IllegalStateException("The spout is asked to emit tuples on a partition it " + + "is not assigned." + " This indicates a bug in the TopicFilter or ManualPartitioner implementations." - + " The current partition is [" + currBatchTp + "], the assigned partitions are [" + assignments + "]."); + + " The current partition is [" + currBatchTp + "], the assigned partitions are [" + + assignments + "]."); } } @@ -238,7 +255,8 @@ private void emitTuple(TridentCollector collector, ConsumerRecord record) } /** - * Determines the offset of the next fetch. Will use the firstPollOffsetStrategy if this is the first poll for the topic partition. + * Determines the offset of the next fetch. Will use the firstPollOffsetStrategy if this is the + * first poll for the topic partition. * Otherwise the next offset will be one past the last batch, based on lastBatchMeta. * *

    lastBatchMeta should only be null in the following cases: @@ -254,22 +272,28 @@ private long seek(TopicPartition tp, KafkaTridentSpoutBatchMetadata lastBatchMet boolean isFirstPollSinceTopologyWasDeployed = lastBatchMeta == null || !topologyContext.getStormId().equals(lastBatchMeta.getTopologyId()); if (firstPollOffsetStrategy == EARLIEST && isFirstPollSinceTopologyWasDeployed) { - LOG.debug("First poll for topic partition [{}], seeking to partition beginning", tp); + LOG.debug("First poll for topic partition [{}], seeking to partition beginning", + tp); consumer.seekToBeginning(Collections.singleton(tp)); } else if (firstPollOffsetStrategy == LATEST && isFirstPollSinceTopologyWasDeployed) { LOG.debug("First poll for topic partition [{}], seeking to partition end", tp); consumer.seekToEnd(Collections.singleton(tp)); - } else if (firstPollOffsetStrategy == TIMESTAMP && isFirstPollSinceTopologyWasDeployed) { - LOG.debug("First poll for topic partition [{}], seeking to partition based on startTimeStamp", tp); + } else if (firstPollOffsetStrategy == TIMESTAMP + && isFirstPollSinceTopologyWasDeployed) { + LOG.debug("First poll for topic partition [{}], seeking to partition based on " + + "startTimeStamp", tp); seekOffsetByStartTimeStamp(tp); } else if (lastBatchMeta != null) { LOG.debug("First poll for topic partition [{}], using last batch metadata", tp); - consumer.seek(tp, lastBatchMeta.getLastOffset() + 1); // seek next offset after last offset from previous batch + consumer.seek(tp, lastBatchMeta.getLastOffset() + + 1); // seek next offset after last offset from previous batch } else if (firstPollOffsetStrategy == UNCOMMITTED_EARLIEST) { - LOG.debug("First poll for topic partition [{}] with no last batch metadata, seeking to partition beginning", tp); + LOG.debug("First poll for topic partition [{}] with no last batch metadata, " + + "seeking to partition beginning", tp); consumer.seekToBeginning(Collections.singleton(tp)); } else if (firstPollOffsetStrategy == UNCOMMITTED_LATEST) { - LOG.debug("First poll for topic partition [{}] with no last batch metadata, seeking to partition end", tp); + LOG.debug("First poll for topic partition [{}] with no last batch metadata, " + + "seeking to partition end", tp); consumer.seekToEnd(Collections.singleton(tp)); } else if (firstPollOffsetStrategy == UNCOMMITTED_TIMESTAMP) { LOG.debug("First poll for topic partition [{}] with no last batch metadata, " @@ -278,12 +302,15 @@ private long seek(TopicPartition tp, KafkaTridentSpoutBatchMetadata lastBatchMet } tpToFirstSeekOffset.put(tp, consumer.position(tp)); } else if (lastBatchMeta != null) { - consumer.seek(tp, lastBatchMeta.getLastOffset() + 1); // seek next offset after last offset from previous batch + consumer.seek(tp, lastBatchMeta.getLastOffset() + + 1); // seek next offset after last offset from previous batch LOG.debug("First poll for topic partition [{}], using last batch metadata", tp); } else { /* - * Last batch meta is null, but this is not the first batch emitted for this partition by this emitter instance. This is - * a replay of the first batch for this partition. Use the offset the consumer started at. + * Last batch meta is null, but this is not the first batch emitted for this partition + * by this emitter instance. This is + * a replay of the first batch for this partition. Use the offset the consumer started + * at. */ long initialFetchOffset = tpToFirstSeekOffset.get(tp); consumer.seek(tp, initialFetchOffset); @@ -300,10 +327,12 @@ private long seek(TopicPartition tp, KafkaTridentSpoutBatchMetadata lastBatchMet * Seek the consumer to offset corresponding to startTimeStamp. */ private void seekOffsetByStartTimeStamp(TopicPartition tp) { - Map offsetsForTimes = consumer.offsetsForTimes(Collections.singletonMap(tp, startTimeStamp)); + Map offsetsForTimes = consumer + .offsetsForTimes(Collections.singletonMap(tp, startTimeStamp)); OffsetAndTimestamp startOffsetAndTimeStamp = offsetsForTimes.get(tp); long startTimeStampOffset = startOffsetAndTimeStamp.offset(); - LOG.debug("First poll for topic partition [{}], seeking to partition from startTimeStamp [{}]", tp, startTimeStamp); + LOG.debug("First poll for topic partition [{}], seeking to partition from startTimeStamp " + + "[{}]", tp, startTimeStamp); consumer.seek(tp, startTimeStampOffset); } @@ -324,13 +353,16 @@ private Collection pauseTopicPartitions(TopicPartition excludedT /** * Get the input partitions in sorted order. */ - public List getOrderedPartitions(final List> allPartitionInfo) { + public List getOrderedPartitions(final List> allPartitionInfo) { List sortedPartitions = allPartitionInfo.stream() .map(map -> tpSerializer.fromMap(map)) .sorted(TopicPartitionComparator.INSTANCE) .collect(Collectors.toList()); - final List allPartitions = newKafkaTridentSpoutTopicPartitions(sortedPartitions); - LOG.debug("Returning all topic-partitions {} across all tasks. Current task index [{}]. Total tasks [{}] ", + final List allPartitions = + newKafkaTridentSpoutTopicPartitions(sortedPartitions); + LOG.debug("Returning all topic-partitions {} across all tasks. Current task index [{}]. " + + "Total tasks [{}] ", allPartitions, topologyContext.getThisTaskIndex(), getNumTasks()); return allPartitions; } @@ -343,9 +375,12 @@ public List getPartitionsForTask(int taskId, in List tps = allPartitionInfoSorted.stream() .map(kttp -> kttp.getTopicPartition()) .collect(Collectors.toList()); - final Set assignedTps = kafkaSpoutConfig.getTopicPartitioner().getPartitionsForThisTask(tps, topologyContext); - LOG.debug("Consumer [{}], running on task with index [{}], has assigned topic-partitions {}", consumer, taskId, assignedTps); - final List taskTps = newKafkaTridentSpoutTopicPartitions(assignedTps); + final Set assignedTps = kafkaSpoutConfig.getTopicPartitioner() + .getPartitionsForThisTask(tps, topologyContext); + LOG.debug("Consumer [{}], running on task with index [{}], has assigned topic-partitions " + + "{}", consumer, taskId, assignedTps); + final List taskTps = + newKafkaTridentSpoutTopicPartitions(assignedTps); return taskTps; } @@ -356,7 +391,8 @@ public void refreshPartitions(List partitionRes Set assignedTps = partitionResponsibilities.stream() .map(kttp -> kttp.getTopicPartition()) .collect(Collectors.toSet()); - topicAssigner.assignPartitions(consumer, assignedTps, new KafkaSpoutConsumerRebalanceListener()); + topicAssigner.assignPartitions(consumer, assignedTps, + new KafkaSpoutConsumerRebalanceListener()); LOG.debug("Assigned partitions [{}] to this task", assignedTps); } diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/trident/KafkaTridentSpoutOpaque.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/trident/KafkaTridentSpoutOpaque.java index a76cb0e90b7..f5b9945ee60 100644 --- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/trident/KafkaTridentSpoutOpaque.java +++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/trident/KafkaTridentSpoutOpaque.java @@ -50,11 +50,13 @@ public KafkaTridentSpoutOpaque(KafkaTridentSpoutConfig kafkaSpoutConfig) { @Override public Emitter>, KafkaTridentSpoutTopicPartition, Map> getEmitter( Map conf, TopologyContext context) { - return new KafkaTridentOpaqueSpoutEmitter<>(new KafkaTridentSpoutEmitter<>(kafkaSpoutConfig, context)); + return new KafkaTridentOpaqueSpoutEmitter<>(new KafkaTridentSpoutEmitter<>(kafkaSpoutConfig, + context)); } @Override - public Coordinator>> getCoordinator(Map conf, TopologyContext context) { + public Coordinator>> getCoordinator(Map conf, + TopologyContext context) { return new KafkaTridentSpoutCoordinator<>(kafkaSpoutConfig); } diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/trident/KafkaTridentSpoutTransactional.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/trident/KafkaTridentSpoutTransactional.java index f2500509fc1..46312b7eb4a 100644 --- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/trident/KafkaTridentSpoutTransactional.java +++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/trident/KafkaTridentSpoutTransactional.java @@ -42,7 +42,8 @@ public KafkaTridentSpoutTransactional(KafkaTridentSpoutConfig kafkaSpoutCo } @Override - public Coordinator>> getCoordinator(Map conf, TopologyContext context) { + public Coordinator>> getCoordinator(Map conf, + TopologyContext context) { return new KafkaTridentSpoutCoordinator<>(kafkaSpoutConfig); } diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/trident/KafkaTridentTransactionalSpoutEmitter.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/trident/KafkaTridentTransactionalSpoutEmitter.java index 61a8b179b69..d7a13d58e66 100644 --- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/trident/KafkaTridentTransactionalSpoutEmitter.java +++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/trident/KafkaTridentTransactionalSpoutEmitter.java @@ -20,7 +20,6 @@ import java.util.List; import java.util.Map; import java.util.Set; - import org.apache.storm.trident.operation.TridentCollector; import org.apache.storm.trident.spout.IPartitionedTridentSpout; import org.apache.storm.trident.topology.TransactionAttempt; @@ -39,7 +38,8 @@ public KafkaTridentTransactionalSpoutEmitter(KafkaTridentSpoutEmitter emit } @Override - public List getOrderedPartitions(List> allPartitionInfo) { + public List getOrderedPartitions(List> allPartitionInfo) { return emitter.getOrderedPartitions(allPartitionInfo); } diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/trident/TopicPartitionSerializer.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/trident/TopicPartitionSerializer.java index 50e78f098a9..155f500b58e 100644 --- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/trident/TopicPartitionSerializer.java +++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/trident/TopicPartitionSerializer.java @@ -36,7 +36,8 @@ public Map toMap(TopicPartition topicPartition) { } /** - * Deserializes the given map into a TopicPartition. The map keys are expected to be those produced by + * Deserializes the given map into a TopicPartition. The map keys are expected to be those + * produced by * {@link #toMap(org.apache.kafka.common.TopicPartition)}. */ public TopicPartition fromMap(Map map) { diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/trident/internal/OutputFieldsExtractor.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/trident/internal/OutputFieldsExtractor.java index e702fef4b09..2c6e52ee8ad 100644 --- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/trident/internal/OutputFieldsExtractor.java +++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/trident/internal/OutputFieldsExtractor.java @@ -26,7 +26,8 @@ public class OutputFieldsExtractor implements Serializable { /** * Extract the output fields from the config. - * Throws an error if there are multiple declared output streams, since Trident only supports one output stream per spout. + * Throws an error if there are multiple declared output streams, since Trident only supports + * one output stream per spout. */ public Fields getOutputFields(KafkaTridentSpoutConfig kafkaSpoutConfig) { RecordTranslator translator = kafkaSpoutConfig.getTranslator(); diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/trident/TridentKafkaState.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/trident/TridentKafkaState.java index ec7ae7de8e5..160ca31bc3c 100644 --- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/trident/TridentKafkaState.java +++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/trident/TridentKafkaState.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

    Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -44,7 +44,8 @@ public class TridentKafkaState implements State { private TridentTupleToKafkaMapper mapper; private KafkaTopicSelector topicSelector; - public TridentKafkaState withTridentTupleToKafkaMapper(TridentTupleToKafkaMapper mapper) { + public TridentKafkaState withTridentTupleToKafkaMapper(TridentTupleToKafkaMapper mapper) { this.mapper = mapper; return this; } @@ -66,6 +67,7 @@ public void commit(Long txid) { /** * Prepare this State. + * * @param options The KafkaProducer config. */ public void prepare(Properties options) { @@ -76,6 +78,7 @@ public void prepare(Properties options) { /** * Write the given tuples to Kafka. + * * @param tuples The tuples to write. * @param collector The Trident collector. */ @@ -92,7 +95,8 @@ public void updateState(List tuples, TridentCollector collector) { if (topic != null) { if (messageFromTuple != null) { - Future result = producer.send(new ProducerRecord<>(topic, keyFromTuple, messageFromTuple)); + Future result = producer.send(new ProducerRecord<>(topic, + keyFromTuple, messageFromTuple)); futures.add(result); } else { LOG.warn("skipping Message with Key {} as message was null", keyFromTuple); @@ -114,19 +118,23 @@ public void updateState(List tuples, TridentCollector collector) { } if (exceptions.size() > 0) { - StringBuilder errorMsg = new StringBuilder("Could not retrieve result for messages "); + StringBuilder errorMsg = + new StringBuilder("Could not retrieve result for messages "); errorMsg.append(tuples).append(" from topic = ").append(topic) - .append(" because of the following exceptions:").append(System.lineSeparator()); + .append(" because of the following exceptions:").append(System + .lineSeparator()); for (ExecutionException exception : exceptions) { - errorMsg = errorMsg.append(exception.getMessage()).append(System.lineSeparator()); + errorMsg = errorMsg.append(exception.getMessage()).append(System + .lineSeparator()); } String message = errorMsg.toString(); LOG.error(message); throw new FailedException(message); } long latestTime = System.currentTimeMillis(); - LOG.info("Emitted record {} sucessfully in {} ms to topic {} ", emittedRecords, latestTime - startTime, topic); + LOG.info("Emitted record {} sucessfully in {} ms to topic {} ", emittedRecords, + latestTime - startTime, topic); } catch (Exception ex) { String errorMsg = "Could not send messages " + tuples + " to topic = " + topic; diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/trident/TridentKafkaStateFactory.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/trident/TridentKafkaStateFactory.java index 38f5c6e5bcb..9d339629dfe 100644 --- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/trident/TridentKafkaStateFactory.java +++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/trident/TridentKafkaStateFactory.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

    Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -37,7 +37,8 @@ public class TridentKafkaStateFactory implements StateFactory { private KafkaTopicSelector topicSelector; private Properties producerProperties = new Properties(); - public TridentKafkaStateFactory withTridentTupleToKafkaMapper(TridentTupleToKafkaMapper mapper) { + public TridentKafkaStateFactory withTridentTupleToKafkaMapper(TridentTupleToKafkaMapper mapper) { this.mapper = mapper; return this; } @@ -53,7 +54,8 @@ public TridentKafkaStateFactory withProducerProperties(Properties props) { } @Override - public State makeState(Map conf, IMetricsContext metrics, int partitionIndex, int numPartitions) { + public State makeState(Map conf, IMetricsContext metrics, int partitionIndex, + int numPartitions) { LOG.info("makeState(partitonIndex={}, numpartitions={}", partitionIndex, numPartitions); TridentKafkaState state = new TridentKafkaState<>(); state.withKafkaTopicSelector(this.topicSelector) diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/trident/TridentKafkaStateUpdater.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/trident/TridentKafkaStateUpdater.java index 19e3d332eb0..0414e174442 100644 --- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/trident/TridentKafkaStateUpdater.java +++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/trident/TridentKafkaStateUpdater.java @@ -28,7 +28,8 @@ public class TridentKafkaStateUpdater extends BaseStateUpdater state, List tuples, TridentCollector collector) { + public void updateState(TridentKafkaState state, List tuples, + TridentCollector collector) { state.updateState(tuples, collector); } } diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/trident/mapper/FieldNameBasedTupleToKafkaMapper.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/trident/mapper/FieldNameBasedTupleToKafkaMapper.java index 15b72cf4b6d..4d9a3ed6c89 100644 --- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/trident/mapper/FieldNameBasedTupleToKafkaMapper.java +++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/trident/mapper/FieldNameBasedTupleToKafkaMapper.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

    Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/trident/mapper/TridentTupleToKafkaMapper.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/trident/mapper/TridentTupleToKafkaMapper.java index 78277767962..7569a62915b 100644 --- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/trident/mapper/TridentTupleToKafkaMapper.java +++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/trident/mapper/TridentTupleToKafkaMapper.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

    Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/trident/selector/DefaultTopicSelector.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/trident/selector/DefaultTopicSelector.java index 63d69da9ef3..22787a94e07 100644 --- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/trident/selector/DefaultTopicSelector.java +++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/trident/selector/DefaultTopicSelector.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

    Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/trident/selector/KafkaTopicSelector.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/trident/selector/KafkaTopicSelector.java index 8426952659a..547608f3bf7 100644 --- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/trident/selector/KafkaTopicSelector.java +++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/trident/selector/KafkaTopicSelector.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

    Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/KafkaUnit.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/KafkaUnit.java index 433c5cbb95d..c1614e9f068 100644 --- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/KafkaUnit.java +++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/KafkaUnit.java @@ -47,7 +47,8 @@ public class KafkaUnit { private Server kafkaServer; private KafkaProducer producer; private AdminClient kafkaAdminClient; - private TmpPath kafkaDir,metadata; + private TmpPath kafkaDir; + private TmpPath metadata; private static final String KAFKA_HOST = "127.0.0.1"; private static final int KAFKA_BROKER_PORT = 9092; private static final int KAFKA_CONTROLLER_PORT = 9093; @@ -61,30 +62,37 @@ public void setUp() throws Exception { kafkaDir = new TmpPath(Files.createTempDirectory("kafka-").toAbsolutePath().toString()); metadata = new TmpPath(Paths.get(kafkaDir.getPath() + "/" + "meta.properties").toString()); Files.createFile(metadata.getFile().toPath()); - String content ="node.id=0" + System.lineSeparator() + "version=1" + System.lineSeparator() + "cluster.id="+ RandomStringUtils.randomAlphanumeric(10); + String content = "node.id=0" + System.lineSeparator() + "version=1" + System + .lineSeparator() + "cluster.id=" + RandomStringUtils.randomAlphanumeric(10); Files.writeString(metadata.getFile().toPath(), content, StandardOpenOption.APPEND); Properties brokerProps = new Properties(); brokerProps.setProperty("broker.id", "0"); brokerProps.setProperty("log.dirs", kafkaDir.getPath()); - String listeners = String.format("BROKER://%s:%d", KAFKA_HOST, KAFKA_BROKER_PORT) + "," + String.format("CONTROLLER://%s:%d", KAFKA_HOST, KAFKA_CONTROLLER_PORT); + String listeners = String.format("BROKER://%s:%d", KAFKA_HOST, KAFKA_BROKER_PORT) + "," + + String.format("CONTROLLER://%s:%d", KAFKA_HOST, KAFKA_CONTROLLER_PORT); brokerProps.setProperty("advertised.listeners", listeners); brokerProps.setProperty("listeners", listeners); brokerProps.setProperty("offsets.topic.replication.factor", "1"); - brokerProps.setProperty("process.roles","broker,controller"); - brokerProps.setProperty("controller.quorum.bootstrap.servers",String.format("%s:%d", KAFKA_HOST, KAFKA_CONTROLLER_PORT)); - brokerProps.setProperty("controller.listener.names","CONTROLLER"); - brokerProps.setProperty("listener.security.protocol.map","CONTROLLER:PLAINTEXT,BROKER:PLAINTEXT"); - brokerProps.setProperty("inter.broker.listener.name","BROKER"); - brokerProps.setProperty("controller.quorum.voters","0@"+String.format("%s:%d", KAFKA_HOST, KAFKA_CONTROLLER_PORT)); + brokerProps.setProperty("process.roles", "broker,controller"); + brokerProps.setProperty("controller.quorum.bootstrap.servers", String.format("%s:%d", + KAFKA_HOST, KAFKA_CONTROLLER_PORT)); + brokerProps.setProperty("controller.listener.names", "CONTROLLER"); + brokerProps.setProperty("listener.security.protocol.map", + "CONTROLLER:PLAINTEXT,BROKER:PLAINTEXT"); + brokerProps.setProperty("inter.broker.listener.name", "BROKER"); + brokerProps.setProperty("controller.quorum.voters", "0@" + String.format("%s:%d", + KAFKA_HOST, KAFKA_CONTROLLER_PORT)); KafkaConfig config = new KafkaConfig(brokerProps); - kafkaServer= new KafkaRaftServer(config, Time.SYSTEM); + kafkaServer = new KafkaRaftServer(config, Time.SYSTEM); kafkaServer.startup(); // setup default Producer createProducer(); - kafkaAdminClient = AdminClient.create(Collections.singletonMap(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, KAFKA_HOST + ":" + KAFKA_BROKER_PORT)); + kafkaAdminClient = AdminClient.create(Collections + .singletonMap(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, KAFKA_HOST + ":" + + KAFKA_BROKER_PORT)); } public void tearDown() throws Exception { @@ -96,7 +104,7 @@ public void tearDown() throws Exception { } public void createTopic(String topicName) throws Exception { - kafkaAdminClient.createTopics(Collections.singleton(new NewTopic(topicName, 1, (short)1))) + kafkaAdminClient.createTopics(Collections.singleton(new NewTopic(topicName, 1, (short) 1))) .all() .get(30, TimeUnit.SECONDS); } @@ -108,12 +116,15 @@ public int getKafkaPort() { private void createProducer() { Properties producerProps = new Properties(); producerProps.setProperty(BOOTSTRAP_SERVERS_CONFIG, KAFKA_HOST + ":" + KAFKA_BROKER_PORT); - producerProps.setProperty(KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer"); - producerProps.setProperty(VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer"); + producerProps.setProperty(KEY_SERIALIZER_CLASS_CONFIG, + "org.apache.kafka.common.serialization.StringSerializer"); + producerProps.setProperty(VALUE_SERIALIZER_CLASS_CONFIG, + "org.apache.kafka.common.serialization.StringSerializer"); producer = new KafkaProducer<>(producerProps); } - public void sendMessage(ProducerRecord producerRecord) throws InterruptedException, ExecutionException, TimeoutException { + public void sendMessage(ProducerRecord producerRecord) throws InterruptedException, ExecutionException, TimeoutException { producer.send(producerRecord).get(10, TimeUnit.SECONDS); } diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/bolt/KafkaBoltTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/bolt/KafkaBoltTest.java index 63b98eb6a36..e53554900ad 100644 --- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/bolt/KafkaBoltTest.java +++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/bolt/KafkaBoltTest.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

    Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -28,7 +28,6 @@ import java.util.HashMap; import java.util.Map; import java.util.Properties; - import org.apache.kafka.clients.producer.MockProducer; import org.apache.kafka.clients.producer.Producer; import org.apache.kafka.clients.producer.ProducerRecord; diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/ByTopicRecordTranslatorTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/ByTopicRecordTranslatorTest.java index 6e6a776a2c7..9fe4acf2299 100644 --- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/ByTopicRecordTranslatorTest.java +++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/ByTopicRecordTranslatorTest.java @@ -35,23 +35,28 @@ public class ByTopicRecordTranslatorTest { public void testBasic() { ByTopicRecordTranslator trans = new ByTopicRecordTranslator<>((r) -> new Values(r.key()), new Fields("key")); - trans.forTopic("TOPIC 1", (r) -> new Values(r.value()), new Fields("value"), "value-stream"); - trans.forTopic("TOPIC 2", (r) -> new Values(r.key(), r.value()), new Fields("key", "value"), "key-value-stream"); + trans.forTopic("TOPIC 1", (r) -> new Values(r.value()), new Fields("value"), + "value-stream"); + trans.forTopic("TOPIC 2", (r) -> new Values(r.key(), r.value()), new Fields("key", "value"), + "key-value-stream"); HashSet expectedStreams = new HashSet<>(); expectedStreams.add("default"); expectedStreams.add("value-stream"); expectedStreams.add("key-value-stream"); assertEquals(expectedStreams, new HashSet<>(trans.streams())); - ConsumerRecord cr1 = new ConsumerRecord<>("TOPIC OTHER", 100, 100, "THE KEY", "THE VALUE"); + ConsumerRecord cr1 = new ConsumerRecord<>("TOPIC OTHER", 100, 100, + "THE KEY", "THE VALUE"); assertEquals(new Fields("key"), trans.getFieldsFor("default")); assertEquals(Collections.singletonList("THE KEY"), trans.apply(cr1)); - ConsumerRecord cr2 = new ConsumerRecord<>("TOPIC 1", 100, 100, "THE KEY", "THE VALUE"); + ConsumerRecord cr2 = new ConsumerRecord<>("TOPIC 1", 100, 100, "THE KEY", + "THE VALUE"); assertEquals(new Fields("value"), trans.getFieldsFor("value-stream")); assertEquals(Collections.singletonList("THE VALUE"), trans.apply(cr2)); - ConsumerRecord cr3 = new ConsumerRecord<>("TOPIC 2", 100, 100, "THE KEY", "THE VALUE"); + ConsumerRecord cr3 = new ConsumerRecord<>("TOPIC 2", 100, 100, "THE KEY", + "THE VALUE"); assertEquals(new Fields("key", "value"), trans.getFieldsFor("key-value-stream")); assertEquals(Arrays.asList("THE KEY", "THE VALUE"), trans.apply(cr3)); } @@ -60,7 +65,8 @@ public void testBasic() { public void testNullTranslation() { ByTopicRecordTranslator trans = new ByTopicRecordTranslator<>((r) -> null, new Fields("key")); - ConsumerRecord cr = new ConsumerRecord<>("TOPIC 1", 100, 100, "THE KEY", "THE VALUE"); + ConsumerRecord cr = new ConsumerRecord<>("TOPIC 1", 100, 100, "THE KEY", + "THE VALUE"); assertNull(trans.apply(cr)); } @@ -79,7 +85,8 @@ public void testTopicCollision() { ByTopicRecordTranslator trans = new ByTopicRecordTranslator<>((r) -> new Values(r.key()), new Fields("key")); trans.forTopic("foo", (r) -> new Values(r.value()), new Fields("value"), "foo1"); - trans.forTopic("foo", (r) -> new Values(r.key(), r.value()), new Fields("key", "value"), "foo2"); + trans.forTopic("foo", (r) -> new Values(r.key(), r.value()), new Fields("key", "value"), + "foo2"); }); } diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/DefaultRecordTranslatorTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/DefaultRecordTranslatorTest.java index f9a4b453ba0..7ef90ab3703 100644 --- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/DefaultRecordTranslatorTest.java +++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/DefaultRecordTranslatorTest.java @@ -21,7 +21,6 @@ import java.util.Arrays; import java.util.Collections; - import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.storm.tuple.Fields; import org.junit.jupiter.api.Test; @@ -31,8 +30,10 @@ public class DefaultRecordTranslatorTest { public void testBasic() { DefaultRecordTranslator trans = new DefaultRecordTranslator<>(); assertEquals(Collections.singletonList("default"), trans.streams()); - assertEquals(new Fields("topic", "partition", "offset", "key", "value"), trans.getFieldsFor("default")); - ConsumerRecord cr = new ConsumerRecord<>("TOPIC", 100, 100, "THE KEY", "THE VALUE"); + assertEquals(new Fields("topic", "partition", "offset", "key", "value"), trans + .getFieldsFor("default")); + ConsumerRecord cr = new ConsumerRecord<>("TOPIC", 100, 100, "THE KEY", + "THE VALUE"); assertEquals(Arrays.asList("TOPIC", 100, 100L, "THE KEY", "THE VALUE"), trans.apply(cr)); } } diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutAbstractTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutAbstractTest.java index 6f92f6d331d..e959657197b 100644 --- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutAbstractTest.java +++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutAbstractTest.java @@ -28,7 +28,6 @@ import java.util.HashMap; import java.util.Map; - import org.apache.kafka.clients.admin.Admin; import org.apache.kafka.clients.admin.KafkaAdminClient; import org.apache.kafka.clients.consumer.KafkaConsumer; @@ -73,8 +72,11 @@ public abstract class KafkaSpoutAbstractTest { private KafkaSpoutConfig spoutConfig; /** - * This constructor should be called by the subclass' default constructor with the desired value - * @param commitOffsetPeriodMs commit offset period to be used in commit and verification of messages committed + * This constructor should be called by the subclass' default constructor with the desired + * value. + * + * @param commitOffsetPeriodMs commit offset period to be used in commit and verification of + * messages committed */ protected KafkaSpoutAbstractTest(long commitOffsetPeriodMs) { this.commitOffsetPeriodMs = commitOffsetPeriodMs; @@ -104,6 +106,7 @@ private ClientFactory createConsumerFactory() { public KafkaConsumer createConsumer(Map consumerProps) { return consumerSpy; } + @Override public Admin createAdmin(Map adminProps) { return adminSpy; @@ -112,11 +115,13 @@ public Admin createAdmin(Map adminProps) { } KafkaConsumer createConsumerSpy() { - return spy(new ClientFactoryDefault().createConsumer(spoutConfig.getKafkaProps())); + return spy(new ClientFactoryDefault().createConsumer(spoutConfig + .getKafkaProps())); } - Admin createAdminSpy(){ - return spy(new ClientFactoryDefault().createAdmin(spoutConfig.getKafkaProps())); + Admin createAdminSpy() { + return spy(new ClientFactoryDefault().createAdmin(spoutConfig + .getKafkaProps())); } @AfterEach @@ -127,8 +132,10 @@ public void tearDown() throws Exception { abstract KafkaSpoutConfig createSpoutConfig(); void prepareSpout(int messageCount) throws Exception { - SingleTopicKafkaUnitSetupHelper.populateTopicData(kafkaUnitExtension.getKafkaUnit(), SingleTopicKafkaSpoutConfiguration.TOPIC, messageCount); - SingleTopicKafkaUnitSetupHelper.initializeSpout(spout, conf, topologyContext, collectorMock); + SingleTopicKafkaUnitSetupHelper.populateTopicData(kafkaUnitExtension.getKafkaUnit(), + SingleTopicKafkaSpoutConfiguration.TOPIC, messageCount); + SingleTopicKafkaUnitSetupHelper.initializeSpout(spout, conf, topologyContext, + collectorMock); } /** @@ -173,7 +180,7 @@ void commitAndVerifyAllMessagesCommitted(long msgCount) { // reset commit timer such that commit happens on next call to nextTuple() Time.advanceTime(commitOffsetPeriodMs + KafkaSpout.TIMER_DELAY_MS); - //Commit offsets + // Commit offsets spout.nextTuple(); verifyAllMessagesCommitted(msgCount); @@ -188,10 +195,12 @@ void verifyAllMessagesCommitted(long messageCount) { verify(consumerSpy).commitSync(commitCapture.capture()); final Map commits = commitCapture.getValue(); - assertThat("Expected commits for only one topic partition", commits.entrySet().size(), is(1)); + assertThat("Expected commits for only one topic partition", commits.entrySet().size(), + is(1)); OffsetAndMetadata offset = commits.entrySet().iterator().next().getValue(); - assertThat("Expected committed offset to cover all emitted messages", offset.offset(), is(messageCount)); + assertThat("Expected committed offset to cover all emitted messages", offset.offset(), + is(messageCount)); reset(consumerSpy); } diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutConfigTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutConfigTest.java index 90a1f1b3a38..9d093397f1f 100644 --- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutConfigTest.java +++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutConfigTest.java @@ -34,8 +34,10 @@ public class KafkaSpoutConfigTest { @Test public void testBasic() { - KafkaSpoutConfig conf = KafkaSpoutConfig.builder("localhost:1234", "topic").build(); - assertEquals(conf.getFirstPollOffsetStrategy(), FirstPollOffsetStrategy.UNCOMMITTED_EARLIEST); + KafkaSpoutConfig conf = KafkaSpoutConfig.builder("localhost:1234", "topic") + .build(); + assertEquals(conf.getFirstPollOffsetStrategy(), + FirstPollOffsetStrategy.UNCOMMITTED_EARLIEST); assertNull(conf.getConsumerGroupId()); assertTrue(conf.getTranslator() instanceof DefaultRecordTranslator); HashMap expected = new HashMap<>(); @@ -45,12 +47,14 @@ public void testBasic() { expected.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); expected.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); assertEquals(conf.getKafkaProps(), expected); - assertEquals(conf.getMetricsTimeBucketSizeInSecs(), KafkaSpoutConfig.DEFAULT_METRICS_TIME_BUCKET_SIZE_SECONDS); + assertEquals(conf.getMetricsTimeBucketSizeInSecs(), + KafkaSpoutConfig.DEFAULT_METRICS_TIME_BUCKET_SIZE_SECONDS); } @Test public void testSetEmitNullTuplesToTrue() { - final KafkaSpoutConfig conf = KafkaSpoutConfig.builder("localhost:1234", "topic") + final KafkaSpoutConfig conf = KafkaSpoutConfig.builder("localhost:1234", + "topic") .setEmitNullTuples(true) .build(); @@ -63,7 +67,8 @@ public void testShouldNotChangeAutoOffsetResetPolicyWhenNotUsingAtLeastOnce() { .setProcessingGuarantee(KafkaSpoutConfig.ProcessingGuarantee.AT_MOST_ONCE) .build(); - assertThat("When at-least-once is not specified, the spout should use the Kafka default auto offset reset policy", + assertThat("When at-least-once is not specified, the spout should use the Kafka default " + + "auto offset reset policy", conf.getKafkaProps().get(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG), nullValue()); } @@ -73,7 +78,8 @@ public void testWillRespectExplicitAutoOffsetResetPolicy() { .setProp(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "none") .build(); - assertThat("Should allow users to pick a different auto offset reset policy than the one recommended for the at-least-once processing guarantee", + assertThat("Should allow users to pick a different auto offset reset policy than the one " + + "recommended for the at-least-once processing guarantee", conf.getKafkaProps().get(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG), is("none")); } @@ -88,7 +94,8 @@ public void testMetricsTimeBucketSizeInSecs() { @Test public void testThrowsIfEnableAutoCommitIsSet() { - Assertions.assertThrows(IllegalStateException.class, () -> KafkaSpoutConfig.builder("localhost:1234", "topic") + Assertions.assertThrows(IllegalStateException.class, () -> KafkaSpoutConfig + .builder("localhost:1234", "topic") .setProp(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, true) .build()); } diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutEmitTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutEmitTest.java index 4d69633f1bd..9118509fee5 100755 --- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutEmitTest.java +++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutEmitTest.java @@ -22,6 +22,7 @@ import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.reset; import static org.mockito.Mockito.times; @@ -45,15 +46,12 @@ import org.apache.storm.kafka.spout.subscription.TopicFilter; import org.apache.storm.spout.SpoutOutputCollector; import org.apache.storm.task.TopologyContext; -import org.apache.storm.utils.Time; import org.apache.storm.utils.Time.SimulatedTime; -import org.mockito.ArgumentCaptor; -import org.mockito.InOrder; - -import static org.mockito.Mockito.mock; - +import org.apache.storm.utils.Time; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.InOrder; public class KafkaSpoutEmitTest { @@ -61,13 +59,15 @@ public class KafkaSpoutEmitTest { private final TopologyContext contextMock = mock(TopologyContext.class); private final SpoutOutputCollector collectorMock = mock(SpoutOutputCollector.class); private final Map conf = new HashMap<>(); - private final TopicPartition partition = new TopicPartition(SingleTopicKafkaSpoutConfiguration.TOPIC, 1); + private final TopicPartition partition = + new TopicPartition(SingleTopicKafkaSpoutConfiguration.TOPIC, 1); private KafkaConsumer consumerMock; private KafkaSpoutConfig spoutConfig; @BeforeEach public void setUp() { - spoutConfig = createKafkaSpoutConfigBuilder(mock(TopicFilter.class), mock(ManualPartitioner.class), -1) + spoutConfig = createKafkaSpoutConfigBuilder(mock(TopicFilter.class), + mock(ManualPartitioner.class), -1) .setOffsetCommitPeriodMs(offsetCommitPeriodMs) .build(); consumerMock = mock(KafkaConsumer.class); @@ -75,9 +75,10 @@ public void setUp() { @Test public void testNextTupleEmitsAtMostOneTuple() { - //The spout should emit at most one message per call to nextTuple - //This is necessary for Storm to be able to throttle the spout according to maxSpoutPending - KafkaSpout spout = SpoutWithMockedConsumerSetupHelper.setupSpout(spoutConfig, conf, contextMock, collectorMock, consumerMock, partition); + // The spout should emit at most one message per call to nextTuple + // This is necessary for Storm to be able to throttle the spout according to maxSpoutPending + KafkaSpout spout = SpoutWithMockedConsumerSetupHelper + .setupSpout(spoutConfig, conf, contextMock, collectorMock, consumerMock, partition); Map>> records = new HashMap<>(); records.put(partition, SpoutWithMockedConsumerSetupHelper.createRecords(partition, 0, 10)); @@ -86,20 +87,27 @@ public void testNextTupleEmitsAtMostOneTuple() { spout.nextTuple(); - verify(collectorMock, times(1)).emit(anyString(), anyList(), any(KafkaSpoutMessageId.class)); + verify(collectorMock, times(1)).emit(anyString(), anyList(), + any(KafkaSpoutMessageId.class)); } @Test public void testNextTupleEmitsFailedMessagesEvenWhenMaxUncommittedOffsetsIsExceeded() throws IOException { - //The spout must reemit failed messages waiting for retry even if it is not allowed to poll for new messages due to maxUncommittedOffsets being exceeded + // The spout must reemit failed messages waiting for retry even if it is not allowed to poll + // for new messages due to maxUncommittedOffsets being exceeded - //Emit maxUncommittedOffsets messages, and fail all of them. Then ensure that the spout will retry them when the retry backoff has passed + // Emit maxUncommittedOffsets messages, and fail all of them. Then ensure that the spout + // will retry them when the retry backoff has passed try (SimulatedTime simulatedTime = new SimulatedTime()) { - KafkaSpout spout = SpoutWithMockedConsumerSetupHelper.setupSpout(spoutConfig, conf, contextMock, collectorMock, consumerMock, partition); + KafkaSpout spout = SpoutWithMockedConsumerSetupHelper + .setupSpout(spoutConfig, conf, contextMock, collectorMock, consumerMock, + partition); Map>> records = new HashMap<>(); int numRecords = spoutConfig.getMaxUncommittedOffsets(); - //This is cheating a bit since maxPollRecords would normally spread this across multiple polls - records.put(partition, SpoutWithMockedConsumerSetupHelper.createRecords(partition, 0, numRecords)); + // This is cheating a bit since maxPollRecords would normally spread this across + // multiple polls + records.put(partition, SpoutWithMockedConsumerSetupHelper.createRecords(partition, 0, + numRecords)); when(consumerMock.poll(any(Duration.class))) .thenReturn(new ConsumerRecords<>(records)); @@ -108,8 +116,10 @@ public void testNextTupleEmitsFailedMessagesEvenWhenMaxUncommittedOffsetsIsExcee spout.nextTuple(); } - ArgumentCaptor messageIds = ArgumentCaptor.forClass(KafkaSpoutMessageId.class); - verify(collectorMock, times(numRecords)).emit(anyString(), anyList(), messageIds.capture()); + ArgumentCaptor messageIds = ArgumentCaptor + .forClass(KafkaSpoutMessageId.class); + verify(collectorMock, times(numRecords)).emit(anyString(), anyList(), messageIds + .capture()); for (KafkaSpoutMessageId messageId : messageIds.getAllValues()) { spout.fail(messageId); @@ -118,15 +128,17 @@ public void testNextTupleEmitsFailedMessagesEvenWhenMaxUncommittedOffsetsIsExcee reset(collectorMock); Time.advanceTime(50); - //No backoff for test retry service, just check that messages will retry immediately + // No backoff for test retry service, just check that messages will retry immediately for (int i = 0; i < numRecords; i++) { spout.nextTuple(); } - ArgumentCaptor retryMessageIds = ArgumentCaptor.forClass(KafkaSpoutMessageId.class); - verify(collectorMock, times(numRecords)).emit(anyString(), anyList(), retryMessageIds.capture()); + ArgumentCaptor retryMessageIds = ArgumentCaptor + .forClass(KafkaSpoutMessageId.class); + verify(collectorMock, times(numRecords)).emit(anyString(), anyList(), retryMessageIds + .capture()); - //Verify that the poll started at the earliest retriable tuple offset + // Verify that the poll started at the earliest retriable tuple offset List failedOffsets = new ArrayList<>(); for (KafkaSpoutMessageId msgId : messageIds.getAllValues()) { failedOffsets.add(msgId.offset()); @@ -139,15 +151,22 @@ public void testNextTupleEmitsFailedMessagesEvenWhenMaxUncommittedOffsetsIsExcee @Test public void testSpoutWillSkipPartitionsAtTheMaxUncommittedOffsetsLimit() { - //This verifies that partitions can't prevent each other from retrying tuples due to the maxUncommittedOffsets limit. + // This verifies that partitions can't prevent each other from retrying tuples due to the + // maxUncommittedOffsets limit. try (SimulatedTime simulatedTime = new SimulatedTime()) { - TopicPartition partitionTwo = new TopicPartition(SingleTopicKafkaSpoutConfiguration.TOPIC, 2); - KafkaSpout spout = SpoutWithMockedConsumerSetupHelper.setupSpout(spoutConfig, conf, contextMock, collectorMock, consumerMock, partition, partitionTwo); + TopicPartition partitionTwo = + new TopicPartition(SingleTopicKafkaSpoutConfiguration.TOPIC, 2); + KafkaSpout spout = SpoutWithMockedConsumerSetupHelper + .setupSpout(spoutConfig, conf, contextMock, collectorMock, consumerMock, + partition, partitionTwo); Map>> records = new HashMap<>(); - //This is cheating a bit since maxPollRecords would normally spread this across multiple polls - records.put(partition, SpoutWithMockedConsumerSetupHelper.createRecords(partition, 0, spoutConfig.getMaxUncommittedOffsets())); - records.put(partitionTwo, SpoutWithMockedConsumerSetupHelper.createRecords(partitionTwo, 0, spoutConfig.getMaxUncommittedOffsets() + 1)); - int numMessages = spoutConfig.getMaxUncommittedOffsets()*2 + 1; + // This is cheating a bit since maxPollRecords would normally spread this across + // multiple polls + records.put(partition, SpoutWithMockedConsumerSetupHelper.createRecords(partition, 0, + spoutConfig.getMaxUncommittedOffsets())); + records.put(partitionTwo, SpoutWithMockedConsumerSetupHelper.createRecords(partitionTwo, + 0, spoutConfig.getMaxUncommittedOffsets() + 1)); + int numMessages = spoutConfig.getMaxUncommittedOffsets() * 2 + 1; when(consumerMock.poll(any(Duration.class))) .thenReturn(new ConsumerRecords<>(records)); @@ -156,20 +175,27 @@ public void testSpoutWillSkipPartitionsAtTheMaxUncommittedOffsetsLimit() { spout.nextTuple(); } - ArgumentCaptor messageIds = ArgumentCaptor.forClass(KafkaSpoutMessageId.class); - verify(collectorMock, times(numMessages)).emit(anyString(), anyList(), messageIds.capture()); + ArgumentCaptor messageIds = ArgumentCaptor + .forClass(KafkaSpoutMessageId.class); + verify(collectorMock, times(numMessages)).emit(anyString(), anyList(), messageIds + .capture()); - //Now fail a tuple on partition one and verify that it is allowed to retry, because the failed tuple is below the maxUncommittedOffsets limit - Optional failedMessageIdPartitionOne = messageIds.getAllValues().stream() + // Now fail a tuple on partition one and verify that it is allowed to retry, because the + // failed tuple is below the maxUncommittedOffsets limit + Optional failedMessageIdPartitionOne = messageIds.getAllValues() + .stream() .filter(messageId -> messageId.partition() == partition.partition()) .findAny(); spout.fail(failedMessageIdPartitionOne.get()); - //Also fail the last tuple from partition two. Since the failed tuple is beyond the maxUncommittedOffsets limit, it should not be retried until earlier messages are acked. - Optional failedMessagePartitionTwo = messageIds.getAllValues().stream() + // Also fail the last tuple from partition two. Since the failed tuple is beyond the + // maxUncommittedOffsets limit, it should not be retried until earlier messages are + // acked. + Optional failedMessagePartitionTwo = messageIds.getAllValues() + .stream() .filter(messageId -> messageId.partition() == partitionTwo.partition()) - .max((msgId, msgId2) -> (int)(msgId.offset() - msgId2.offset())); + .max((msgId, msgId2) -> (int) (msgId.offset() - msgId2.offset())); spout.fail(failedMessagePartitionTwo.get()); @@ -177,15 +203,18 @@ public void testSpoutWillSkipPartitionsAtTheMaxUncommittedOffsetsLimit() { Time.advanceTime(50); when(consumerMock.poll(any(Duration.class))) - .thenReturn(new ConsumerRecords<>(Collections.singletonMap(partition, SpoutWithMockedConsumerSetupHelper.createRecords(partition, failedMessageIdPartitionOne.get().offset(), 1)))); + .thenReturn(new ConsumerRecords<>(Collections.singletonMap(partition, + SpoutWithMockedConsumerSetupHelper.createRecords(partition, + failedMessageIdPartitionOne.get().offset(), 1)))); spout.nextTuple(); verify(collectorMock, times(1)).emit(anyString(), anyList(), any()); InOrder inOrder = inOrder(consumerMock); - inOrder.verify(consumerMock).seek(partition, failedMessageIdPartitionOne.get().offset()); - //Should not seek on the paused partition + inOrder.verify(consumerMock).seek(partition, failedMessageIdPartitionOne.get() + .offset()); + // Should not seek on the paused partition inOrder.verify(consumerMock, never()).seek(eq(partitionTwo), anyLong()); inOrder.verify(consumerMock).pause(Collections.singleton(partitionTwo)); inOrder.verify(consumerMock).poll(any(Duration.class)); @@ -193,7 +222,8 @@ public void testSpoutWillSkipPartitionsAtTheMaxUncommittedOffsetsLimit() { reset(collectorMock); - //Now also check that no more tuples are polled for, since both partitions are at their limits + // Now also check that no more tuples are polled for, since both partitions are at their + // limits spout.nextTuple(); verify(collectorMock, never()).emit(anyString(), anyList(), any()); diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutLogCompactionSupportTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutLogCompactionSupportTest.java index a19bb9d265f..67b9a37f8bc 100644 --- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutLogCompactionSupportTest.java +++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutLogCompactionSupportTest.java @@ -21,8 +21,8 @@ import static org.hamcrest.Matchers.hasKey; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.inOrder; import static org.mockito.Mockito.mock; @@ -37,15 +37,18 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import org.apache.kafka.clients.consumer.*; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.common.TopicPartition; import org.apache.storm.kafka.spout.config.builder.SingleTopicKafkaSpoutConfiguration; import org.apache.storm.kafka.spout.subscription.ManualPartitioner; import org.apache.storm.kafka.spout.subscription.TopicFilter; import org.apache.storm.spout.SpoutOutputCollector; import org.apache.storm.task.TopologyContext; -import org.apache.storm.utils.Time; import org.apache.storm.utils.Time.SimulatedTime; +import org.apache.storm.utils.Time; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.ArgumentCaptor; @@ -59,7 +62,8 @@ public class KafkaSpoutLogCompactionSupportTest { private final TopologyContext contextMock = mock(TopologyContext.class); private final SpoutOutputCollector collectorMock = mock(SpoutOutputCollector.class); private final Map conf = new HashMap<>(); - private final TopicPartition partition = new TopicPartition(SingleTopicKafkaSpoutConfiguration.TOPIC, 1); + private final TopicPartition partition = + new TopicPartition(SingleTopicKafkaSpoutConfiguration.TOPIC, 1); private KafkaConsumer consumerMock; private KafkaSpoutConfig spoutConfig; @@ -69,7 +73,8 @@ public class KafkaSpoutLogCompactionSupportTest { @BeforeEach public void setUp() { MockitoAnnotations.initMocks(this); - spoutConfig = createKafkaSpoutConfigBuilder(mock(TopicFilter.class), mock(ManualPartitioner.class), -1) + spoutConfig = createKafkaSpoutConfigBuilder(mock(TopicFilter.class), + mock(ManualPartitioner.class), -1) .setOffsetCommitPeriodMs(offsetCommitPeriodMs) .build(); consumerMock = mock(KafkaConsumer.class); @@ -77,14 +82,18 @@ public void setUp() { @Test public void testCommitSuccessWithOffsetVoids() { - //Verify that the commit logic can handle offset voids due to log compaction + // Verify that the commit logic can handle offset voids due to log compaction try (SimulatedTime simulatedTime = new SimulatedTime()) { - KafkaSpout spout = SpoutWithMockedConsumerSetupHelper.setupSpout(spoutConfig, conf, contextMock, collectorMock, consumerMock, partition); + KafkaSpout spout = SpoutWithMockedConsumerSetupHelper + .setupSpout(spoutConfig, conf, contextMock, collectorMock, consumerMock, + partition); Map>> records = new HashMap<>(); List> recordsForPartition = new ArrayList<>(); // Offsets emitted are 0,1,2,3,4,,8,9 - recordsForPartition.addAll(SpoutWithMockedConsumerSetupHelper.createRecords(partition, 0, 5)); - recordsForPartition.addAll(SpoutWithMockedConsumerSetupHelper.createRecords(partition, 8, 2)); + recordsForPartition.addAll(SpoutWithMockedConsumerSetupHelper.createRecords(partition, + 0, 5)); + recordsForPartition.addAll(SpoutWithMockedConsumerSetupHelper.createRecords(partition, + 8, 2)); records.put(partition, recordsForPartition); when(consumerMock.poll(any(Duration.class))) @@ -94,14 +103,17 @@ public void testCommitSuccessWithOffsetVoids() { spout.nextTuple(); } - ArgumentCaptor messageIds = ArgumentCaptor.forClass(KafkaSpoutMessageId.class); - verify(collectorMock, times(recordsForPartition.size())).emit(anyString(), anyList(), messageIds.capture()); + ArgumentCaptor messageIds = ArgumentCaptor + .forClass(KafkaSpoutMessageId.class); + verify(collectorMock, times(recordsForPartition.size())).emit(anyString(), anyList(), + messageIds.capture()); for (KafkaSpoutMessageId messageId : messageIds.getAllValues()) { spout.ack(messageId); } - // Advance time and then trigger first call to kafka consumer commit; the commit must progress to offset 9 + // Advance time and then trigger first call to kafka consumer commit; the commit must + // progress to offset 9 Time.advanceTime(KafkaSpout.TIMER_DELAY_MS + offsetCommitPeriodMs); when(consumerMock.poll(any(Duration.class))) .thenReturn(new ConsumerRecords<>(Collections.emptyMap())); @@ -111,7 +123,8 @@ public void testCommitSuccessWithOffsetVoids() { inOrder.verify(consumerMock).commitSync(commitCapture.capture()); inOrder.verify(consumerMock).poll(any(Duration.class)); - //verify that Offset 10 was last committed offset, since this is the offset the spout should resume at + // verify that Offset 10 was last committed offset, since this is the offset the spout + // should resume at Map commits = commitCapture.getValue(); assertTrue(commits.containsKey(partition)); assertEquals(10, commits.get(partition).offset()); @@ -119,16 +132,20 @@ public void testCommitSuccessWithOffsetVoids() { } /** - Verify that failed offsets will only retry if the corresponding message exists. - When log compaction is enabled in Kafka it is possible that a tuple can fail, - and then be impossible to retry because the message in Kafka has been deleted. - The spout needs to quietly ack such tuples to allow commits to progress past the deleted offset. + * Verify that failed offsets will only retry if the corresponding message exists. + * When log compaction is enabled in Kafka it is possible that a tuple can fail, + * and then be impossible to retry because the message in Kafka has been deleted. + * The spout needs to quietly ack such tuples to allow commits to progress past the deleted + * offset. */ @Test public void testWillSkipRetriableTuplesIfOffsetsAreCompactedAway() { try (SimulatedTime ignored = new SimulatedTime()) { - TopicPartition partitionTwo = new TopicPartition(SingleTopicKafkaSpoutConfiguration.TOPIC, 2); - KafkaSpout spout = SpoutWithMockedConsumerSetupHelper.setupSpout(spoutConfig, conf, contextMock, collectorMock, consumerMock, partition, partitionTwo); + TopicPartition partitionTwo = + new TopicPartition(SingleTopicKafkaSpoutConfiguration.TOPIC, 2); + KafkaSpout spout = SpoutWithMockedConsumerSetupHelper + .setupSpout(spoutConfig, conf, contextMock, collectorMock, consumerMock, + partition, partitionTwo); List firstPartitionMsgIds = SpoutWithMockedConsumerSetupHelper .pollAndEmit(spout, consumerMock, 3, collectorMock, partition, 0, 1, 2); @@ -137,20 +154,23 @@ public void testWillSkipRetriableTuplesIfOffsetsAreCompactedAway() { .pollAndEmit(spout, consumerMock, 3, collectorMock, partitionTwo, 0, 1, 2); reset(collectorMock); - for(int i = 0; i < 3; i++) { + for (int i = 0; i < 3; i++) { spout.fail(firstPartitionMsgIds.get(i)); spout.fail(secondPartitionMsgIds.get(i)); } Time.advanceTime(50); - //The failed tuples are ready for retry. Make it appear like 0 and 1 on the first partition were compacted away. - //In this case the second partition acts as control to verify that we only skip past offsets that are no longer present. + // The failed tuples are ready for retry. Make it appear like 0 and 1 on the first + // partition were compacted away. + // In this case the second partition acts as control to verify that we only skip past + // offsets that are no longer present. Map retryOffsets = new HashMap<>(); retryOffsets.put(partition, new int[] {2}); retryOffsets.put(partitionTwo, new int[] {0, 1, 2}); - int expectedEmits = 4; //2 on first partition, 0-2 on second partition - List retryMessageIds = SpoutWithMockedConsumerSetupHelper.pollAndEmit(spout, consumerMock, expectedEmits, collectorMock, retryOffsets); + int expectedEmits = 4; // 2 on first partition, 0-2 on second partition + List retryMessageIds = SpoutWithMockedConsumerSetupHelper + .pollAndEmit(spout, consumerMock, expectedEmits, collectorMock, retryOffsets); Time.advanceTime(KafkaSpout.TIMER_DELAY_MS + offsetCommitPeriodMs); spout.nextTuple(); @@ -158,13 +178,15 @@ public void testWillSkipRetriableTuplesIfOffsetsAreCompactedAway() { verify(consumerMock).commitSync(commitCapture.capture()); Map committed = commitCapture.getValue(); assertThat(committed.keySet(), is(Collections.singleton(partition))); - assertThat("The first partition should have committed up to the first retriable tuple that is not missing", committed.get(partition).offset(), is(2L)); + assertThat("The first partition should have committed up to the first retriable tuple " + + "that is not missing", committed.get(partition).offset(), is(2L)); - for(KafkaSpoutMessageId msgId : retryMessageIds) { + for (KafkaSpoutMessageId msgId : retryMessageIds) { spout.ack(msgId); } - //The spout should now commit all the offsets, since all offsets are either acked or were missing when retrying + // The spout should now commit all the offsets, since all offsets are either acked or + // were missing when retrying Time.advanceTime(KafkaSpout.TIMER_DELAY_MS + offsetCommitPeriodMs); spout.nextTuple(); @@ -179,9 +201,12 @@ public void testWillSkipRetriableTuplesIfOffsetsAreCompactedAway() { @Test public void testWillSkipRetriableTuplesIfOffsetsAreCompactedAwayWithoutAckingPendingTuples() { - //Demonstrate that the spout doesn't ack pending tuples when skipping compacted tuples. The pending tuples should be allowed to finish normally. + // Demonstrate that the spout doesn't ack pending tuples when skipping compacted tuples. The + // pending tuples should be allowed to finish normally. try (SimulatedTime ignored = new SimulatedTime()) { - KafkaSpout spout = SpoutWithMockedConsumerSetupHelper.setupSpout(spoutConfig, conf, contextMock, collectorMock, consumerMock, partition); + KafkaSpout spout = SpoutWithMockedConsumerSetupHelper + .setupSpout(spoutConfig, conf, contextMock, collectorMock, consumerMock, + partition); List firstPartitionMsgIds = SpoutWithMockedConsumerSetupHelper .pollAndEmit(spout, consumerMock, 3, collectorMock, partition, 0, 1, 2); @@ -192,9 +217,11 @@ public void testWillSkipRetriableTuplesIfOffsetsAreCompactedAwayWithoutAckingPen Time.advanceTime(50); - //The failed tuples are ready for retry. Make it appear like 0 and 1 were compacted away. - List retryMessageIds = SpoutWithMockedConsumerSetupHelper.pollAndEmit(spout, consumerMock, 1, collectorMock, partition, 2); - for(KafkaSpoutMessageId msgId : retryMessageIds) { + // The failed tuples are ready for retry. Make it appear like 0 and 1 were compacted + // away. + List retryMessageIds = SpoutWithMockedConsumerSetupHelper + .pollAndEmit(spout, consumerMock, 1, collectorMock, partition, 2); + for (KafkaSpoutMessageId msgId : retryMessageIds) { spout.ack(msgId); } @@ -204,7 +231,8 @@ public void testWillSkipRetriableTuplesIfOffsetsAreCompactedAwayWithoutAckingPen verify(consumerMock).commitSync(commitCapture.capture()); Map committed = commitCapture.getValue(); assertThat(committed.keySet(), is(Collections.singleton(partition))); - assertThat("The first partition should have committed the missing offset, but no further since the next tuple is pending", + assertThat("The first partition should have committed the missing offset, but no " + + "further since the next tuple is pending", committed.get(partition).offset(), is(1L)); spout.ack(firstPartitionMsgIds.get(1)); @@ -215,21 +243,26 @@ public void testWillSkipRetriableTuplesIfOffsetsAreCompactedAwayWithoutAckingPen verify(consumerMock, times(2)).commitSync(commitCapture.capture()); committed = commitCapture.getValue(); assertThat(committed.keySet(), is(Collections.singleton(partition))); - assertThat("The first partition should have committed all offsets", committed.get(partition).offset(), is(3L)); + assertThat("The first partition should have committed all offsets", committed + .get(partition).offset(), is(3L)); } } @Test public void testCommitTupleAfterCompactionGap() { - //If there is an acked tupled after a compaction gap, the spout should commit it immediately + // If there is an acked tupled after a compaction gap, the spout should commit it + // immediately try (SimulatedTime simulatedTime = new SimulatedTime()) { - KafkaSpout spout = SpoutWithMockedConsumerSetupHelper.setupSpout(spoutConfig, conf, contextMock, collectorMock, consumerMock, partition); + KafkaSpout spout = SpoutWithMockedConsumerSetupHelper + .setupSpout(spoutConfig, conf, contextMock, collectorMock, consumerMock, + partition); List firstMessage = SpoutWithMockedConsumerSetupHelper .pollAndEmit(spout, consumerMock, 1, collectorMock, partition, 0); reset(collectorMock); - List messageAfterGap = SpoutWithMockedConsumerSetupHelper.pollAndEmit(spout, consumerMock, 1, collectorMock, partition, 2); + List messageAfterGap = SpoutWithMockedConsumerSetupHelper + .pollAndEmit(spout, consumerMock, 1, collectorMock, partition, 2); reset(collectorMock); spout.ack(firstMessage.get(0)); @@ -251,9 +284,10 @@ public void testCommitTupleAfterCompactionGap() { verify(consumerMock).commitSync(commitCapture.capture()); committed = commitCapture.getValue(); assertThat(committed.keySet(), is(Collections.singleton(partition))); - assertThat("The consumer should have committed the offset after the gap, since offset 1 wasn't emitted and both 0 and 2 are acked", + assertThat("The consumer should have committed the offset after the gap, since offset " + + "1 wasn't emitted and both 0 and 2 are acked", committed.get(partition).offset(), is(3L)); } } -} \ No newline at end of file +} diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutMessagingGuaranteeTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutMessagingGuaranteeTest.java index 44c5d0a031e..0b29ce790ef 100644 --- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutMessagingGuaranteeTest.java +++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutMessagingGuaranteeTest.java @@ -51,8 +51,8 @@ import org.apache.storm.kafka.spout.subscription.TopicFilter; import org.apache.storm.spout.SpoutOutputCollector; import org.apache.storm.task.TopologyContext; -import org.apache.storm.utils.Time; import org.apache.storm.utils.Time.SimulatedTime; +import org.apache.storm.utils.Time; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -70,7 +70,8 @@ public class KafkaSpoutMessagingGuaranteeTest { private final TopologyContext contextMock = mock(TopologyContext.class); private final SpoutOutputCollector collectorMock = mock(SpoutOutputCollector.class); private final Map conf = new HashMap<>(); - private final TopicPartition partition = new TopicPartition(SingleTopicKafkaSpoutConfiguration.TOPIC, 1); + private final TopicPartition partition = + new TopicPartition(SingleTopicKafkaSpoutConfiguration.TOPIC, 1); private KafkaConsumer consumerMock; @BeforeEach @@ -80,50 +81,66 @@ public void setUp() { @Test public void testAtMostOnceModeCommitsBeforeEmit() { - //At-most-once mode must commit tuples before they are emitted to the topology to ensure that a spout crash won't cause replays. - KafkaSpoutConfig spoutConfig = createKafkaSpoutConfigBuilder(mock(TopicFilter.class), mock(ManualPartitioner.class), -1) + // At-most-once mode must commit tuples before they are emitted to the topology to ensure + // that a spout crash won't cause replays. + KafkaSpoutConfig spoutConfig = + createKafkaSpoutConfigBuilder(mock(TopicFilter.class), + mock(ManualPartitioner.class), -1) .setProcessingGuarantee(KafkaSpoutConfig.ProcessingGuarantee.AT_MOST_ONCE) .build(); - KafkaSpout spout = SpoutWithMockedConsumerSetupHelper.setupSpout(spoutConfig, conf, contextMock, collectorMock, consumerMock, partition); + KafkaSpout spout = SpoutWithMockedConsumerSetupHelper + .setupSpout(spoutConfig, conf, contextMock, collectorMock, consumerMock, partition); - when(consumerMock.poll(any(Duration.class))).thenReturn(new ConsumerRecords<>(Collections.singletonMap(partition, + when(consumerMock.poll(any(Duration.class))).thenReturn(new ConsumerRecords<>(Collections + .singletonMap(partition, SpoutWithMockedConsumerSetupHelper.createRecords(partition, 0, 1)))); spout.nextTuple(); - //The spout should have emitted the tuple, and must have committed it before emit + // The spout should have emitted the tuple, and must have committed it before emit InOrder inOrder = inOrder(consumerMock, collectorMock); inOrder.verify(consumerMock).poll(any(Duration.class)); inOrder.verify(consumerMock).commitSync(commitCapture.capture()); - inOrder.verify(collectorMock).emit(eq(SingleTopicKafkaSpoutConfiguration.STREAM), anyList()); + inOrder.verify(collectorMock).emit(eq(SingleTopicKafkaSpoutConfiguration.STREAM), + anyList()); - CommitMetadataManager metadataManager = new CommitMetadataManager(contextMock, KafkaSpoutConfig.ProcessingGuarantee.AT_MOST_ONCE); + CommitMetadataManager metadataManager = new CommitMetadataManager(contextMock, + KafkaSpoutConfig.ProcessingGuarantee.AT_MOST_ONCE); Map committedOffsets = commitCapture.getValue(); assertThat(committedOffsets.get(partition).offset(), is(0L)); - assertThat(committedOffsets.get(partition).metadata(), is(metadataManager.getCommitMetadata())); + assertThat(committedOffsets.get(partition).metadata(), is(metadataManager + .getCommitMetadata())); } - private void doTestModeDisregardsMaxUncommittedOffsets(KafkaSpoutConfig spoutConfig) { - KafkaSpout spout = SpoutWithMockedConsumerSetupHelper.setupSpout(spoutConfig, conf, contextMock, collectorMock, consumerMock, partition); + private void doTestModeDisregardsMaxUncommittedOffsets(KafkaSpoutConfig spoutConfig) { + KafkaSpout spout = SpoutWithMockedConsumerSetupHelper + .setupSpout(spoutConfig, conf, contextMock, collectorMock, consumerMock, partition); when(consumerMock.poll(any(Duration.class))) .thenReturn(new ConsumerRecords<>(Collections.singletonMap(partition, - SpoutWithMockedConsumerSetupHelper.createRecords(partition, 0, spoutConfig.getMaxUncommittedOffsets())))) + SpoutWithMockedConsumerSetupHelper.createRecords(partition, 0, spoutConfig + .getMaxUncommittedOffsets())))) .thenReturn(new ConsumerRecords<>(Collections.singletonMap(partition, - SpoutWithMockedConsumerSetupHelper.createRecords(partition, spoutConfig.getMaxUncommittedOffsets() - 1, spoutConfig.getMaxUncommittedOffsets())))); + SpoutWithMockedConsumerSetupHelper.createRecords(partition, spoutConfig + .getMaxUncommittedOffsets() - 1, spoutConfig.getMaxUncommittedOffsets())))); for (int i = 0; i < spoutConfig.getMaxUncommittedOffsets() * 2; i++) { spout.nextTuple(); } verify(consumerMock, times(2)).poll(any(Duration.class)); - verify(collectorMock, times(spoutConfig.getMaxUncommittedOffsets() * 2)).emit(eq(SingleTopicKafkaSpoutConfiguration.STREAM), anyList()); + verify(collectorMock, times(spoutConfig.getMaxUncommittedOffsets() * 2)) + .emit(eq(SingleTopicKafkaSpoutConfiguration.STREAM), anyList()); } @Test public void testAtMostOnceModeDisregardsMaxUncommittedOffsets() { - //The maxUncommittedOffsets limit should not be enforced, since it is only meaningful in at-least-once mode - KafkaSpoutConfig spoutConfig = createKafkaSpoutConfigBuilder(mock(TopicFilter.class), mock(ManualPartitioner.class), -1) + // The maxUncommittedOffsets limit should not be enforced, since it is only meaningful in + // at-least-once mode + KafkaSpoutConfig spoutConfig = + createKafkaSpoutConfigBuilder(mock(TopicFilter.class), + mock(ManualPartitioner.class), -1) .setProcessingGuarantee(KafkaSpoutConfig.ProcessingGuarantee.AT_MOST_ONCE) .build(); doTestModeDisregardsMaxUncommittedOffsets(spoutConfig); @@ -131,23 +148,30 @@ public void testAtMostOnceModeDisregardsMaxUncommittedOffsets() { @Test public void testNoGuaranteeModeDisregardsMaxUncommittedOffsets() { - //The maxUncommittedOffsets limit should not be enforced, since it is only meaningful in at-least-once mode - KafkaSpoutConfig spoutConfig = createKafkaSpoutConfigBuilder(mock(TopicFilter.class), mock(ManualPartitioner.class), -1) + // The maxUncommittedOffsets limit should not be enforced, since it is only meaningful in + // at-least-once mode + KafkaSpoutConfig spoutConfig = + createKafkaSpoutConfigBuilder(mock(TopicFilter.class), + mock(ManualPartitioner.class), -1) .setProcessingGuarantee(KafkaSpoutConfig.ProcessingGuarantee.NO_GUARANTEE) .build(); doTestModeDisregardsMaxUncommittedOffsets(spoutConfig); } private void doTestModeCannotReplayTuples(KafkaSpoutConfig spoutConfig) { - KafkaSpout spout = SpoutWithMockedConsumerSetupHelper.setupSpout(spoutConfig, conf, contextMock, collectorMock, consumerMock, partition); + KafkaSpout spout = SpoutWithMockedConsumerSetupHelper + .setupSpout(spoutConfig, conf, contextMock, collectorMock, consumerMock, partition); - when(consumerMock.poll(any(Duration.class))).thenReturn(new ConsumerRecords<>(Collections.singletonMap(partition, + when(consumerMock.poll(any(Duration.class))).thenReturn(new ConsumerRecords<>(Collections + .singletonMap(partition, SpoutWithMockedConsumerSetupHelper.createRecords(partition, 0, 1)))); spout.nextTuple(); - ArgumentCaptor msgIdCaptor = ArgumentCaptor.forClass(KafkaSpoutMessageId.class); - verify(collectorMock).emit(eq(SingleTopicKafkaSpoutConfiguration.STREAM), anyList(), msgIdCaptor.capture()); + ArgumentCaptor msgIdCaptor = ArgumentCaptor + .forClass(KafkaSpoutMessageId.class); + verify(collectorMock).emit(eq(SingleTopicKafkaSpoutConfiguration.STREAM), anyList(), + msgIdCaptor.capture()); assertThat("Should have captured a message id", msgIdCaptor.getValue(), not(nullValue())); spout.fail(msgIdCaptor.getValue()); @@ -156,14 +180,17 @@ private void doTestModeCannotReplayTuples(KafkaSpoutConfig spout spout.nextTuple(); - //The consumer should not be seeking to retry the failed tuple, it should just be continuing from the current position + // The consumer should not be seeking to retry the failed tuple, it should just be + // continuing from the current position verify(consumerMock, never()).seek(eq(partition), anyLong()); } @Test public void testAtMostOnceModeCannotReplayTuples() { - //When tuple tracking is enabled, the spout must not replay tuples in at-most-once mode - KafkaSpoutConfig spoutConfig = createKafkaSpoutConfigBuilder(mock(TopicFilter.class), mock(ManualPartitioner.class), -1) + // When tuple tracking is enabled, the spout must not replay tuples in at-most-once mode + KafkaSpoutConfig spoutConfig = + createKafkaSpoutConfigBuilder(mock(TopicFilter.class), + mock(ManualPartitioner.class), -1) .setProcessingGuarantee(KafkaSpoutConfig.ProcessingGuarantee.AT_MOST_ONCE) .setTupleTrackingEnforced(true) .build(); @@ -172,8 +199,10 @@ public void testAtMostOnceModeCannotReplayTuples() { @Test public void testNoGuaranteeModeCannotReplayTuples() { - //When tuple tracking is enabled, the spout must not replay tuples in no guarantee mode - KafkaSpoutConfig spoutConfig = createKafkaSpoutConfigBuilder(mock(TopicFilter.class), mock(ManualPartitioner.class), -1) + // When tuple tracking is enabled, the spout must not replay tuples in no guarantee mode + KafkaSpoutConfig spoutConfig = + createKafkaSpoutConfigBuilder(mock(TopicFilter.class), + mock(ManualPartitioner.class), -1) .setProcessingGuarantee(KafkaSpoutConfig.ProcessingGuarantee.NO_GUARANTEE) .setTupleTrackingEnforced(true) .build(); @@ -182,57 +211,77 @@ public void testNoGuaranteeModeCannotReplayTuples() { @Test public void testAtMostOnceModeDoesNotCommitAckedTuples() { - //When tuple tracking is enabled, the spout must not commit acked tuples in at-most-once mode because they were committed before being emitted - KafkaSpoutConfig spoutConfig = createKafkaSpoutConfigBuilder(mock(TopicFilter.class), mock(ManualPartitioner.class), -1) + // When tuple tracking is enabled, the spout must not commit acked tuples in at-most-once + // mode because they were committed before being emitted + KafkaSpoutConfig spoutConfig = + createKafkaSpoutConfigBuilder(mock(TopicFilter.class), + mock(ManualPartitioner.class), -1) .setProcessingGuarantee(KafkaSpoutConfig.ProcessingGuarantee.AT_MOST_ONCE) .setTupleTrackingEnforced(true) .build(); try (SimulatedTime ignored = new SimulatedTime()) { - KafkaSpout spout = SpoutWithMockedConsumerSetupHelper.setupSpout(spoutConfig, conf, contextMock, collectorMock, consumerMock, partition); + KafkaSpout spout = SpoutWithMockedConsumerSetupHelper + .setupSpout(spoutConfig, conf, contextMock, collectorMock, consumerMock, + partition); - when(consumerMock.poll(any(Duration.class))).thenReturn(new ConsumerRecords<>(Collections.singletonMap(partition, + when(consumerMock.poll(any(Duration.class))) + .thenReturn(new ConsumerRecords<>(Collections.singletonMap(partition, SpoutWithMockedConsumerSetupHelper.createRecords(partition, 0, 1)))); spout.nextTuple(); clearInvocations(consumerMock); - ArgumentCaptor msgIdCaptor = ArgumentCaptor.forClass(KafkaSpoutMessageId.class); - verify(collectorMock).emit(eq(SingleTopicKafkaSpoutConfiguration.STREAM), anyList(), msgIdCaptor.capture()); - assertThat("Should have captured a message id", msgIdCaptor.getValue(), not(nullValue())); + ArgumentCaptor msgIdCaptor = ArgumentCaptor + .forClass(KafkaSpoutMessageId.class); + verify(collectorMock).emit(eq(SingleTopicKafkaSpoutConfiguration.STREAM), anyList(), + msgIdCaptor.capture()); + assertThat("Should have captured a message id", msgIdCaptor.getValue(), + not(nullValue())); spout.ack(msgIdCaptor.getValue()); Time.advanceTime(KafkaSpout.TIMER_DELAY_MS + spoutConfig.getOffsetsCommitPeriodMs()); - when(consumerMock.poll(any(Duration.class))).thenReturn(new ConsumerRecords<>(Collections.emptyMap())); + when(consumerMock.poll(any(Duration.class))) + .thenReturn(new ConsumerRecords<>(Collections.emptyMap())); spout.nextTuple(); - verify(consumerMock, never()).commitSync(argThat((Map arg) -> !arg.containsKey(partition))); + verify(consumerMock, never()).commitSync(argThat((Map arg) -> !arg.containsKey(partition))); } } @Test public void testNoGuaranteeModeCommitsPolledTuples() { - //When using the no guarantee mode, the spout must commit tuples periodically, regardless of whether they've been acked - KafkaSpoutConfig spoutConfig = createKafkaSpoutConfigBuilder(mock(TopicFilter.class), mock(ManualPartitioner.class), -1) + // When using the no guarantee mode, the spout must commit tuples periodically, regardless + // of whether they've been acked + KafkaSpoutConfig spoutConfig = + createKafkaSpoutConfigBuilder(mock(TopicFilter.class), + mock(ManualPartitioner.class), -1) .setProcessingGuarantee(KafkaSpoutConfig.ProcessingGuarantee.NO_GUARANTEE) .setTupleTrackingEnforced(true) .build(); try (SimulatedTime ignored = new SimulatedTime()) { - KafkaSpout spout = SpoutWithMockedConsumerSetupHelper.setupSpout(spoutConfig, conf, contextMock, collectorMock, consumerMock, partition); + KafkaSpout spout = SpoutWithMockedConsumerSetupHelper + .setupSpout(spoutConfig, conf, contextMock, collectorMock, consumerMock, + partition); - when(consumerMock.poll(any(Duration.class))).thenReturn(new ConsumerRecords<>(Collections.singletonMap(partition, + when(consumerMock.poll(any(Duration.class))) + .thenReturn(new ConsumerRecords<>(Collections.singletonMap(partition, SpoutWithMockedConsumerSetupHelper.createRecords(partition, 0, 1)))); spout.nextTuple(); when(consumerMock.position(partition)).thenReturn(1L); - ArgumentCaptor msgIdCaptor = ArgumentCaptor.forClass(KafkaSpoutMessageId.class); - verify(collectorMock).emit(eq(SingleTopicKafkaSpoutConfiguration.STREAM), anyList(), msgIdCaptor.capture()); - assertThat("Should have captured a message id", msgIdCaptor.getValue(), not(nullValue())); + ArgumentCaptor msgIdCaptor = ArgumentCaptor + .forClass(KafkaSpoutMessageId.class); + verify(collectorMock).emit(eq(SingleTopicKafkaSpoutConfiguration.STREAM), anyList(), + msgIdCaptor.capture()); + assertThat("Should have captured a message id", msgIdCaptor.getValue(), + not(nullValue())); Time.advanceTime(KafkaSpout.TIMER_DELAY_MS + spoutConfig.getOffsetsCommitPeriodMs()); @@ -240,24 +289,30 @@ public void testNoGuaranteeModeCommitsPolledTuples() { verify(consumerMock).commitAsync(commitCapture.capture(), isNull()); - CommitMetadataManager metadataManager = new CommitMetadataManager(contextMock, KafkaSpoutConfig.ProcessingGuarantee.NO_GUARANTEE); + CommitMetadataManager metadataManager = new CommitMetadataManager(contextMock, + KafkaSpoutConfig.ProcessingGuarantee.NO_GUARANTEE); Map committedOffsets = commitCapture.getValue(); assertThat(committedOffsets.get(partition).offset(), is(1L)); - assertThat(committedOffsets.get(partition).metadata(), is(metadataManager.getCommitMetadata())); + assertThat(committedOffsets.get(partition).metadata(), is(metadataManager + .getCommitMetadata())); } } private void doFilterNullTupleTest(KafkaSpoutConfig.ProcessingGuarantee processingGuarantee) { - //STORM-3059 - KafkaSpoutConfig spoutConfig = createKafkaSpoutConfigBuilder(mock(TopicFilter.class), mock(ManualPartitioner.class), -1) + // STORM-3059 + KafkaSpoutConfig spoutConfig = + createKafkaSpoutConfigBuilder(mock(TopicFilter.class), + mock(ManualPartitioner.class), -1) .setProcessingGuarantee(processingGuarantee) .setTupleTrackingEnforced(true) .setRecordTranslator(new NullRecordTranslator<>()) .build(); - KafkaSpout spout = SpoutWithMockedConsumerSetupHelper.setupSpout(spoutConfig, conf, contextMock, collectorMock, consumerMock, partition); + KafkaSpout spout = SpoutWithMockedConsumerSetupHelper + .setupSpout(spoutConfig, conf, contextMock, collectorMock, consumerMock, partition); - when(consumerMock.poll(any(Duration.class))).thenReturn(new ConsumerRecords<>(Collections.singletonMap(partition, + when(consumerMock.poll(any(Duration.class))).thenReturn(new ConsumerRecords<>(Collections + .singletonMap(partition, SpoutWithMockedConsumerSetupHelper.createRecords(partition, 0, 1)))); spout.nextTuple(); diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutNullTupleTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutNullTupleTest.java index e6dee2e1c11..8ea72b9e93e 100644 --- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutNullTupleTest.java +++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutNullTupleTest.java @@ -17,8 +17,9 @@ */ package org.apache.storm.kafka.spout; - -import static org.mockito.ArgumentMatchers.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; @@ -34,10 +35,10 @@ public KafkaSpoutNullTupleTest() { super(2_000); } - @Override KafkaSpoutConfig createSpoutConfig() { - return KafkaSpoutConfig.builder("127.0.0.1:" + kafkaUnitExtension.getKafkaUnit().getKafkaPort(), + return KafkaSpoutConfig.builder("127.0.0.1:" + kafkaUnitExtension.getKafkaUnit() + .getKafkaPort(), Pattern.compile(SingleTopicKafkaSpoutConfiguration.TOPIC)) .setGroupId("test") .setOffsetCommitPeriodMs(commitOffsetPeriodMs) @@ -50,18 +51,19 @@ public void testShouldCommitAllMessagesIfNotSetToEmitNullTuples() throws Excepti final int messageCount = 10; prepareSpout(messageCount); - //All null tuples should be commited, meaning they were considered by to be emitted and acked - for(int i = 0; i < messageCount; i++) { + // All null tuples should be commited, meaning they were considered by to be emitted and + // acked + for (int i = 0; i < messageCount; i++) { spout.nextTuple(); } - verify(collectorMock,never()).emit( + verify(collectorMock, never()).emit( anyString(), anyList(), any()); Time.advanceTime(commitOffsetPeriodMs + KafkaSpout.TIMER_DELAY_MS); - //Commit offsets + // Commit offsets spout.nextTuple(); verifyAllMessagesCommitted(messageCount); diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutReactivationTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutReactivationTest.java index b7ae2824b7f..c4e1ec762d4 100644 --- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutReactivationTest.java +++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutReactivationTest.java @@ -33,7 +33,6 @@ import com.codahale.metrics.Metric; import java.util.HashMap; import java.util.Map; - import org.apache.kafka.clients.admin.Admin; import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.consumer.ConsumerConfig; @@ -72,9 +71,11 @@ public class KafkaSpoutReactivationTest { private KafkaSpout spout; private final int maxPollRecords = 10; - public void prepareSpout(int messageCount, FirstPollOffsetStrategy firstPollOffsetStrategy) throws Exception { + public void prepareSpout(int messageCount, + FirstPollOffsetStrategy firstPollOffsetStrategy) throws Exception { KafkaSpoutConfig spoutConfig = - SingleTopicKafkaSpoutConfiguration.setCommonSpoutConfig(KafkaSpoutConfig.builder("127.0.0.1:" + kafkaUnitExtension.getKafkaUnit().getKafkaPort(), + SingleTopicKafkaSpoutConfiguration.setCommonSpoutConfig(KafkaSpoutConfig + .builder("127.0.0.1:" + kafkaUnitExtension.getKafkaUnit().getKafkaPort(), SingleTopicKafkaSpoutConfiguration.TOPIC)) .setFirstPollOffsetStrategy(firstPollOffsetStrategy) .setOffsetCommitPeriodMs(commitOffsetPeriodMs) @@ -89,12 +90,14 @@ public void prepareSpout(int messageCount, FirstPollOffsetStrategy firstPollOffs when(clientFactoryMock.createAdmin(any())) .thenReturn(adminSpy); this.spout = new KafkaSpout<>(spoutConfig, clientFactoryMock, new TopicAssigner()); - SingleTopicKafkaUnitSetupHelper.populateTopicData(kafkaUnitExtension.getKafkaUnit(), SingleTopicKafkaSpoutConfiguration.TOPIC, messageCount); + SingleTopicKafkaUnitSetupHelper.populateTopicData(kafkaUnitExtension.getKafkaUnit(), + SingleTopicKafkaSpoutConfiguration.TOPIC, messageCount); SingleTopicKafkaUnitSetupHelper.initializeSpout(spout, conf, topologyContext, collector); } private KafkaSpoutMessageId emitOne() { - ArgumentCaptor messageId = ArgumentCaptor.forClass(KafkaSpoutMessageId.class); + ArgumentCaptor messageId = ArgumentCaptor + .forClass(KafkaSpoutMessageId.class); spout.nextTuple(); verify(collector).emit(anyString(), anyList(), messageId.capture()); clearInvocations(collector); @@ -106,7 +109,8 @@ private void doReactivationTest(FirstPollOffsetStrategy firstPollOffsetStrategy) int messageCount = maxPollRecords * 2; prepareSpout(messageCount, firstPollOffsetStrategy); - //Emit and ack some tuples, ensure that some polled tuples remain cached in the spout by emitting less than maxPollRecords + // Emit and ack some tuples, ensure that some polled tuples remain cached in the spout + // by emitting less than maxPollRecords int beforeReactivationEmits = maxPollRecords - 3; for (int i = 0; i < beforeReactivationEmits - 1; i++) { KafkaSpoutMessageId msgId = emitOne(); @@ -115,26 +119,29 @@ private void doReactivationTest(FirstPollOffsetStrategy firstPollOffsetStrategy) KafkaSpoutMessageId ackAfterDeactivateMessageId = emitOne(); - //Cycle spout activation + // Cycle spout activation spout.deactivate(); - SingleTopicKafkaUnitSetupHelper.verifyAllMessagesCommitted(consumerSpy, commitCapture, beforeReactivationEmits - 1); + SingleTopicKafkaUnitSetupHelper.verifyAllMessagesCommitted(consumerSpy, commitCapture, + beforeReactivationEmits - 1); clearInvocations(consumerSpy); - //Tuples may be acked/failed after the spout deactivates, so we have to be able to handle this too + // Tuples may be acked/failed after the spout deactivates, so we have to be able to + // handle this too spout.ack(ackAfterDeactivateMessageId); spout.activate(); - //Emit and ack the rest + // Emit and ack the rest for (int i = beforeReactivationEmits; i < messageCount; i++) { KafkaSpoutMessageId msgId = emitOne(); spout.ack(msgId); } - //Commit + // Commit Time.advanceTime(TIMER_DELAY_MS + commitOffsetPeriodMs); spout.nextTuple(); - //Verify that no more tuples are emitted and all tuples are committed - SingleTopicKafkaUnitSetupHelper.verifyAllMessagesCommitted(consumerSpy, commitCapture, messageCount); + // Verify that no more tuples are emitted and all tuples are committed + SingleTopicKafkaUnitSetupHelper.verifyAllMessagesCommitted(consumerSpy, commitCapture, + messageCount); clearInvocations(collector); spout.nextTuple(); @@ -145,19 +152,21 @@ private void doReactivationTest(FirstPollOffsetStrategy firstPollOffsetStrategy) @Test public void testSpoutShouldResumeWhereItLeftOffWithUncommittedEarliestStrategy() throws Exception { - //With uncommitted earliest the spout should pick up where it left off when reactivating. + // With uncommitted earliest the spout should pick up where it left off when reactivating. doReactivationTest(FirstPollOffsetStrategy.UNCOMMITTED_EARLIEST); } @Test public void testSpoutShouldResumeWhereItLeftOffWithEarliestStrategy() throws Exception { - //With earliest, the spout should also resume where it left off, rather than restart at the earliest offset. + // With earliest, the spout should also resume where it left off, rather than restart at the + // earliest offset. doReactivationTest(FirstPollOffsetStrategy.EARLIEST); } @Test public void testSpoutMustHandleGettingMetricsWhileDeactivated() throws Exception { - //Storm will try to get metrics from the spout even while deactivated, the spout must be able to handle this + // Storm will try to get metrics from the spout even while deactivated, the spout must be + // able to handle this prepareSpout(10, FirstPollOffsetStrategy.UNCOMMITTED_EARLIEST); for (int i = 0; i < 5; i++) { @@ -166,12 +175,18 @@ public void testSpoutMustHandleGettingMetricsWhileDeactivated() throws Exception } spout.deactivate(); - Map partitionsOffsetMetric = spout.getKafkaOffsetMetricManager().getTopicPartitionMetricsMap().get(new TopicPartition(SingleTopicKafkaSpoutConfiguration.TOPIC ,0)).getMetrics(); - Long partitionLag = (Long) ((Gauge) partitionsOffsetMetric.get(SingleTopicKafkaSpoutConfiguration.TOPIC + "/partition_0/spoutLag")).getValue(); + Map partitionsOffsetMetric = spout.getKafkaOffsetMetricManager() + .getTopicPartitionMetricsMap() + .get(new TopicPartition(SingleTopicKafkaSpoutConfiguration.TOPIC, 0)).getMetrics(); + Long partitionLag = (Long) ((Gauge) partitionsOffsetMetric + .get(SingleTopicKafkaSpoutConfiguration.TOPIC + "/partition_0/spoutLag")) + .getValue(); assertThat(partitionLag, is(5L)); - Map topicOffsetMetric = spout.getKafkaOffsetMetricManager().getTopicMetricsMap().get(SingleTopicKafkaSpoutConfiguration.TOPIC).getMetrics(); - Long totalSpoutLag = (Long) ((Gauge) topicOffsetMetric.get(SingleTopicKafkaSpoutConfiguration.TOPIC + "/totalSpoutLag")).getValue(); + Map topicOffsetMetric = spout.getKafkaOffsetMetricManager() + .getTopicMetricsMap().get(SingleTopicKafkaSpoutConfiguration.TOPIC).getMetrics(); + Long totalSpoutLag = (Long) ((Gauge) topicOffsetMetric + .get(SingleTopicKafkaSpoutConfiguration.TOPIC + "/totalSpoutLag")).getValue(); assertThat(totalSpoutLag, is(5L)); } } diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRebalanceTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRebalanceTest.java index c8847750cf4..bcab97a0522 100644 --- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRebalanceTest.java +++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRebalanceTest.java @@ -24,7 +24,14 @@ import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.*; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import java.time.Duration; import java.util.ArrayList; @@ -34,9 +41,12 @@ import java.util.List; import java.util.Map; import java.util.Set; - import org.apache.kafka.clients.admin.Admin; -import org.apache.kafka.clients.consumer.*; +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.consumer.ConsumerRebalanceListener; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.common.TopicPartition; import org.apache.storm.kafka.spout.config.builder.SingleTopicKafkaSpoutConfiguration; import org.apache.storm.kafka.spout.internal.ClientFactory; @@ -45,8 +55,8 @@ import org.apache.storm.kafka.spout.subscription.TopicFilter; import org.apache.storm.spout.SpoutOutputCollector; import org.apache.storm.task.TopologyContext; -import org.apache.storm.utils.Time; import org.apache.storm.utils.Time.SimulatedTime; +import org.apache.storm.utils.Time; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -96,14 +106,15 @@ public Admin createAdmin(Map adminProps) { doAnswer(set).when(partitionerMock).getPartitionsForThisTask(any(), any()); } - //Returns messageIds in order of emission + // Returns messageIds in order of emission private List emitOneMessagePerPartitionThenRevokeOnePartition(KafkaSpout spout, TopicPartition partitionThatWillBeRevoked, TopicPartition assignedPartition, TopicAssigner topicAssigner) { - //Setup spout with mock consumer so we can get at the rebalance listener + // Setup spout with mock consumer so we can get at the rebalance listener spout.open(conf, contextMock, collectorMock); spout.activate(); - //Assign partitions to the spout - ArgumentCaptor rebalanceListenerCapture = ArgumentCaptor.forClass(ConsumerRebalanceListener.class); + // Assign partitions to the spout + ArgumentCaptor rebalanceListenerCapture = ArgumentCaptor + .forClass(ConsumerRebalanceListener.class); verify(topicAssigner).assignPartitions(any(), any(), rebalanceListenerCapture.capture()); ConsumerRebalanceListener consumerRebalanceListener = rebalanceListenerCapture.getValue(); Set assignedPartitions = new HashSet<>(); @@ -112,22 +123,27 @@ private List emitOneMessagePerPartitionThenRevokeOnePartiti consumerRebalanceListener.onPartitionsAssigned(assignedPartitions); when(consumerMock.assignment()).thenReturn(assignedPartitions); - //Make the consumer return a single message for each partition + // Make the consumer return a single message for each partition when(consumerMock.poll(any(Duration.class))) - .thenReturn(new ConsumerRecords<>(Collections.singletonMap(partitionThatWillBeRevoked, SpoutWithMockedConsumerSetupHelper.createRecords(partitionThatWillBeRevoked, 0, 1)), Map.of())) - .thenReturn(new ConsumerRecords<>(Collections.singletonMap(assignedPartition, SpoutWithMockedConsumerSetupHelper.createRecords(assignedPartition, 0, 1)), Map.of())) + .thenReturn(new ConsumerRecords<>(Collections.singletonMap(partitionThatWillBeRevoked, + SpoutWithMockedConsumerSetupHelper.createRecords(partitionThatWillBeRevoked, 0, 1)), Map + .of())) + .thenReturn(new ConsumerRecords<>(Collections.singletonMap(assignedPartition, + SpoutWithMockedConsumerSetupHelper.createRecords(assignedPartition, 0, 1)), Map.of())) .thenReturn(new ConsumerRecords<>(Collections.emptyMap(), Map.of())); - //Emit the messages + // Emit the messages spout.nextTuple(); - ArgumentCaptor messageIdForRevokedPartition = ArgumentCaptor.forClass(KafkaSpoutMessageId.class); + ArgumentCaptor messageIdForRevokedPartition = ArgumentCaptor + .forClass(KafkaSpoutMessageId.class); verify(collectorMock).emit(anyString(), anyList(), messageIdForRevokedPartition.capture()); reset(collectorMock); spout.nextTuple(); - ArgumentCaptor messageIdForAssignedPartition = ArgumentCaptor.forClass(KafkaSpoutMessageId.class); + ArgumentCaptor messageIdForAssignedPartition = ArgumentCaptor + .forClass(KafkaSpoutMessageId.class); verify(collectorMock).emit(anyString(), anyList(), messageIdForAssignedPartition.capture()); - //Now rebalance + // Now rebalance consumerRebalanceListener.onPartitionsRevoked(assignedPartitions); consumerRebalanceListener.onPartitionsAssigned(Collections.singleton(assignedPartition)); final Answer assignedP = invocation -> Collections.singleton(assignedPartition); @@ -141,29 +157,33 @@ private List emitOneMessagePerPartitionThenRevokeOnePartiti @Test public void spoutMustIgnoreAcksForTuplesItIsNotAssignedAfterRebalance() { - // Acking tuples for partitions that are no longer assigned is useless since the spout will not be allowed to commit them + // Acking tuples for partitions that are no longer assigned is useless since the spout will + // not be allowed to commit them try (SimulatedTime ignored = new SimulatedTime()) { TopicAssigner assignerMock = mock(TopicAssigner.class); - KafkaSpout spout = new KafkaSpout<>(createKafkaSpoutConfigBuilder(topicFilterMock, partitionerMock, -1) + KafkaSpout spout = + new KafkaSpout<>(createKafkaSpoutConfigBuilder(topicFilterMock, partitionerMock, + -1) .setOffsetCommitPeriodMs(offsetCommitPeriodMs) .build(), clientFactory, assignerMock); String topic = SingleTopicKafkaSpoutConfiguration.TOPIC; TopicPartition partitionThatWillBeRevoked = new TopicPartition(topic, 1); TopicPartition assignedPartition = new TopicPartition(topic, 2); - //Emit a message on each partition and revoke the first partition - List emittedMessageIds = emitOneMessagePerPartitionThenRevokeOnePartition( + // Emit a message on each partition and revoke the first partition + List emittedMessageIds = + emitOneMessagePerPartitionThenRevokeOnePartition( spout, partitionThatWillBeRevoked, assignedPartition, assignerMock); - //Ack both emitted tuples + // Ack both emitted tuples spout.ack(emittedMessageIds.get(0)); spout.ack(emittedMessageIds.get(1)); - //Ensure the commit timer has expired + // Ensure the commit timer has expired Time.advanceTime(offsetCommitPeriodMs + KafkaSpout.TIMER_DELAY_MS); - //Make the spout commit any acked tuples + // Make the spout commit any acked tuples spout.nextTuple(); - //Verify that it only committed the message on the assigned partition + // Verify that it only committed the message on the assigned partition verify(consumerMock, times(1)).commitSync(commitCapture.capture()); Map commitCaptureMap = commitCapture.getValue(); @@ -174,10 +194,12 @@ public void spoutMustIgnoreAcksForTuplesItIsNotAssignedAfterRebalance() { @Test public void spoutMustIgnoreFailsForTuplesItIsNotAssignedAfterRebalance() { - //Failing tuples for partitions that are no longer assigned is useless since the spout will not be allowed to commit them if they later pass + // Failing tuples for partitions that are no longer assigned is useless since the spout will + // not be allowed to commit them if they later pass TopicAssigner assignerMock = mock(TopicAssigner.class); KafkaSpoutRetryService retryServiceMock = mock(KafkaSpoutRetryService.class); - KafkaSpout spout = new KafkaSpout<>(createKafkaSpoutConfigBuilder(topicFilterMock, partitionerMock, -1) + KafkaSpout spout = + new KafkaSpout<>(createKafkaSpoutConfigBuilder(topicFilterMock, partitionerMock, -1) .setOffsetCommitPeriodMs(10) .setRetry(retryServiceMock) .build(), clientFactory, assignerMock); @@ -189,18 +211,19 @@ public void spoutMustIgnoreFailsForTuplesItIsNotAssignedAfterRebalance() { .thenReturn(new KafkaSpoutMessageId(partitionThatWillBeRevoked, 0)) .thenReturn(new KafkaSpoutMessageId(assignedPartition, 0)); - //Emit a message on each partition and revoke the first partition - List emittedMessageIds = emitOneMessagePerPartitionThenRevokeOnePartition( + // Emit a message on each partition and revoke the first partition + List emittedMessageIds = + emitOneMessagePerPartitionThenRevokeOnePartition( spout, partitionThatWillBeRevoked, assignedPartition, assignerMock); - //Check that only two message ids were generated + // Check that only two message ids were generated verify(retryServiceMock, times(2)).getMessageId(any(TopicPartition.class), anyLong()); - //Fail both emitted tuples + // Fail both emitted tuples spout.fail(emittedMessageIds.get(0)); spout.fail(emittedMessageIds.get(1)); - //Check that only the tuple on the currently assigned partition is retried + // Check that only the tuple on the currently assigned partition is retried verify(retryServiceMock, never()).schedule(emittedMessageIds.get(0)); verify(retryServiceMock).schedule(emittedMessageIds.get(1)); } @@ -208,50 +231,55 @@ public void spoutMustIgnoreFailsForTuplesItIsNotAssignedAfterRebalance() { @Test public void testReassignPartitionSeeksForOnlyNewPartitions() { /* - * When partitions are reassigned, the spout should seek with the first poll offset strategy for new partitions. - * Previously assigned partitions should be left alone, since the spout keeps the emitted and acked state for those. + * When partitions are reassigned, the spout should seek with the first poll offset strategy + * for new partitions. + * Previously assigned partitions should be left alone, since the spout keeps the emitted + * and acked state for those. */ TopicAssigner assignerMock = mock(TopicAssigner.class); - KafkaSpout spout = new KafkaSpout<>(createKafkaSpoutConfigBuilder(topicFilterMock, partitionerMock, -1) + KafkaSpout spout = + new KafkaSpout<>(createKafkaSpoutConfigBuilder(topicFilterMock, partitionerMock, -1) .setFirstPollOffsetStrategy(FirstPollOffsetStrategy.UNCOMMITTED_EARLIEST) .build(), clientFactory, assignerMock); String topic = SingleTopicKafkaSpoutConfiguration.TOPIC; TopicPartition assignedPartition = new TopicPartition(topic, 1); TopicPartition newPartition = new TopicPartition(topic, 2); - //Setup spout with mock consumer so we can get at the rebalance listener + // Setup spout with mock consumer so we can get at the rebalance listener spout.open(conf, contextMock, collectorMock); spout.activate(); - ArgumentCaptor rebalanceListenerCapture = ArgumentCaptor.forClass(ConsumerRebalanceListener.class); + ArgumentCaptor rebalanceListenerCapture = ArgumentCaptor + .forClass(ConsumerRebalanceListener.class); verify(assignerMock).assignPartitions(any(), any(), rebalanceListenerCapture.capture()); - //Assign partitions to the spout + // Assign partitions to the spout ConsumerRebalanceListener consumerRebalanceListener = rebalanceListenerCapture.getValue(); Set assignedPartitions = new HashSet<>(); assignedPartitions.add(assignedPartition); consumerRebalanceListener.onPartitionsAssigned(assignedPartitions); reset(consumerMock); - //Set up committed so it looks like some messages have been committed on each partition + // Set up committed so it looks like some messages have been committed on each partition long committedOffset = 500; final Map mapAnswer = new HashMap<>(); - mapAnswer.put(newPartition,new OffsetAndMetadata(committedOffset)); + mapAnswer.put(newPartition, new OffsetAndMetadata(committedOffset)); final Answer objectAnswer = invocation -> mapAnswer; - lenient().doAnswer(objectAnswer).when(consumerMock).committed(Collections.singleton(newPartition)); + lenient().doAnswer(objectAnswer).when(consumerMock).committed(Collections + .singleton(newPartition)); doAnswer(objectAnswer).when(consumerMock).committed(Collections.singleton(newPartition)); - //Now rebalance and add a new partition + // Now rebalance and add a new partition consumerRebalanceListener.onPartitionsRevoked(assignedPartitions); Set newAssignedPartitions = new HashSet<>(); newAssignedPartitions.add(assignedPartition); newAssignedPartitions.add(newPartition); consumerRebalanceListener.onPartitionsAssigned(newAssignedPartitions); - //This partition was previously assigned, so the consumer position shouldn't change + // This partition was previously assigned, so the consumer position shouldn't change verify(consumerMock, never()).seek(eq(assignedPartition), anyLong()); - //This partition is new, and should start at the committed offset + // This partition is new, and should start at the committed offset verify(consumerMock).seek(newPartition, committedOffset); } } diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRetryExponentialBackoffTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRetryExponentialBackoffTest.java index 269794a62f9..cc67159c16f 100644 --- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRetryExponentialBackoffTest.java +++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRetryExponentialBackoffTest.java @@ -25,8 +25,8 @@ import java.util.Map; import org.apache.kafka.common.TopicPartition; import org.apache.storm.kafka.spout.KafkaSpoutRetryExponentialBackoff.TimeInterval; -import org.apache.storm.utils.Time; import org.apache.storm.utils.Time.SimulatedTime; +import org.apache.storm.utils.Time; import org.junit.jupiter.api.Test; public class KafkaSpoutRetryExponentialBackoffTest { @@ -35,11 +35,13 @@ public class KafkaSpoutRetryExponentialBackoffTest { private final TopicPartition testTopic2 = new TopicPartition("other-topic", 0); private KafkaSpoutRetryExponentialBackoff createNoWaitRetryService() { - return new KafkaSpoutRetryExponentialBackoff(TimeInterval.seconds(0), TimeInterval.seconds(0), 1, TimeInterval.seconds(0)); + return new KafkaSpoutRetryExponentialBackoff(TimeInterval.seconds(0), TimeInterval + .seconds(0), 1, TimeInterval.seconds(0)); } private KafkaSpoutRetryExponentialBackoff createOneSecondWaitRetryService() { - return new KafkaSpoutRetryExponentialBackoff(TimeInterval.seconds(1), TimeInterval.seconds(0), 1, TimeInterval.seconds(1)); + return new KafkaSpoutRetryExponentialBackoff(TimeInterval.seconds(1), TimeInterval + .seconds(0), 1, TimeInterval.seconds(1)); } @Test @@ -53,11 +55,13 @@ public void testCanScheduleRetry() { assertThat("The service must schedule the message for retry", scheduled, is(true)); KafkaSpoutMessageId retrievedMessageId = retryService.getMessageId(testTopic, offset); - assertThat("The service should return the original message id when asked for the same tp/offset twice", retrievedMessageId, sameInstance(msgId)); + assertThat("The service should return the original message id when asked for the same " + + "tp/offset twice", retrievedMessageId, sameInstance(msgId)); assertThat(retryService.isScheduled(msgId), is(true)); assertThat(retryService.isReady(msgId), is(true)); assertThat(retryService.readyMessageCount(), is(1)); - assertThat(retryService.earliestRetriableOffsets(), is(Collections.singletonMap(testTopic, msgId.offset()))); + assertThat(retryService.earliestRetriableOffsets(), is(Collections.singletonMap(testTopic, + msgId.offset()))); } @Test @@ -73,16 +77,20 @@ public void testCanRescheduleRetry() { Time.advanceTime(500); boolean scheduled = retryService.schedule(msgId); - assertThat("The service must be able to reschedule an already scheduled id", scheduled, is(true)); + assertThat("The service must be able to reschedule an already scheduled id", scheduled, + is(true)); Time.advanceTime(500); - assertThat("The message should not be ready for retry yet since it was rescheduled", retryService.isReady(msgId), is(false)); + assertThat("The message should not be ready for retry yet since it was rescheduled", + retryService.isReady(msgId), is(false)); assertThat(retryService.isScheduled(msgId), is(true)); assertThat(retryService.earliestRetriableOffsets(), is(Collections.emptyMap())); assertThat(retryService.readyMessageCount(), is(0)); Time.advanceTime(500); - assertThat("The message should be ready for retry once the full delay has passed", retryService.isReady(msgId), is(true)); + assertThat("The message should be ready for retry once the full delay has passed", + retryService.isReady(msgId), is(true)); assertThat(retryService.isScheduled(msgId), is(true)); - assertThat(retryService.earliestRetriableOffsets(), is(Collections.singletonMap(testTopic, msgId.offset()))); + assertThat(retryService.earliestRetriableOffsets(), is(Collections + .singletonMap(testTopic, msgId.offset()))); assertThat(retryService.readyMessageCount(), is(1)); } } @@ -101,9 +109,11 @@ public void testCannotContainMultipleSchedulesForId() { boolean scheduled = retryService.schedule(msgId); retryService.remove(msgId); - assertThat("The message should no longer be scheduled", retryService.isScheduled(msgId), is(false)); + assertThat("The message should no longer be scheduled", retryService.isScheduled(msgId), + is(false)); Time.advanceTime(500); - assertThat("The message should not be ready for retry because it isn't scheduled", retryService.isReady(msgId), is(false)); + assertThat("The message should not be ready for retry because it isn't scheduled", + retryService.isReady(msgId), is(false)); } } @@ -127,7 +137,8 @@ public void testCanRemoveRetry() { @Test public void testCanHandleMultipleTopics() { try (SimulatedTime ignored = new SimulatedTime()) { - //Tests that isScheduled, isReady and earliestRetriableOffsets are mutually consistent when there are messages from multiple partitions scheduled + // Tests that isScheduled, isReady and earliestRetriableOffsets are mutually consistent + // when there are messages from multiple partitions scheduled KafkaSpoutRetryExponentialBackoff retryService = createOneSecondWaitRetryService(); long offset = 0; @@ -140,7 +151,7 @@ public void testCanHandleMultipleTopics() { Time.advanceTime(500); boolean scheduledTwo = retryService.schedule(msgIdTp2); - //The retry schedules for two messages should be unrelated + // The retry schedules for two messages should be unrelated assertThat(scheduledOne, is(true)); assertThat(retryService.isScheduled(msgIdTp1), is(true)); assertThat(scheduledTwo, is(true)); @@ -151,7 +162,8 @@ public void testCanHandleMultipleTopics() { Time.advanceTime(500); assertThat(retryService.isReady(msgIdTp1), is(true)); assertThat(retryService.isReady(msgIdTp2), is(false)); - assertThat(retryService.earliestRetriableOffsets(), is(Collections.singletonMap(testTopic, offset))); + assertThat(retryService.earliestRetriableOffsets(), is(Collections + .singletonMap(testTopic, offset))); Time.advanceTime(500); assertThat(retryService.isReady(msgIdTp2), is(true)); @@ -160,20 +172,22 @@ public void testCanHandleMultipleTopics() { earliestOffsets.put(testTopic2, offset); assertThat(retryService.earliestRetriableOffsets(), is(earliestOffsets)); - //The service must be able to remove retry schedules for unnecessary partitions + // The service must be able to remove retry schedules for unnecessary partitions retryService.retainAll(Collections.singleton(testTopic2)); assertThat(retryService.isScheduled(msgIdTp1), is(false)); assertThat(retryService.isScheduled(msgIdTp2), is(true)); assertThat(retryService.isReady(msgIdTp1), is(false)); assertThat(retryService.isReady(msgIdTp2), is(true)); - assertThat(retryService.earliestRetriableOffsets(), is(Collections.singletonMap(testTopic2, offset))); + assertThat(retryService.earliestRetriableOffsets(), is(Collections + .singletonMap(testTopic2, offset))); } } @Test public void testCanHandleMultipleMessagesOnPartition() { try (SimulatedTime ignored = new SimulatedTime()) { - //Tests that isScheduled, isReady and earliestRetriableOffsets are mutually consistent when there are multiple messages scheduled on a partition + // Tests that isScheduled, isReady and earliestRetriableOffsets are mutually consistent + // when there are multiple messages scheduled on a partition KafkaSpoutRetryExponentialBackoff retryService = createOneSecondWaitRetryService(); long offset = 0; @@ -192,15 +206,18 @@ public void testCanHandleMultipleMessagesOnPartition() { Time.advanceTime(500); assertThat(retryService.isReady(msgIdEarliest), is(true)); assertThat(retryService.isReady(msgIdLatest), is(false)); - assertThat(retryService.earliestRetriableOffsets(), is(Collections.singletonMap(testTopic, msgIdEarliest.offset()))); + assertThat(retryService.earliestRetriableOffsets(), is(Collections + .singletonMap(testTopic, msgIdEarliest.offset()))); Time.advanceTime(500); assertThat(retryService.isReady(msgIdEarliest), is(true)); assertThat(retryService.isReady(msgIdLatest), is(true)); - assertThat(retryService.earliestRetriableOffsets(), is(Collections.singletonMap(testTopic, msgIdEarliest.offset()))); + assertThat(retryService.earliestRetriableOffsets(), is(Collections + .singletonMap(testTopic, msgIdEarliest.offset()))); retryService.remove(msgIdEarliest); - assertThat(retryService.earliestRetriableOffsets(), is(Collections.singletonMap(testTopic, msgIdLatest.offset()))); + assertThat(retryService.earliestRetriableOffsets(), is(Collections + .singletonMap(testTopic, msgIdLatest.offset()))); } } @@ -208,7 +225,9 @@ public void testCanHandleMultipleMessagesOnPartition() { public void testMaxRetries() { try (SimulatedTime ignored = new SimulatedTime()) { int maxRetries = 3; - KafkaSpoutRetryExponentialBackoff retryService = new KafkaSpoutRetryExponentialBackoff(TimeInterval.seconds(0), TimeInterval.seconds(0), maxRetries, TimeInterval.seconds(0)); + KafkaSpoutRetryExponentialBackoff retryService = + new KafkaSpoutRetryExponentialBackoff(TimeInterval.seconds(0), TimeInterval + .seconds(0), maxRetries, TimeInterval.seconds(0)); long offset = 0; KafkaSpoutMessageId msgId = retryService.getMessageId(testTopic, offset); @@ -216,7 +235,7 @@ public void testMaxRetries() { msgId.incrementNumFails(); } - //Should be allowed to retry 3 times, in addition to original try + // Should be allowed to retry 3 times, in addition to original try boolean scheduled = retryService.schedule(msgId); assertThat(scheduled, is(true)); @@ -226,7 +245,8 @@ public void testMaxRetries() { msgId.incrementNumFails(); boolean rescheduled = retryService.schedule(msgId); - assertThat("The message should not be allowed to retry once the limit is reached", rescheduled, is(false)); + assertThat("The message should not be allowed to retry once the limit is reached", + rescheduled, is(false)); assertThat(retryService.isScheduled(msgId), is(false)); } } @@ -235,7 +255,9 @@ public void testMaxRetries() { public void testMaxDelay() { try (SimulatedTime ignored = new SimulatedTime()) { int maxDelaySecs = 2; - KafkaSpoutRetryExponentialBackoff retryService = new KafkaSpoutRetryExponentialBackoff(TimeInterval.seconds(500), TimeInterval.seconds(0), 1, TimeInterval.seconds(maxDelaySecs)); + KafkaSpoutRetryExponentialBackoff retryService = + new KafkaSpoutRetryExponentialBackoff(TimeInterval.seconds(500), TimeInterval + .seconds(0), 1, TimeInterval.seconds(maxDelaySecs)); long offset = 0; KafkaSpoutMessageId msgId = retryService.getMessageId(testTopic, offset); @@ -244,30 +266,37 @@ public void testMaxDelay() { retryService.schedule(msgId); Time.advanceTimeSecs(maxDelaySecs); - assertThat("The message should be ready for retry after the max delay", retryService.isReady(msgId), is(true)); + assertThat("The message should be ready for retry after the max delay", retryService + .isReady(msgId), is(true)); } } @Test public void testExponentialBackoff() { try (SimulatedTime ignored = new SimulatedTime()) { - KafkaSpoutRetryExponentialBackoff retryService = new KafkaSpoutRetryExponentialBackoff(TimeInterval.seconds(0), TimeInterval.seconds(4), Integer.MAX_VALUE, TimeInterval.seconds(Integer.MAX_VALUE)); + KafkaSpoutRetryExponentialBackoff retryService = + new KafkaSpoutRetryExponentialBackoff(TimeInterval.seconds(0), TimeInterval + .seconds(4), Integer.MAX_VALUE, TimeInterval.seconds(Integer.MAX_VALUE)); long offset = 0; KafkaSpoutMessageId msgId = retryService.getMessageId(testTopic, offset); msgId.incrementNumFails(); - msgId.incrementNumFails(); //First failure is the initial delay, so not interesting + msgId.incrementNumFails(); // First failure is the initial delay, so not interesting - //Expecting 4*2^(failCount-1) + // Expecting 4*2^(failCount-1) Integer[] expectedBackoffsSecs = new Integer[]{8, 16, 32}; for (Integer expectedBackoffSecs : expectedBackoffsSecs) { retryService.schedule(msgId); Time.advanceTimeSecs(expectedBackoffSecs - 1); - assertThat("The message should not be ready for retry until backoff " + expectedBackoffSecs + " has expired", retryService.isReady(msgId), is(false)); + assertThat("The message should not be ready for retry until backoff " + + expectedBackoffSecs + " has expired", retryService + .isReady(msgId), is(false)); Time.advanceTimeSecs(1); - assertThat("The message should be ready for retry once backoff " + expectedBackoffSecs + " has expired", retryService.isReady(msgId), is(true)); + assertThat("The message should be ready for retry once backoff " + + expectedBackoffSecs + " has expired", retryService + .isReady(msgId), is(true)); msgId.incrementNumFails(); retryService.remove(msgId); diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRetryLimitTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRetryLimitTest.java index 381796b7340..3530e97cf4e 100644 --- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRetryLimitTest.java +++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRetryLimitTest.java @@ -18,8 +18,8 @@ import static org.apache.storm.kafka.spout.config.builder.SingleTopicKafkaSpoutConfiguration.createKafkaSpoutConfigBuilder; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.inOrder; import static org.mockito.Mockito.mock; @@ -31,15 +31,18 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import org.apache.kafka.clients.consumer.*; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.common.TopicPartition; import org.apache.storm.kafka.spout.config.builder.SingleTopicKafkaSpoutConfiguration; import org.apache.storm.kafka.spout.subscription.ManualPartitioner; import org.apache.storm.kafka.spout.subscription.TopicFilter; import org.apache.storm.spout.SpoutOutputCollector; import org.apache.storm.task.TopologyContext; -import org.apache.storm.utils.Time; import org.apache.storm.utils.Time.SimulatedTime; +import org.apache.storm.utils.Time; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -55,13 +58,15 @@ public class KafkaSpoutRetryLimitTest { private final TopologyContext contextMock = mock(TopologyContext.class); private final SpoutOutputCollector collectorMock = mock(SpoutOutputCollector.class); private final Map conf = new HashMap<>(); - private final TopicPartition partition = new TopicPartition(SingleTopicKafkaSpoutConfiguration.TOPIC, 1); + private final TopicPartition partition = + new TopicPartition(SingleTopicKafkaSpoutConfiguration.TOPIC, 1); @Mock private KafkaConsumer consumerMock; private KafkaSpoutConfig spoutConfig; public static final KafkaSpoutRetryService ZERO_RETRIES_RETRY_SERVICE = - new KafkaSpoutRetryExponentialBackoff(KafkaSpoutRetryExponentialBackoff.TimeInterval.seconds(0), KafkaSpoutRetryExponentialBackoff.TimeInterval.milliSeconds(0), + new KafkaSpoutRetryExponentialBackoff(KafkaSpoutRetryExponentialBackoff.TimeInterval + .seconds(0), KafkaSpoutRetryExponentialBackoff.TimeInterval.milliSeconds(0), 0, KafkaSpoutRetryExponentialBackoff.TimeInterval.milliSeconds(0)); @Captor @@ -69,7 +74,8 @@ public class KafkaSpoutRetryLimitTest { @BeforeEach public void setUp() { - spoutConfig = createKafkaSpoutConfigBuilder(mock(TopicFilter.class), mock(ManualPartitioner.class), -1) + spoutConfig = createKafkaSpoutConfigBuilder(mock(TopicFilter.class), + mock(ManualPartitioner.class), -1) .setOffsetCommitPeriodMs(offsetCommitPeriodMs) .setRetry(ZERO_RETRIES_RETRY_SERVICE) .build(); @@ -77,13 +83,16 @@ public void setUp() { @Test public void testFailingTupleCompletesAckAfterRetryLimitIsMet() { - //Spout should ack failed messages after they hit the retry limit + // Spout should ack failed messages after they hit the retry limit try (SimulatedTime ignored = new SimulatedTime()) { - KafkaSpout spout = SpoutWithMockedConsumerSetupHelper.setupSpout(spoutConfig, conf, contextMock, collectorMock, consumerMock, partition); + KafkaSpout spout = SpoutWithMockedConsumerSetupHelper + .setupSpout(spoutConfig, conf, contextMock, collectorMock, consumerMock, + partition); Map>> records = new HashMap<>(); int lastOffset = 3; int numRecords = lastOffset + 1; - records.put(partition, SpoutWithMockedConsumerSetupHelper.createRecords(partition, 0, numRecords)); + records.put(partition, SpoutWithMockedConsumerSetupHelper.createRecords(partition, 0, + numRecords)); when(consumerMock.poll(any(Duration.class))) .thenReturn(new ConsumerRecords<>(records)); @@ -92,8 +101,10 @@ public void testFailingTupleCompletesAckAfterRetryLimitIsMet() { spout.nextTuple(); } - ArgumentCaptor messageIds = ArgumentCaptor.forClass(KafkaSpoutMessageId.class); - verify(collectorMock, times(numRecords)).emit(anyString(), anyList(), messageIds.capture()); + ArgumentCaptor messageIds = ArgumentCaptor + .forClass(KafkaSpoutMessageId.class); + verify(collectorMock, times(numRecords)).emit(anyString(), anyList(), messageIds + .capture()); for (KafkaSpoutMessageId messageId : messageIds.getAllValues()) { spout.fail(messageId); @@ -107,7 +118,8 @@ public void testFailingTupleCompletesAckAfterRetryLimitIsMet() { inOrder.verify(consumerMock).commitSync(commitCapture.capture()); inOrder.verify(consumerMock).poll(any(Duration.class)); - //verify that offset 4 was committed for the given TopicPartition, since processing should resume at 4. + // verify that offset 4 was committed for the given TopicPartition, since processing + // should resume at 4. assertTrue(commitCapture.getValue().containsKey(partition)); assertEquals(lastOffset + 1, commitCapture.getValue().get(partition).offset()); } diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutSingleTopicTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutSingleTopicTest.java index e648faf93bd..34689c49cee 100644 --- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutSingleTopicTest.java +++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutSingleTopicTest.java @@ -17,14 +17,17 @@ */ package org.apache.storm.kafka.spout; - import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.*; +import static org.mockito.Mockito.clearInvocations; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import java.util.HashSet; import java.util.List; @@ -52,7 +55,8 @@ public KafkaSpoutSingleTopicTest() { @Override KafkaSpoutConfig createSpoutConfig() { return SingleTopicKafkaSpoutConfiguration.setCommonSpoutConfig( - KafkaSpoutConfig.builder("127.0.0.1:" + kafkaUnitExtension.getKafkaUnit().getKafkaPort(), + KafkaSpoutConfig.builder("127.0.0.1:" + kafkaUnitExtension.getKafkaUnit() + .getKafkaPort(), Pattern.compile(SingleTopicKafkaSpoutConfiguration.TOPIC))) .setOffsetCommitPeriodMs(commitOffsetPeriodMs) .setRetry(new KafkaSpoutRetryExponentialBackoff(KafkaSpoutRetryExponentialBackoff.TimeInterval.seconds(0), KafkaSpoutRetryExponentialBackoff.TimeInterval.seconds(0), @@ -66,12 +70,14 @@ public void testSeekToCommittedOffsetIfConsumerPositionIsBehindWhenCommitting() final int messageCount = maxPollRecords * 2; prepareSpout(messageCount); - //Emit all messages and fail the first one while acking the rest + // Emit all messages and fail the first one while acking the rest for (int i = 0; i < messageCount; i++) { spout.nextTuple(); } - ArgumentCaptor messageIdCaptor = ArgumentCaptor.forClass(KafkaSpoutMessageId.class); - verify(collectorMock, times(messageCount)).emit(anyString(), anyList(), messageIdCaptor.capture()); + ArgumentCaptor messageIdCaptor = ArgumentCaptor + .forClass(KafkaSpoutMessageId.class); + verify(collectorMock, times(messageCount)).emit(anyString(), anyList(), messageIdCaptor + .capture()); List messageIds = messageIdCaptor.getAllValues(); for (int i = 1; i < messageIds.size(); i++) { spout.ack(messageIds.get(i)); @@ -79,16 +85,19 @@ public void testSeekToCommittedOffsetIfConsumerPositionIsBehindWhenCommitting() KafkaSpoutMessageId failedTuple = messageIds.get(0); spout.fail(failedTuple); - //Advance the time and replay the failed tuple. + // Advance the time and replay the failed tuple. reset(collectorMock); spout.nextTuple(); - ArgumentCaptor failedIdReplayCaptor = ArgumentCaptor.forClass(KafkaSpoutMessageId.class); + ArgumentCaptor failedIdReplayCaptor = ArgumentCaptor + .forClass(KafkaSpoutMessageId.class); verify(collectorMock).emit(anyString(), anyList(), failedIdReplayCaptor.capture()); - assertThat("Expected replay of failed tuple", failedIdReplayCaptor.getValue(), is(failedTuple)); + assertThat("Expected replay of failed tuple", failedIdReplayCaptor.getValue(), + is(failedTuple)); /* Ack the tuple, and commit. - * Since the tuple is more than max poll records behind the most recent emitted tuple, the consumer won't catch up in this poll. + * Since the tuple is more than max poll records behind the most recent emitted tuple, the + * consumer won't catch up in this poll. */ clearInvocations(collectorMock); Time.advanceTime(KafkaSpout.TIMER_DELAY_MS + commitOffsetPeriodMs); @@ -98,15 +107,19 @@ public void testSeekToCommittedOffsetIfConsumerPositionIsBehindWhenCommitting() Map capturedCommit = commitCapture.getValue(); TopicPartition expectedTp = new TopicPartition(SingleTopicKafkaSpoutConfiguration.TOPIC, 0); - assertThat("Should have committed to the right topic", capturedCommit, Matchers.hasKey(expectedTp)); - assertThat("Should have committed all the acked messages", capturedCommit.get(expectedTp).offset(), is((long)messageCount)); + assertThat("Should have committed to the right topic", capturedCommit, Matchers + .hasKey(expectedTp)); + assertThat("Should have committed all the acked messages", capturedCommit.get(expectedTp) + .offset(), is((long) messageCount)); /* Verify that the following acked (now committed) tuples are not emitted again - * Since the consumer position was somewhere in the middle of the acked tuples when the commit happened, - * this verifies that the spout keeps the consumer position ahead of the committed offset when committing + * Since the consumer position was somewhere in the middle of the acked tuples when the + * commit happened, + * this verifies that the spout keeps the consumer position ahead of the committed offset + * when committing */ - //Just do a few polls to check that nothing more is emitted - for(int i = 0; i < 3; i++) { + // Just do a few polls to check that nothing more is emitted + for (int i = 0; i < 3; i++) { spout.nextTuple(); } verify(collectorMock, never()).emit(anyString(), anyList(), any()); @@ -118,12 +131,14 @@ public void testClearingWaitingToEmitIfConsumerPositionIsNotBehindWhenCommitting int messagesInKafka = messageCountExcludingLast + 1; prepareSpout(messagesInKafka); - //Emit all messages and fail the first one while acking the rest + // Emit all messages and fail the first one while acking the rest for (int i = 0; i < messageCountExcludingLast; i++) { spout.nextTuple(); } - ArgumentCaptor messageIdCaptor = ArgumentCaptor.forClass(KafkaSpoutMessageId.class); - verify(collectorMock, times(messageCountExcludingLast)).emit(anyString(), anyList(), messageIdCaptor.capture()); + ArgumentCaptor messageIdCaptor = ArgumentCaptor + .forClass(KafkaSpoutMessageId.class); + verify(collectorMock, times(messageCountExcludingLast)).emit(anyString(), anyList(), + messageIdCaptor.capture()); List messageIds = messageIdCaptor.getAllValues(); for (int i = 1; i < messageIds.size(); i++) { spout.ack(messageIds.get(i)); @@ -131,18 +146,22 @@ public void testClearingWaitingToEmitIfConsumerPositionIsNotBehindWhenCommitting KafkaSpoutMessageId failedTuple = messageIds.get(0); spout.fail(failedTuple); - //Advance the time and replay the failed tuple. - //Since the last tuple on the partition is more than maxPollRecords ahead of the failed tuple, it shouldn't be emitted here + // Advance the time and replay the failed tuple. + // Since the last tuple on the partition is more than maxPollRecords ahead of the failed + // tuple, it shouldn't be emitted here reset(collectorMock); spout.nextTuple(); - ArgumentCaptor failedIdReplayCaptor = ArgumentCaptor.forClass(KafkaSpoutMessageId.class); + ArgumentCaptor failedIdReplayCaptor = ArgumentCaptor + .forClass(KafkaSpoutMessageId.class); verify(collectorMock).emit(anyString(), anyList(), failedIdReplayCaptor.capture()); - assertThat("Expected replay of failed tuple", failedIdReplayCaptor.getValue(), is(failedTuple)); + assertThat("Expected replay of failed tuple", failedIdReplayCaptor.getValue(), + is(failedTuple)); /* Ack the tuple, and commit. * - * The waiting to emit list should now be cleared, and the next emitted tuple should be the last tuple on the partition, + * The waiting to emit list should now be cleared, and the next emitted tuple should be the + * last tuple on the partition, * which hasn't been emitted yet */ reset(collectorMock); @@ -153,16 +172,21 @@ public void testClearingWaitingToEmitIfConsumerPositionIsNotBehindWhenCommitting Map capturedCommit = commitCapture.getValue(); TopicPartition expectedTp = new TopicPartition(SingleTopicKafkaSpoutConfiguration.TOPIC, 0); - assertThat("Should have committed to the right topic", capturedCommit, Matchers.hasKey(expectedTp)); - assertThat("Should have committed all the acked messages", capturedCommit.get(expectedTp).offset(), is((long)messageCountExcludingLast)); + assertThat("Should have committed to the right topic", capturedCommit, Matchers + .hasKey(expectedTp)); + assertThat("Should have committed all the acked messages", capturedCommit.get(expectedTp) + .offset(), is((long) messageCountExcludingLast)); - ArgumentCaptor lastOffsetMessageCaptor = ArgumentCaptor.forClass(KafkaSpoutMessageId.class); + ArgumentCaptor lastOffsetMessageCaptor = ArgumentCaptor + .forClass(KafkaSpoutMessageId.class); verify(collectorMock).emit(anyString(), anyList(), lastOffsetMessageCaptor.capture()); - assertThat("Expected emit of the final tuple in the partition", lastOffsetMessageCaptor.getValue().offset(), is(messagesInKafka - 1L)); + assertThat("Expected emit of the final tuple in the partition", lastOffsetMessageCaptor + .getValue().offset(), is(messagesInKafka - 1L)); reset(collectorMock); - //Nothing else should be emitted, all tuples are acked except for the final tuple, which is pending. - for(int i = 0; i < 3; i++) { + // Nothing else should be emitted, all tuples are acked except for the final tuple, which is + // pending. + for (int i = 0; i < 3; i++) { spout.nextTuple(); } verify(collectorMock, never()).emit(anyString(), anyList(), any()); @@ -173,35 +197,36 @@ public void testShouldContinueWithSlowDoubleAcks() throws Exception { final int messageCount = 20; prepareSpout(messageCount); - //play 1st tuple + // play 1st tuple ArgumentCaptor messageIdToDoubleAck = ArgumentCaptor.forClass(Object.class); spout.nextTuple(); verify(collectorMock).emit(anyString(), anyList(), messageIdToDoubleAck.capture()); spout.ack(messageIdToDoubleAck.getValue()); - //Emit some more messages - for(int i = 0; i < messageCount / 2; i++) { + // Emit some more messages + for (int i = 0; i < messageCount / 2; i++) { spout.nextTuple(); } spout.ack(messageIdToDoubleAck.getValue()); - //Emit any remaining messages - for(int i = 0; i < messageCount; i++) { + // Emit any remaining messages + for (int i = 0; i < messageCount; i++) { spout.nextTuple(); } - //Verify that all messages are emitted, ack all the messages + // Verify that all messages are emitted, ack all the messages ArgumentCaptor messageIds = ArgumentCaptor.forClass(Object.class); - verify(collectorMock, times(messageCount)).emit(eq(SingleTopicKafkaSpoutConfiguration.STREAM), + verify(collectorMock, times(messageCount)) + .emit(eq(SingleTopicKafkaSpoutConfiguration.STREAM), anyList(), messageIds.capture()); - for(Object id : messageIds.getAllValues()) { + for (Object id : messageIds.getAllValues()) { spout.ack(id); } Time.advanceTime(commitOffsetPeriodMs + KafkaSpout.TIMER_DELAY_MS); - //Commit offsets + // Commit offsets spout.nextTuple(); verifyAllMessagesCommitted(messageCount); @@ -212,8 +237,8 @@ public void testShouldEmitAllMessages() throws Exception { final int messageCount = 10; prepareSpout(messageCount); - //Emit all messages and check that they are emitted. Ack the messages too - for(int i = 0; i < messageCount; i++) { + // Emit all messages and check that they are emitted. Ack the messages too + for (int i = 0; i < messageCount; i++) { spout.nextTuple(); ArgumentCaptor messageId = ArgumentCaptor.forClass(Object.class); verify(collectorMock).emit( @@ -227,7 +252,7 @@ public void testShouldEmitAllMessages() throws Exception { } Time.advanceTime(commitOffsetPeriodMs + KafkaSpout.TIMER_DELAY_MS); - //Commit offsets + // Commit offsets spout.nextTuple(); verifyAllMessagesCommitted(messageCount); @@ -238,37 +263,38 @@ public void testShouldReplayInOrderFailedMessages() throws Exception { final int messageCount = 10; prepareSpout(messageCount); - //play and ack 1 tuple + // play and ack 1 tuple ArgumentCaptor messageIdAcked = ArgumentCaptor.forClass(Object.class); spout.nextTuple(); verify(collectorMock).emit(anyString(), anyList(), messageIdAcked.capture()); spout.ack(messageIdAcked.getValue()); reset(collectorMock); - //play and fail 1 tuple + // play and fail 1 tuple ArgumentCaptor messageIdFailed = ArgumentCaptor.forClass(Object.class); spout.nextTuple(); verify(collectorMock).emit(anyString(), anyList(), messageIdFailed.capture()); spout.fail(messageIdFailed.getValue()); reset(collectorMock); - //Emit all remaining messages. Failed tuples retry immediately with current configuration, so no need to wait. - for(int i = 0; i < messageCount; i++) { + // Emit all remaining messages. Failed tuples retry immediately with current configuration, + // so no need to wait. + for (int i = 0; i < messageCount; i++) { spout.nextTuple(); } ArgumentCaptor remainingMessageIds = ArgumentCaptor.forClass(Object.class); - //All messages except the first acked message should have been emitted + // All messages except the first acked message should have been emitted verify(collectorMock, times(messageCount - 1)).emit( eq(SingleTopicKafkaSpoutConfiguration.STREAM), anyList(), remainingMessageIds.capture()); - for(Object id : remainingMessageIds.getAllValues()) { + for (Object id : remainingMessageIds.getAllValues()) { spout.ack(id); } Time.advanceTime(commitOffsetPeriodMs + KafkaSpout.TIMER_DELAY_MS); - //Commit offsets + // Commit offsets spout.nextTuple(); verifyAllMessagesCommitted(messageCount); @@ -279,40 +305,41 @@ public void testShouldReplayFirstTupleFailedOutOfOrder() throws Exception { final int messageCount = 10; prepareSpout(messageCount); - //play 1st tuple + // play 1st tuple ArgumentCaptor messageIdToFail = ArgumentCaptor.forClass(Object.class); spout.nextTuple(); verify(collectorMock).emit(anyString(), anyList(), messageIdToFail.capture()); reset(collectorMock); - //play 2nd tuple + // play 2nd tuple ArgumentCaptor messageIdToAck = ArgumentCaptor.forClass(Object.class); spout.nextTuple(); verify(collectorMock).emit(anyString(), anyList(), messageIdToAck.capture()); reset(collectorMock); - //ack 2nd tuple + // ack 2nd tuple spout.ack(messageIdToAck.getValue()); - //fail 1st tuple + // fail 1st tuple spout.fail(messageIdToFail.getValue()); - //Emit all remaining messages. Failed tuples retry immediately with current configuration, so no need to wait. - for(int i = 0; i < messageCount; i++) { + // Emit all remaining messages. Failed tuples retry immediately with current configuration, + // so no need to wait. + for (int i = 0; i < messageCount; i++) { spout.nextTuple(); } ArgumentCaptor remainingIds = ArgumentCaptor.forClass(Object.class); - //All messages except the first acked message should have been emitted + // All messages except the first acked message should have been emitted verify(collectorMock, times(messageCount - 1)).emit( eq(SingleTopicKafkaSpoutConfiguration.STREAM), anyList(), remainingIds.capture()); - for(Object id : remainingIds.getAllValues()) { + for (Object id : remainingIds.getAllValues()) { spout.ack(id); } Time.advanceTime(commitOffsetPeriodMs + KafkaSpout.TIMER_DELAY_MS); - //Commit offsets + // Commit offsets spout.nextTuple(); verifyAllMessagesCommitted(messageCount); @@ -320,27 +347,31 @@ public void testShouldReplayFirstTupleFailedOutOfOrder() throws Exception { @Test public void testShouldReplayAllFailedTuplesWhenFailedOutOfOrder() throws Exception { - //The spout must reemit retriable tuples, even if they fail out of order. - //The spout should be able to skip tuples it has already emitted when retrying messages, even if those tuples are also retries. + // The spout must reemit retriable tuples, even if they fail out of order. + // The spout should be able to skip tuples it has already emitted when retrying messages, + // even if those tuples are also retries. final int messageCount = 10; prepareSpout(messageCount); - //play all tuples + // play all tuples for (int i = 0; i < messageCount; i++) { spout.nextTuple(); } - ArgumentCaptor messageIds = ArgumentCaptor.forClass(KafkaSpoutMessageId.class); - verify(collectorMock, times(messageCount)).emit(anyString(), anyList(), messageIds.capture()); + ArgumentCaptor messageIds = ArgumentCaptor + .forClass(KafkaSpoutMessageId.class); + verify(collectorMock, times(messageCount)).emit(anyString(), anyList(), messageIds + .capture()); reset(collectorMock); - //Fail tuple 5 and 3, call nextTuple, then fail tuple 2 + // Fail tuple 5 and 3, call nextTuple, then fail tuple 2 List capturedMessageIds = messageIds.getAllValues(); spout.fail(capturedMessageIds.get(5)); spout.fail(capturedMessageIds.get(3)); spout.nextTuple(); spout.fail(capturedMessageIds.get(2)); - //Check that the spout will reemit all 3 failed tuples and no other tuples - ArgumentCaptor reemittedMessageIds = ArgumentCaptor.forClass(KafkaSpoutMessageId.class); + // Check that the spout will reemit all 3 failed tuples and no other tuples + ArgumentCaptor reemittedMessageIds = ArgumentCaptor + .forClass(KafkaSpoutMessageId.class); for (int i = 0; i < messageCount; i++) { spout.nextTuple(); } @@ -349,43 +380,50 @@ public void testShouldReplayAllFailedTuplesWhenFailedOutOfOrder() throws Excepti expectedReemitIds.add(capturedMessageIds.get(5)); expectedReemitIds.add(capturedMessageIds.get(3)); expectedReemitIds.add(capturedMessageIds.get(2)); - assertThat("Expected reemits to be the 3 failed tuples", new HashSet<>(reemittedMessageIds.getAllValues()), is(expectedReemitIds)); + assertThat("Expected reemits to be the 3 failed tuples", new HashSet<>(reemittedMessageIds + .getAllValues()), is(expectedReemitIds)); } @Test public void testShouldDropMessagesAfterMaxRetriesAreReached() throws Exception { - //Check that if one message fails repeatedly, the retry cap limits how many times the message can be reemitted + // Check that if one message fails repeatedly, the retry cap limits how many times the + // message can be reemitted final int messageCount = 1; prepareSpout(messageCount); - //Emit and fail the same tuple until we've reached retry limit + // Emit and fail the same tuple until we've reached retry limit for (int i = 0; i <= maxRetries; i++) { - ArgumentCaptor messageIdFailed = ArgumentCaptor.forClass(KafkaSpoutMessageId.class); + ArgumentCaptor messageIdFailed = ArgumentCaptor + .forClass(KafkaSpoutMessageId.class); spout.nextTuple(); verify(collectorMock).emit(anyString(), anyList(), messageIdFailed.capture()); KafkaSpoutMessageId msgId = messageIdFailed.getValue(); spout.fail(msgId); - assertThat("Expected message id number of failures to match the number of times the message has failed", msgId.numFails(), is(i + 1)); + assertThat("Expected message id number of failures to match the number of times the " + + "message has failed", msgId.numFails(), is(i + 1)); reset(collectorMock); } - //Verify that the tuple is not emitted again + // Verify that the tuple is not emitted again spout.nextTuple(); verify(collectorMock, never()).emit(anyString(), anyList(), any()); } @Test public void testSpoutMustRefreshPartitionsEvenIfNotPolling() throws Exception { - SingleTopicKafkaUnitSetupHelper.initializeSpout(spout, conf, topologyContext, collectorMock); + SingleTopicKafkaUnitSetupHelper.initializeSpout(spout, conf, topologyContext, + collectorMock); - //Nothing is assigned yet, should emit nothing + // Nothing is assigned yet, should emit nothing spout.nextTuple(); verify(collectorMock, never()).emit(anyString(), anyList(), any(KafkaSpoutMessageId.class)); - SingleTopicKafkaUnitSetupHelper.populateTopicData(kafkaUnitExtension.getKafkaUnit(), SingleTopicKafkaSpoutConfiguration.TOPIC, 1); - Time.advanceTime(KafkaSpoutConfig.DEFAULT_PARTITION_REFRESH_PERIOD_MS + KafkaSpout.TIMER_DELAY_MS); + SingleTopicKafkaUnitSetupHelper.populateTopicData(kafkaUnitExtension.getKafkaUnit(), + SingleTopicKafkaSpoutConfiguration.TOPIC, 1); + Time.advanceTime(KafkaSpoutConfig.DEFAULT_PARTITION_REFRESH_PERIOD_MS + + KafkaSpout.TIMER_DELAY_MS); - //The new partition should be discovered and the message should be emitted + // The new partition should be discovered and the message should be emitted spout.nextTuple(); verify(collectorMock).emit(anyString(), anyList(), any(KafkaSpoutMessageId.class)); } diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutTopologyDeployActivateDeactivateTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutTopologyDeployActivateDeactivateTest.java index a8fac1f1e40..9fbacbe1339 100644 --- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutTopologyDeployActivateDeactivateTest.java +++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutTopologyDeployActivateDeactivateTest.java @@ -36,7 +36,8 @@ public KafkaSpoutTopologyDeployActivateDeactivateTest() { @Override KafkaSpoutConfig createSpoutConfig() { return SingleTopicKafkaSpoutConfiguration.setCommonSpoutConfig( - KafkaSpoutConfig.builder("127.0.0.1:" + kafkaUnitExtension.getKafkaUnit().getKafkaPort(), + KafkaSpoutConfig.builder("127.0.0.1:" + kafkaUnitExtension.getKafkaUnit() + .getKafkaPort(), Pattern.compile(SingleTopicKafkaSpoutConfiguration.TOPIC))) .setOffsetCommitPeriodMs(commitOffsetPeriodMs) .setFirstPollOffsetStrategy(FirstPollOffsetStrategy.EARLIEST) @@ -50,7 +51,7 @@ public void test_FirstPollStrategy_Earliest_NotEnforced_OnTopologyActivateDeacti nextTuple_verifyEmitted_ack_resetCollector(0); - //Commits offsets during deactivation + // Commits offsets during deactivation spout.deactivate(); verifyAllMessagesCommitted(1); @@ -71,15 +72,17 @@ public void test_FirstPollStrategy_Earliest_NotEnforced_OnPartitionReassignment( nextTuple_verifyEmitted_ack_resetCollector(0); - //Commits offsets during deactivation + // Commits offsets during deactivation spout.deactivate(); verifyAllMessagesCommitted(1); - // Restart topology with the same topology id, which mimics the behavior of partition reassignment + // Restart topology with the same topology id, which mimics the behavior of partition + // reassignment setUp(); // Initialize spout using the same populated data (i.e same kafkaUnitRule) - SingleTopicKafkaUnitSetupHelper.initializeSpout(spout, conf, topologyContext, collectorMock); + SingleTopicKafkaUnitSetupHelper.initializeSpout(spout, conf, topologyContext, + collectorMock); nextTuple_verifyEmitted_ack_resetCollector(1); @@ -95,7 +98,7 @@ public void test_FirstPollStrategy_Earliest_Enforced_OnlyOnTopologyDeployment() nextTuple_verifyEmitted_ack_resetCollector(0); - //Commits offsets during deactivation + // Commits offsets during deactivation spout.deactivate(); verifyAllMessagesCommitted(1); @@ -104,9 +107,10 @@ public void test_FirstPollStrategy_Earliest_Enforced_OnlyOnTopologyDeployment() setUp(); when(topologyContext.getStormId()).thenReturn("topology-2"); // Initialize spout using the same populated data (i.e same kafkaUnitRule) - SingleTopicKafkaUnitSetupHelper.initializeSpout(spout, conf, topologyContext, collectorMock); + SingleTopicKafkaUnitSetupHelper.initializeSpout(spout, conf, topologyContext, + collectorMock); - //Emit all messages and check that they are emitted. Ack the messages too + // Emit all messages and check that they are emitted. Ack the messages too for (int i = 0; i < messageCount; i++) { nextTuple_verifyEmitted_ack_resetCollector(i); } diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/MaxUncommittedOffsetTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/MaxUncommittedOffsetTest.java index 326410ed1c0..569b7722639 100755 --- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/MaxUncommittedOffsetTest.java +++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/MaxUncommittedOffsetTest.java @@ -51,7 +51,6 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; - @ExtendWith(MockitoExtension.class) public class MaxUncommittedOffsetTest { @@ -68,39 +67,50 @@ public class MaxUncommittedOffsetTest { private final int maxUncommittedOffsets = 10; private final int maxPollRecords = 5; private final int initialRetryDelaySecs = 60; - private final KafkaSpoutConfig spoutConfig = createKafkaSpoutConfigBuilder(kafkaUnitExtension.getKafkaUnit().getKafkaPort()) + private final KafkaSpoutConfig spoutConfig = + createKafkaSpoutConfigBuilder(kafkaUnitExtension.getKafkaUnit().getKafkaPort()) .setOffsetCommitPeriodMs(commitOffsetPeriodMs) .setProp(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, maxPollRecords) .setMaxUncommittedOffsets(maxUncommittedOffsets) .setRetry(new KafkaSpoutRetryExponentialBackoff(KafkaSpoutRetryExponentialBackoff.TimeInterval.seconds(initialRetryDelaySecs), KafkaSpoutRetryExponentialBackoff.TimeInterval.seconds(0), - 1, KafkaSpoutRetryExponentialBackoff.TimeInterval.seconds(initialRetryDelaySecs))) //Retry once after a minute + 1, KafkaSpoutRetryExponentialBackoff.TimeInterval + .seconds(initialRetryDelaySecs))) // Retry once after a minute .build(); private KafkaSpout spout; @BeforeEach public void setUp() { - //This is because the tests are checking that a hard cap of maxUncommittedOffsets + maxPollRecords - 1 uncommitted offsets exists - //so Kafka must be able to return more messages than that in order for the tests to be meaningful - assertThat("Current tests require numMessages >= 2*maxUncommittedOffsets", numMessages, greaterThanOrEqualTo(maxUncommittedOffsets * 2)); - //This is to verify that a low maxPollRecords does not interfere with reemitting failed tuples - //The spout must be able to reemit all retriable tuples, even if the maxPollRecords is set to a low value compared to maxUncommittedOffsets. - assertThat("Current tests require maxPollRecords < maxUncommittedOffsets", maxPollRecords, lessThanOrEqualTo(maxUncommittedOffsets)); + // This is because the tests are checking that a hard cap of maxUncommittedOffsets + + // maxPollRecords - 1 uncommitted offsets exists + // so Kafka must be able to return more messages than that in order for the tests to be + // meaningful + assertThat("Current tests require numMessages >= 2*maxUncommittedOffsets", numMessages, + greaterThanOrEqualTo(maxUncommittedOffsets * 2)); + // This is to verify that a low maxPollRecords does not interfere with reemitting failed + // tuples + // The spout must be able to reemit all retriable tuples, even if the maxPollRecords is set + // to a low value compared to maxUncommittedOffsets. + assertThat("Current tests require maxPollRecords < maxUncommittedOffsets", maxPollRecords, + lessThanOrEqualTo(maxUncommittedOffsets)); spout = new KafkaSpout<>(spoutConfig); new ClientFactoryDefault().createConsumer(spoutConfig.getKafkaProps()); } private void prepareSpout(int msgCount) throws Exception { - SingleTopicKafkaUnitSetupHelper.populateTopicData(kafkaUnitExtension.getKafkaUnit(), SingleTopicKafkaSpoutConfiguration.TOPIC, msgCount); + SingleTopicKafkaUnitSetupHelper.populateTopicData(kafkaUnitExtension.getKafkaUnit(), + SingleTopicKafkaSpoutConfiguration.TOPIC, msgCount); SingleTopicKafkaUnitSetupHelper.initializeSpout(spout, conf, topologyContext, collector); } private ArgumentCaptor emitMaxUncommittedOffsetsMessagesAndCheckNoMoreAreEmitted(int messageCount) throws Exception { - assertThat("The message count is less than maxUncommittedOffsets. This test is not meaningful with this configuration.", messageCount, greaterThanOrEqualTo(maxUncommittedOffsets)); - //The spout must respect maxUncommittedOffsets when requesting/emitting tuples + assertThat("The message count is less than maxUncommittedOffsets. This test is not " + + "meaningful with this configuration.", messageCount, greaterThanOrEqualTo(maxUncommittedOffsets)); + // The spout must respect maxUncommittedOffsets when requesting/emitting tuples prepareSpout(messageCount); - //Try to emit all messages. Ensure only maxUncommittedOffsets are emitted - ArgumentCaptor messageIds = ArgumentCaptor.forClass(KafkaSpoutMessageId.class); + // Try to emit all messages. Ensure only maxUncommittedOffsets are emitted + ArgumentCaptor messageIds = ArgumentCaptor + .forClass(KafkaSpoutMessageId.class); for (int i = 0; i < messageCount; i++) { spout.nextTuple(); } @@ -113,13 +123,14 @@ private ArgumentCaptor emitMaxUncommittedOffsetsMessagesAnd @Test public void testNextTupleCanEmitMoreMessagesWhenDroppingBelowMaxUncommittedOffsetsDueToCommit() throws Exception { - //The spout must respect maxUncommittedOffsets after committing a set of records + // The spout must respect maxUncommittedOffsets after committing a set of records try (Time.SimulatedTime simulatedTime = new Time.SimulatedTime()) { - //First check that maxUncommittedOffsets is respected when emitting from scratch - ArgumentCaptor messageIds = emitMaxUncommittedOffsetsMessagesAndCheckNoMoreAreEmitted(numMessages); + // First check that maxUncommittedOffsets is respected when emitting from scratch + ArgumentCaptor messageIds = + emitMaxUncommittedOffsetsMessagesAndCheckNoMoreAreEmitted(numMessages); reset(collector); - //Ack all emitted messages and commit them + // Ack all emitted messages and commit them for (KafkaSpoutMessageId messageId : messageIds.getAllValues()) { spout.ack(messageId); } @@ -127,7 +138,7 @@ public void testNextTupleCanEmitMoreMessagesWhenDroppingBelowMaxUncommittedOffse spout.nextTuple(); - //Now check that the spout will emit another maxUncommittedOffsets messages + // Now check that the spout will emit another maxUncommittedOffsets messages for (int i = 0; i < numMessages; i++) { spout.nextTuple(); } @@ -140,13 +151,15 @@ public void testNextTupleCanEmitMoreMessagesWhenDroppingBelowMaxUncommittedOffse @Test public void testNextTupleWillRespectMaxUncommittedOffsetsWhenThereAreAckedUncommittedTuples() throws Exception { - //The spout must respect maxUncommittedOffsets even if some tuples have been acked but not committed + // The spout must respect maxUncommittedOffsets even if some tuples have been acked but not + // committed try (Time.SimulatedTime simulatedTime = new Time.SimulatedTime()) { - //First check that maxUncommittedOffsets is respected when emitting from scratch - ArgumentCaptor messageIds = emitMaxUncommittedOffsetsMessagesAndCheckNoMoreAreEmitted(numMessages); + // First check that maxUncommittedOffsets is respected when emitting from scratch + ArgumentCaptor messageIds = + emitMaxUncommittedOffsetsMessagesAndCheckNoMoreAreEmitted(numMessages); reset(collector); - //Fail all emitted messages except the last one. Try to commit. + // Fail all emitted messages except the last one. Try to commit. List messageIdList = messageIds.getAllValues(); for (int i = 0; i < messageIdList.size() - 1; i++) { spout.fail(messageIdList.get(i)); @@ -155,7 +168,7 @@ public void testNextTupleWillRespectMaxUncommittedOffsetsWhenThereAreAckedUncomm Time.advanceTime(commitOffsetPeriodMs + KafkaSpout.TIMER_DELAY_MS); spout.nextTuple(); - //Now check that the spout will not emit anything else since nothing has been committed + // Now check that the spout will not emit anything else since nothing has been committed for (int i = 0; i < numMessages; i++) { spout.nextTuple(); } @@ -168,7 +181,7 @@ public void testNextTupleWillRespectMaxUncommittedOffsetsWhenThereAreAckedUncomm } private void failAllExceptTheFirstMessageThenCommit(ArgumentCaptor messageIds) { - //Fail all emitted messages except the first. Commit the first. + // Fail all emitted messages except the first. Commit the first. List messageIdList = messageIds.getAllValues(); for (int i = 1; i < messageIdList.size(); i++) { spout.fail(messageIdList.get(i)); @@ -189,29 +202,32 @@ public void testNextTupleWillNotEmitMoreThanMaxUncommittedOffsetsPlusMaxPollReco */ try (Time.SimulatedTime simulatedTime = new Time.SimulatedTime()) { - //First check that maxUncommittedOffsets is respected when emitting from scratch - ArgumentCaptor messageIds = emitMaxUncommittedOffsetsMessagesAndCheckNoMoreAreEmitted(numMessages); + // First check that maxUncommittedOffsets is respected when emitting from scratch + ArgumentCaptor messageIds = + emitMaxUncommittedOffsetsMessagesAndCheckNoMoreAreEmitted(numMessages); reset(collector); - //Fail only the last tuple + // Fail only the last tuple List messageIdList = messageIds.getAllValues(); KafkaSpoutMessageId failedMessageId = messageIdList.get(messageIdList.size() - 1); spout.fail(failedMessageId); - //Offset 0 to maxUncommittedOffsets - 2 are pending, maxUncommittedOffsets - 1 is failed but not retriable - //The spout should not emit any more tuples. + // Offset 0 to maxUncommittedOffsets - 2 are pending, maxUncommittedOffsets - 1 is + // failed but not retriable + // The spout should not emit any more tuples. spout.nextTuple(); verify(collector, never()).emit( any(), any(), any()); - //Allow the failed record to retry + // Allow the failed record to retry Time.advanceTimeSecs(initialRetryDelaySecs); for (int i = 0; i < maxPollRecords; i++) { spout.nextTuple(); } - ArgumentCaptor secondRunMessageIds = ArgumentCaptor.forClass(KafkaSpoutMessageId.class); + ArgumentCaptor secondRunMessageIds = ArgumentCaptor + .forClass(KafkaSpoutMessageId.class); verify(collector, times(maxPollRecords)).emit( any(), any(), @@ -219,9 +235,11 @@ public void testNextTupleWillNotEmitMoreThanMaxUncommittedOffsetsPlusMaxPollReco reset(collector); assertThat(secondRunMessageIds.getAllValues().get(0), is(failedMessageId)); - //There should now be maxUncommittedOffsets + maxPollRecords emitted in all. - //Fail the last emitted tuple and verify that the spout won't retry it because it's above the emit limit. - spout.fail(secondRunMessageIds.getAllValues().get(secondRunMessageIds.getAllValues().size() - 1)); + // There should now be maxUncommittedOffsets + maxPollRecords emitted in all. + // Fail the last emitted tuple and verify that the spout won't retry it because it's + // above the emit limit. + spout.fail(secondRunMessageIds.getAllValues().get(secondRunMessageIds.getAllValues() + .size() - 1)); Time.advanceTimeSecs(initialRetryDelaySecs); spout.nextTuple(); verify(collector, never()).emit(any(), any(), any()); @@ -235,20 +253,23 @@ public void testNextTupleWillAllowRetryForTuplesBelowEmitLimit() throws Exceptio It must retry tuples within that limit, even if more tuples were emitted. */ try (Time.SimulatedTime simulatedTime = new Time.SimulatedTime()) { - //First check that maxUncommittedOffsets is respected when emitting from scratch - ArgumentCaptor messageIds = emitMaxUncommittedOffsetsMessagesAndCheckNoMoreAreEmitted(numMessages); + // First check that maxUncommittedOffsets is respected when emitting from scratch + ArgumentCaptor messageIds = + emitMaxUncommittedOffsetsMessagesAndCheckNoMoreAreEmitted(numMessages); reset(collector); failAllExceptTheFirstMessageThenCommit(messageIds); - //Offset 0 is committed, 1 to maxUncommittedOffsets - 1 are failed but not retriable - //The spout should now emit another maxPollRecords messages - //This is allowed because the committed message brings the numUncommittedOffsets below the cap + // Offset 0 is committed, 1 to maxUncommittedOffsets - 1 are failed but not retriable + // The spout should now emit another maxPollRecords messages + // This is allowed because the committed message brings the numUncommittedOffsets below + // the cap for (int i = 0; i < maxUncommittedOffsets; i++) { spout.nextTuple(); } - ArgumentCaptor secondRunMessageIds = ArgumentCaptor.forClass(KafkaSpoutMessageId.class); + ArgumentCaptor secondRunMessageIds = ArgumentCaptor + .forClass(KafkaSpoutMessageId.class); verify(collector, times(maxPollRecords)).emit( any(), any(), @@ -261,13 +282,18 @@ public void testNextTupleWillAllowRetryForTuplesBelowEmitLimit() throws Exceptio List secondRunOffsets = secondRunMessageIds.getAllValues().stream() .map(messageId -> messageId.offset()) .collect(Collectors.toList()); - assertThat("Expected the newly emitted messages to have no overlap with the first batch", secondRunOffsets.removeAll(firstRunOffsets), is(false)); + assertThat("Expected the newly emitted messages to have no overlap with the first " + + "batch", secondRunOffsets.removeAll(firstRunOffsets), is(false)); - //Offset 0 is committed, 1 to maxUncommittedOffsets-1 are failed, maxUncommittedOffsets to maxUncommittedOffsets + maxPollRecords-1 are emitted - //Fail the last tuples so only offset 0 is not failed. - //Advance time so the failed tuples become ready for retry, and check that the spout will emit retriable tuples - //for all the failed tuples that are within maxUncommittedOffsets tuples of the committed offset - //This means 1 to maxUncommitteddOffsets, but not maxUncommittedOffsets+1...maxUncommittedOffsets+maxPollRecords-1 + // Offset 0 is committed, 1 to maxUncommittedOffsets-1 are failed, maxUncommittedOffsets + // to maxUncommittedOffsets + maxPollRecords-1 are emitted + // Fail the last tuples so only offset 0 is not failed. + // Advance time so the failed tuples become ready for retry, and check that the spout + // will emit retriable tuples + // for all the failed tuples that are within maxUncommittedOffsets tuples of the + // committed offset + // This means 1 to maxUncommitteddOffsets, but not + // maxUncommittedOffsets+1...maxUncommittedOffsets+maxPollRecords-1 for (KafkaSpoutMessageId msgId : secondRunMessageIds.getAllValues()) { spout.fail(msgId); } @@ -275,7 +301,8 @@ public void testNextTupleWillAllowRetryForTuplesBelowEmitLimit() throws Exceptio for (int i = 0; i < numMessages; i++) { spout.nextTuple(); } - ArgumentCaptor thirdRunMessageIds = ArgumentCaptor.forClass(KafkaSpoutMessageId.class); + ArgumentCaptor thirdRunMessageIds = ArgumentCaptor + .forClass(KafkaSpoutMessageId.class); verify(collector, times(maxUncommittedOffsets)).emit( anyString(), anyList(), @@ -285,7 +312,8 @@ public void testNextTupleWillAllowRetryForTuplesBelowEmitLimit() throws Exceptio List thirdRunOffsets = thirdRunMessageIds.getAllValues().stream() .map(msgId -> msgId.offset()) .collect(Collectors.toList()); - assertThat("Expected the emitted messages to be retries of the failed tuples from the first batch, plus the first failed tuple from the second batch", thirdRunOffsets, everyItem(either(isIn(firstRunOffsets)).or(is(secondRunMessageIds.getAllValues().get(0).offset())))); + assertThat("Expected the emitted messages to be retries of the failed tuples from the " + + "first batch, plus the first failed tuple from the second batch", thirdRunOffsets, everyItem(either(isIn(firstRunOffsets)).or(is(secondRunMessageIds.getAllValues().get(0).offset())))); } } } diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/SimpleRecordTranslatorTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/SimpleRecordTranslatorTest.java index 7db07f533f8..34b7a16e28a 100644 --- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/SimpleRecordTranslatorTest.java +++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/SimpleRecordTranslatorTest.java @@ -17,23 +17,23 @@ */ package org.apache.storm.kafka.spout; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +import java.util.Collections; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.storm.tuple.Fields; import org.apache.storm.tuple.Values; import org.junit.jupiter.api.Test; -import java.util.Collections; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; - public class SimpleRecordTranslatorTest { @Test public void testBasic() { SimpleRecordTranslator trans = new SimpleRecordTranslator<>((r) -> new Values(r.value()), new Fields("value")); assertEquals(Collections.singletonList("default"), trans.streams()); - ConsumerRecord cr = new ConsumerRecord<>("TOPIC", 100, 100, "THE KEY", "THE VALUE"); + ConsumerRecord cr = new ConsumerRecord<>("TOPIC", 100, 100, "THE KEY", + "THE VALUE"); assertEquals(Collections.singletonList("THE VALUE"), trans.apply(cr)); } diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/SingleTopicKafkaUnitSetupHelper.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/SingleTopicKafkaUnitSetupHelper.java index 81c489e57f5..7c8547327e3 100644 --- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/SingleTopicKafkaUnitSetupHelper.java +++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/SingleTopicKafkaUnitSetupHelper.java @@ -43,7 +43,8 @@ public class SingleTopicKafkaUnitSetupHelper { * @param topicName The topic to produce messages for * @param msgCount The number of messages to produce */ - public static void populateTopicData(KafkaUnit kafkaUnit, String topicName, int msgCount) throws Exception { + public static void populateTopicData(KafkaUnit kafkaUnit, String topicName, + int msgCount) throws Exception { kafkaUnit.createTopic(topicName); for (int i = 0; i < msgCount; i++) { @@ -63,9 +64,11 @@ public static void verifyAllMessagesCommitted(Consumer consumerSpy, ArgumentCaptor> commitCapture, long messageCount) { verify(consumerSpy, times(1)).commitSync(commitCapture.capture()); Map commits = commitCapture.getValue(); - assertThat("Expected commits for only one topic partition", commits.entrySet().size(), is(1)); + assertThat("Expected commits for only one topic partition", commits.entrySet().size(), + is(1)); OffsetAndMetadata offset = commits.entrySet().iterator().next().getValue(); - assertThat("Expected committed offset to cover all emitted messages", offset.offset(), is(messageCount)); + assertThat("Expected committed offset to cover all emitted messages", offset.offset(), + is(messageCount)); } /** @@ -78,7 +81,8 @@ public static void verifyAllMessagesCommitted(Consumer consumerSpy, * @param topoContextMock The TopologyContext mock * @param collectorMock The output collector mock */ - public static void initializeSpout(KafkaSpout spout, Map topoConf, TopologyContext topoContextMock, + public static void initializeSpout(KafkaSpout spout, Map topoConf, + TopologyContext topoContextMock, SpoutOutputCollector collectorMock) throws Exception { when(topoContextMock.getThisTaskIndex()).thenReturn(0); when(topoContextMock.getComponentTasks(any())).thenReturn(Collections.singletonList(0)); diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/SpoutWithMockedConsumerSetupHelper.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/SpoutWithMockedConsumerSetupHelper.java index 70fae9073b2..fa85a85759c 100644 --- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/SpoutWithMockedConsumerSetupHelper.java +++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/SpoutWithMockedConsumerSetupHelper.java @@ -33,13 +33,16 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Map.Entry; +import java.util.Map; import java.util.Set; - import org.apache.kafka.clients.admin.Admin; import org.apache.kafka.clients.admin.MockAdminClient; -import org.apache.kafka.clients.consumer.*; +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.consumer.ConsumerRebalanceListener; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.KafkaConsumer; import org.apache.kafka.common.TopicPartition; import org.apache.storm.kafka.spout.internal.ClientFactory; import org.apache.storm.kafka.spout.subscription.ManualPartitioner; @@ -53,8 +56,10 @@ public class SpoutWithMockedConsumerSetupHelper { /** - * Creates, opens and activates a KafkaSpout using a mocked consumer. The TopicFilter and ManualPartitioner should be mock objects, - * since this method shortcircuits the TopicPartition assignment process and just calls onPartitionsAssigned on the rebalance listener. + * Creates, opens and activates a KafkaSpout using a mocked consumer. The TopicFilter and + * ManualPartitioner should be mock objects, + * since this method shortcircuits the TopicPartition assignment process and just calls + * onPartitionsAssigned on the rebalance listener. * * @param The Kafka key type * @param The Kafka value type @@ -63,18 +68,22 @@ public class SpoutWithMockedConsumerSetupHelper { * @param contextMock The topo context to pass to the spout * @param collectorMock The mocked collector to pass to the spout * @param consumerMock The mocked consumer - * @param assignedPartitions The partitions to assign to this spout. The consumer will act like these partitions are assigned to it. + * @param assignedPartitions The partitions to assign to this spout. The consumer will act like + * these partitions are assigned to it. * @return The spout */ - public static KafkaSpout setupSpout(KafkaSpoutConfig spoutConfig, Map topoConf, + public static KafkaSpout setupSpout(KafkaSpoutConfig spoutConfig, Map topoConf, TopologyContext contextMock, SpoutOutputCollector collectorMock, KafkaConsumer consumerMock, TopicPartition... assignedPartitions) { TopicFilter topicFilter = spoutConfig.getTopicFilter(); ManualPartitioner topicPartitioner = spoutConfig.getTopicPartitioner(); if (!mockingDetails(topicFilter).isMock() || !mockingDetails(topicPartitioner).isMock()) { - throw new IllegalStateException("Use a mocked TopicFilter and a mocked ManualPartitioner when using this method, it helps avoid complex stubbing"); + throw new IllegalStateException("Use a mocked TopicFilter and a mocked " + + "ManualPartitioner when using this method, it helps avoid complex stubbing"); } - Set assignedPartitionsSet = new HashSet<>(Arrays.asList(assignedPartitions)); + Set assignedPartitionsSet = new HashSet<>(Arrays + .asList(assignedPartitions)); TopicAssigner assigner = mock(TopicAssigner.class); doAnswer(invocation -> { @@ -106,7 +115,7 @@ public Admin createAdmin(Map adminProps) { } /** - * Creates sequential dummy records + * Creates sequential dummy records. * * @param The Kafka key type * @param The Kafka value type @@ -115,16 +124,20 @@ public Admin createAdmin(Map adminProps) { * @param numRecords The number of records to create * @return The dummy records */ - public static List> createRecords(TopicPartition topic, long startingOffset, int numRecords) { + public static List> createRecords(TopicPartition topic, + long startingOffset, int numRecords) { List> recordsForPartition = new ArrayList<>(); for (int i = 0; i < numRecords; i++) { - recordsForPartition.add(new ConsumerRecord<>(topic.topic(), topic.partition(), startingOffset + i, null, null)); + recordsForPartition.add(new ConsumerRecord<>(topic.topic(), topic.partition(), + startingOffset + i, null, null)); } return recordsForPartition; } /** - * Creates messages for the input offsets, emits the messages by calling nextTuple once per offset and returns the captured message ids + * Creates messages for the input offsets, emits the messages by calling nextTuple once per + * offset and returns the captured message ids. + * * @param The Kafka key type * @param The Kafka value type * @param spout The spout @@ -135,12 +148,16 @@ public static List> createRecords(TopicPartition top * @param offsetsToEmit The offsets to emit * @return The message ids emitted by the spout during the nextTuple calls */ - public static List pollAndEmit(KafkaSpout spout, KafkaConsumer consumerMock, int expectedEmits, SpoutOutputCollector collectorMock, TopicPartition partition, int... offsetsToEmit) { - return pollAndEmit(spout, consumerMock, expectedEmits, collectorMock, Collections.singletonMap(partition, offsetsToEmit)); + public static List pollAndEmit(KafkaSpout spout, + KafkaConsumer consumerMock, int expectedEmits, SpoutOutputCollector collectorMock, TopicPartition partition, int... offsetsToEmit) { + return pollAndEmit(spout, consumerMock, expectedEmits, collectorMock, Collections + .singletonMap(partition, offsetsToEmit)); } /** - * Creates messages for the input offsets, emits the messages by calling nextTuple once per offset and returns the captured message ids + * Creates messages for the input offsets, emits the messages by calling nextTuple once per + * offset and returns the captured message ids. + * * @param The Kafka key type * @param The Kafka value type * @param spout The spout @@ -149,7 +166,8 @@ public static List pollAndEmit(KafkaSpout spou * @param offsetsToEmit The offsets to emit per partition * @return The message ids emitted by the spout during the nextTuple calls */ - public static List pollAndEmit(KafkaSpout spout, KafkaConsumer consumerMock, int expectedEmits, SpoutOutputCollector collectorMock, Map offsetsToEmit) { + public static List pollAndEmit(KafkaSpout spout, + KafkaConsumer consumerMock, int expectedEmits, SpoutOutputCollector collectorMock, Map offsetsToEmit) { int totalOffsets = 0; Map>> records = new HashMap<>(); for (Entry entry : offsetsToEmit.entrySet()) { @@ -169,8 +187,10 @@ public static List pollAndEmit(KafkaSpout spou spout.nextTuple(); } - ArgumentCaptor messageIds = ArgumentCaptor.forClass(KafkaSpoutMessageId.class); - verify(collectorMock, times(expectedEmits)).emit(anyString(), anyList(), messageIds.capture()); + ArgumentCaptor messageIds = ArgumentCaptor + .forClass(KafkaSpoutMessageId.class); + verify(collectorMock, times(expectedEmits)).emit(anyString(), anyList(), messageIds + .capture()); return messageIds.getAllValues(); } diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/config/builder/SingleTopicKafkaSpoutConfiguration.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/config/builder/SingleTopicKafkaSpoutConfiguration.java index 7c2e588e5c5..a5bb22ead3c 100644 --- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/config/builder/SingleTopicKafkaSpoutConfiguration.java +++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/config/builder/SingleTopicKafkaSpoutConfiguration.java @@ -29,7 +29,6 @@ import org.apache.storm.tuple.Fields; import org.apache.storm.tuple.Values; - public class SingleTopicKafkaSpoutConfiguration { public static final String STREAM = "test_stream"; @@ -40,7 +39,8 @@ public static KafkaSpoutConfig.Builder createKafkaSpoutConfigBui } public static KafkaSpoutConfig.Builder createKafkaSpoutConfigBuilder(TopicFilter topicFilter, ManualPartitioner topicPartitioner, int port) { - return setCommonSpoutConfig(new KafkaSpoutConfig.Builder<>("127.0.0.1:" + port, topicFilter, topicPartitioner)); + return setCommonSpoutConfig(new KafkaSpoutConfig.Builder<>("127.0.0.1:" + port, topicFilter, + topicPartitioner)); } public static KafkaSpoutConfig.Builder setCommonSpoutConfig(KafkaSpoutConfig.Builder config) { @@ -55,12 +55,12 @@ public static KafkaSpoutConfig.Builder setCommonSpoutConfig(Kafk .setPollTimeoutMs(1000); } - protected static KafkaSpoutRetryService getNoDelayRetryService() { - /** + /* * Retry in a tight loop (keep unit tests fasts). */ - return new KafkaSpoutRetryExponentialBackoff(KafkaSpoutRetryExponentialBackoff.TimeInterval.seconds(0), KafkaSpoutRetryExponentialBackoff.TimeInterval.milliSeconds(0), + return new KafkaSpoutRetryExponentialBackoff(KafkaSpoutRetryExponentialBackoff.TimeInterval + .seconds(0), KafkaSpoutRetryExponentialBackoff.TimeInterval.milliSeconds(0), DEFAULT_MAX_RETRIES, KafkaSpoutRetryExponentialBackoff.TimeInterval.milliSeconds(0)); } diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/internal/OffsetManagerTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/internal/OffsetManagerTest.java index 6aa37d16312..33bfd3e26eb 100644 --- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/internal/OffsetManagerTest.java +++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/internal/OffsetManagerTest.java @@ -31,7 +31,8 @@ import org.junit.jupiter.api.Test; public class OffsetManagerTest { - private static final String COMMIT_METADATA = "{\"topologyId\":\"tp1\",\"taskId\":3,\"threadName\":\"Thread-20\"}"; + private static final String COMMIT_METADATA = + "{\"topologyId\":\"tp1\",\"taskId\":3,\"threadName\":\"Thread-20\"}"; private final long initialFetchOffset = 0; private final TopicPartition testTp = new TopicPartition("testTopic", 0); @@ -40,14 +41,16 @@ public class OffsetManagerTest { @Test public void testSkipMissingOffsetsWhenFindingNextCommitOffsetWithGapInMiddleOfAcked() { /* If topic compaction is enabled in Kafka, we sometimes need to commit past a gap of deleted offsets - * Since the Kafka consumer should return offsets in order, we can assume that if a message is acked + * Since the Kafka consumer should return offsets in order, we can assume that if a message + * is acked * then any prior message will have been emitted at least once. - * If we see an acked message and some of the offsets preceding it were not emitted, they must have been compacted away and should be skipped. + * If we see an acked message and some of the offsets preceding it were not emitted, they + * must have been compacted away and should be skipped. */ manager.addToEmitMsgs(0); manager.addToEmitMsgs(1); manager.addToEmitMsgs(2); - //3, 4 compacted away + // 3, 4 compacted away manager.addToEmitMsgs(initialFetchOffset + 5); manager.addToEmitMsgs(initialFetchOffset + 6); manager.addToAckMsgs(getMessageId(initialFetchOffset)); @@ -55,44 +58,51 @@ public void testSkipMissingOffsetsWhenFindingNextCommitOffsetWithGapInMiddleOfAc manager.addToAckMsgs(getMessageId(initialFetchOffset + 2)); manager.addToAckMsgs(getMessageId(initialFetchOffset + 6)); - assertThat("The offset manager should not skip past offset 5 which is still pending", manager.findNextCommitOffset(COMMIT_METADATA).offset(), is(initialFetchOffset + 3)); + assertThat("The offset manager should not skip past offset 5 which is still pending", + manager.findNextCommitOffset(COMMIT_METADATA).offset(), is(initialFetchOffset + 3)); manager.addToAckMsgs(getMessageId(initialFetchOffset + 5)); - assertThat("The offset manager should skip past the gap in acked messages, since the messages were not emitted", + assertThat("The offset manager should skip past the gap in acked messages, since the " + + "messages were not emitted", manager.findNextCommitOffset(COMMIT_METADATA), is(new OffsetAndMetadata(initialFetchOffset + 7, COMMIT_METADATA))); } @Test public void testSkipMissingOffsetsWhenFindingNextCommitOffsetWithGapBeforeAcked() { - //0-4 compacted away + // 0-4 compacted away manager.addToEmitMsgs(initialFetchOffset + 5); manager.addToEmitMsgs(initialFetchOffset + 6); manager.addToAckMsgs(getMessageId(initialFetchOffset + 6)); - assertThat("The offset manager should not skip past offset 5 which is still pending", manager.findNextCommitOffset(COMMIT_METADATA), is(nullValue())); + assertThat("The offset manager should not skip past offset 5 which is still pending", + manager.findNextCommitOffset(COMMIT_METADATA), is(nullValue())); manager.addToAckMsgs(getMessageId(initialFetchOffset + 5)); - assertThat("The offset manager should skip past the gap in acked messages, since the messages were not emitted", + assertThat("The offset manager should skip past the gap in acked messages, since the " + + "messages were not emitted", manager.findNextCommitOffset(COMMIT_METADATA), is(new OffsetAndMetadata(initialFetchOffset + 7, COMMIT_METADATA))); } @Test public void testFindNextCommittedOffsetWithNoAcks() { OffsetAndMetadata nextCommitOffset = manager.findNextCommitOffset(COMMIT_METADATA); - assertThat("There shouldn't be a next commit offset when nothing has been acked", nextCommitOffset, is(nullValue())); + assertThat("There shouldn't be a next commit offset when nothing has been acked", + nextCommitOffset, is(nullValue())); } @Test public void testFindNextCommitOffsetWithOneAck() { /* - * The KafkaConsumer commitSync API docs: "The committed offset should be the next message your application will consume, i.e. + * The KafkaConsumer commitSync API docs: "The committed offset should be the next message + * your application will consume, i.e. * lastProcessedMessageOffset + 1. " */ emitAndAckMessage(getMessageId(initialFetchOffset)); OffsetAndMetadata nextCommitOffset = manager.findNextCommitOffset(COMMIT_METADATA); - assertThat("The next commit offset should be one past the processed message offset", nextCommitOffset.offset(), is(initialFetchOffset + 1)); + assertThat("The next commit offset should be one past the processed message offset", + nextCommitOffset.offset(), is(initialFetchOffset + 1)); } @Test @@ -100,7 +110,8 @@ public void testFindNextCommitOffsetWithMultipleOutOfOrderAcks() { emitAndAckMessage(getMessageId(initialFetchOffset + 1)); emitAndAckMessage(getMessageId(initialFetchOffset)); OffsetAndMetadata nextCommitOffset = manager.findNextCommitOffset(COMMIT_METADATA); - assertThat("The next commit offset should be one past the processed message offset", nextCommitOffset.offset(), is(initialFetchOffset + 2)); + assertThat("The next commit offset should be one past the processed message offset", + nextCommitOffset.offset(), is(initialFetchOffset + 2)); } @Test @@ -109,21 +120,24 @@ public void testFindNextCommitOffsetWithAckedOffsetGap() { manager.addToEmitMsgs(initialFetchOffset + 1); emitAndAckMessage(getMessageId(initialFetchOffset)); OffsetAndMetadata nextCommitOffset = manager.findNextCommitOffset(COMMIT_METADATA); - assertThat("The next commit offset should cover the sequential acked offsets", nextCommitOffset.offset(), is(initialFetchOffset + 1)); + assertThat("The next commit offset should cover the sequential acked offsets", + nextCommitOffset.offset(), is(initialFetchOffset + 1)); } @Test public void testFindNextOffsetWithAckedButNotEmittedOffsetGap() { - /** + /* * If topic compaction is enabled in Kafka some offsets may be deleted. - * We distinguish this case from regular gaps in the acked offset sequence caused by out of order acking + * We distinguish this case from regular gaps in the acked offset sequence caused by out of + * order acking * by checking that offsets in the gap have been emitted at some point previously. * If they haven't then they can't exist in Kafka, since the spout emits tuples in order. */ emitAndAckMessage(getMessageId(initialFetchOffset + 2)); emitAndAckMessage(getMessageId(initialFetchOffset)); OffsetAndMetadata nextCommitOffset = manager.findNextCommitOffset(COMMIT_METADATA); - assertThat("The next commit offset should cover all the acked offsets, since the offset in the gap hasn't been emitted and doesn't exist", + assertThat("The next commit offset should cover all the acked offsets, since the offset " + + "in the gap hasn't been emitted and doesn't exist", nextCommitOffset.offset(), is(initialFetchOffset + 3)); } @@ -132,15 +146,18 @@ public void testFindNextCommitOffsetWithUnackedOffsetGap() { manager.addToEmitMsgs(initialFetchOffset + 1); emitAndAckMessage(getMessageId(initialFetchOffset)); OffsetAndMetadata nextCommitOffset = manager.findNextCommitOffset(COMMIT_METADATA); - assertThat("The next commit offset should cover the contiguously acked offsets", nextCommitOffset.offset(), is(initialFetchOffset + 1)); + assertThat("The next commit offset should cover the contiguously acked offsets", + nextCommitOffset.offset(), is(initialFetchOffset + 1)); } @Test public void testFindNextCommitOffsetWhenTooLowOffsetIsAcked() { OffsetManager startAtHighOffsetManager = new OffsetManager(testTp, 10); emitAndAckMessage(getMessageId(0)); - OffsetAndMetadata nextCommitOffset = startAtHighOffsetManager.findNextCommitOffset(COMMIT_METADATA); - assertThat("Acking an offset earlier than the committed offset should have no effect", nextCommitOffset, is(nullValue())); + OffsetAndMetadata nextCommitOffset = startAtHighOffsetManager + .findNextCommitOffset(COMMIT_METADATA); + assertThat("Acking an offset earlier than the committed offset should have no effect", + nextCommitOffset, is(nullValue())); } @Test @@ -151,13 +168,20 @@ public void testCommit() { long committedMessages = manager.commit(new OffsetAndMetadata(initialFetchOffset + 2)); - assertThat("Should have committed all messages to the left of the earliest uncommitted offset", committedMessages, is(2L)); - assertThat("The committed messages should not be in the acked list anymore", manager.contains(getMessageId(initialFetchOffset)), is(false)); - assertThat("The committed messages should not be in the emitted list anymore", manager.containsEmitted(initialFetchOffset), is(false)); - assertThat("The committed messages should not be in the acked list anymore", manager.contains(getMessageId(initialFetchOffset + 1)), is(false)); - assertThat("The committed messages should not be in the emitted list anymore", manager.containsEmitted(initialFetchOffset + 1), is(false)); - assertThat("The uncommitted message should still be in the acked list", manager.contains(getMessageId(initialFetchOffset + 2)), is(true)); - assertThat("The uncommitted message should still be in the emitted list", manager.containsEmitted(initialFetchOffset + 2), is(true)); + assertThat("Should have committed all messages to the left of the earliest uncommitted " + + "offset", committedMessages, is(2L)); + assertThat("The committed messages should not be in the acked list anymore", manager + .contains(getMessageId(initialFetchOffset)), is(false)); + assertThat("The committed messages should not be in the emitted list anymore", manager + .containsEmitted(initialFetchOffset), is(false)); + assertThat("The committed messages should not be in the acked list anymore", manager + .contains(getMessageId(initialFetchOffset + 1)), is(false)); + assertThat("The committed messages should not be in the emitted list anymore", manager + .containsEmitted(initialFetchOffset + 1), is(false)); + assertThat("The uncommitted message should still be in the acked list", manager + .contains(getMessageId(initialFetchOffset + 2)), is(true)); + assertThat("The uncommitted message should still be in the emitted list", manager + .containsEmitted(initialFetchOffset + 2), is(true)); } private KafkaSpoutMessageId getMessageId(long offset) { @@ -176,10 +200,13 @@ public void testGetNthUncommittedOffsetAfterCommittedOffset() { manager.addToEmitMsgs(initialFetchOffset + 5); manager.addToEmitMsgs(initialFetchOffset + 30); - assertThat("The third uncommitted offset should be 5", manager.getNthUncommittedOffsetAfterCommittedOffset(3), is(initialFetchOffset + 5L)); - assertThat("The fourth uncommitted offset should be 30", manager.getNthUncommittedOffsetAfterCommittedOffset(4), is(initialFetchOffset + 30L)); + assertThat("The third uncommitted offset should be 5", manager + .getNthUncommittedOffsetAfterCommittedOffset(3), is(initialFetchOffset + 5L)); + assertThat("The fourth uncommitted offset should be 30", manager + .getNthUncommittedOffsetAfterCommittedOffset(4), is(initialFetchOffset + 30L)); - Assertions.assertThrows(NoSuchElementException.class, () -> manager.getNthUncommittedOffsetAfterCommittedOffset(5)); + Assertions.assertThrows(NoSuchElementException.class, () -> manager + .getNthUncommittedOffsetAfterCommittedOffset(5)); } @Test diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/metric2/KafkaOffsetPartitionMetricsTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/metric2/KafkaOffsetPartitionMetricsTest.java index ee085858e86..a9facbbff44 100644 --- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/metric2/KafkaOffsetPartitionMetricsTest.java +++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/metric2/KafkaOffsetPartitionMetricsTest.java @@ -18,9 +18,11 @@ package org.apache.storm.kafka.spout.metric2; - import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.Mockito.*; +import static org.mockito.Mockito.anyMap; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.when; import com.codahale.metrics.Gauge; import com.codahale.metrics.Metric; @@ -61,10 +63,13 @@ public void registerMetricsGetSpoutLagAndPartitionRecords() throws ExecutionExce TopicPartition topicAPartition1 = new TopicPartition("topicA", 1); - ListOffsetsResult.ListOffsetsResultInfo topicAPartition1LatestListOffsetsResultInfo = new ListOffsetsResult.ListOffsetsResultInfo(100, System.currentTimeMillis(), Optional.empty()); + ListOffsetsResult.ListOffsetsResultInfo topicAPartition1LatestListOffsetsResultInfo = + new ListOffsetsResult.ListOffsetsResultInfo(100, System.currentTimeMillis(), + Optional.empty()); Map topicPartitionLatestListOffsetsResultInfoMap = new HashMap<>(); - topicPartitionLatestListOffsetsResultInfoMap.put(topicAPartition1, topicAPartition1LatestListOffsetsResultInfo); + topicPartitionLatestListOffsetsResultInfoMap.put(topicAPartition1, + topicAPartition1LatestListOffsetsResultInfo); when(kafkaFuture.get()).thenReturn(topicPartitionLatestListOffsetsResultInfoMap); @@ -81,21 +86,28 @@ public void registerMetricsGetSpoutLagAndPartitionRecords() throws ExecutionExce offsetManagers = new HashMap<>(); offsetManagers.put(topicAPartition1, offsetManagertopicAPartition1); - KafkaOffsetPartitionMetrics kafkaOffsetPartitionAndTopicMetrics = new KafkaOffsetPartitionMetrics(() -> Collections.unmodifiableMap(offsetManagers), () -> admin, topicAPartition1); + KafkaOffsetPartitionMetrics kafkaOffsetPartitionAndTopicMetrics = + new KafkaOffsetPartitionMetrics(() -> Collections.unmodifiableMap(offsetManagers), + () -> admin, topicAPartition1); Map result = kafkaOffsetPartitionAndTopicMetrics.getMetrics(); Gauge g1 = (Gauge) result.get("topicA/partition_1/spoutLag"); assertEquals(10L, g1.getValue()); - //get partition records + // get partition records - ListOffsetsResult.ListOffsetsResultInfo topicAPartition1EarliestListOffsetsResultInfo = new ListOffsetsResult.ListOffsetsResultInfo(1, System.currentTimeMillis(), Optional.empty()); + ListOffsetsResult.ListOffsetsResultInfo topicAPartition1EarliestListOffsetsResultInfo = + new ListOffsetsResult.ListOffsetsResultInfo(1, System.currentTimeMillis(), Optional + .empty()); Map topicPartitionEarliestListOffsetsResultInfoMap = new HashMap<>(); - topicPartitionEarliestListOffsetsResultInfoMap.put(topicAPartition1, topicAPartition1EarliestListOffsetsResultInfo); + topicPartitionEarliestListOffsetsResultInfoMap.put(topicAPartition1, + topicAPartition1EarliestListOffsetsResultInfo); - //mock consecutive calls. Each call to the recordsInPartition gauge will call kafkaFuture.get() twice - when(kafkaFuture.get()).thenReturn(topicPartitionLatestListOffsetsResultInfoMap, topicPartitionEarliestListOffsetsResultInfoMap); + // mock consecutive calls. Each call to the recordsInPartition gauge will call + // kafkaFuture.get() twice + when(kafkaFuture.get()).thenReturn(topicPartitionLatestListOffsetsResultInfoMap, + topicPartitionEarliestListOffsetsResultInfoMap); result = kafkaOffsetPartitionAndTopicMetrics.getMetrics(); g1 = (Gauge) result.get("topicA/partition_1/recordsInPartition"); @@ -107,10 +119,13 @@ public void registerMetricsGetEarliestAndLatest() throws ExecutionException, Int TopicPartition topicAPartition1 = new TopicPartition("topicA", 1); - ListOffsetsResult.ListOffsetsResultInfo topicAPartition1EarliestListOffsetsResultInfo = new ListOffsetsResult.ListOffsetsResultInfo(1, System.currentTimeMillis(), Optional.empty()); + ListOffsetsResult.ListOffsetsResultInfo topicAPartition1EarliestListOffsetsResultInfo = + new ListOffsetsResult.ListOffsetsResultInfo(1, System.currentTimeMillis(), Optional + .empty()); Map topicPartitionEarliestListOffsetsResultInfoMap = new HashMap<>(); - topicPartitionEarliestListOffsetsResultInfoMap.put(topicAPartition1, topicAPartition1EarliestListOffsetsResultInfo); + topicPartitionEarliestListOffsetsResultInfoMap.put(topicAPartition1, + topicAPartition1EarliestListOffsetsResultInfo); when(kafkaFuture.get()).thenReturn(topicPartitionEarliestListOffsetsResultInfoMap); @@ -131,19 +146,24 @@ public void registerMetricsGetEarliestAndLatest() throws ExecutionException, Int assignment = new HashSet<>(); assignment.add(topicAPartition1); - KafkaOffsetPartitionMetrics kafkaOffsetPartitionAndTopicMetrics = new KafkaOffsetPartitionMetrics(() -> Collections.unmodifiableMap(offsetManagers), () -> admin, topicAPartition1); + KafkaOffsetPartitionMetrics kafkaOffsetPartitionAndTopicMetrics = + new KafkaOffsetPartitionMetrics(() -> Collections.unmodifiableMap(offsetManagers), + () -> admin, topicAPartition1); Map result = kafkaOffsetPartitionAndTopicMetrics.getMetrics(); Gauge g1 = (Gauge) result.get("topicA/partition_1/earliestTimeOffset"); assertEquals(g1.getValue(), 1L); - //get the latest offsets + // get the latest offsets - ListOffsetsResult.ListOffsetsResultInfo topicAPartition1LatestListOffsetsResultInfo = new ListOffsetsResult.ListOffsetsResultInfo(100, System.currentTimeMillis(), Optional.empty()); + ListOffsetsResult.ListOffsetsResultInfo topicAPartition1LatestListOffsetsResultInfo = + new ListOffsetsResult.ListOffsetsResultInfo(100, System.currentTimeMillis(), + Optional.empty()); Map topicPartitionLatestListOffsetsResultInfoMap = new HashMap<>(); - topicPartitionLatestListOffsetsResultInfoMap.put(topicAPartition1, topicAPartition1LatestListOffsetsResultInfo); + topicPartitionLatestListOffsetsResultInfoMap.put(topicAPartition1, + topicAPartition1LatestListOffsetsResultInfo); when(kafkaFuture.get()).thenReturn(topicPartitionLatestListOffsetsResultInfoMap); diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/metric2/KafkaOffsetTopicMetricsTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/metric2/KafkaOffsetTopicMetricsTest.java index 26462503389..5e8a09af176 100644 --- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/metric2/KafkaOffsetTopicMetricsTest.java +++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/metric2/KafkaOffsetTopicMetricsTest.java @@ -6,10 +6,10 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - *

    - * http://www.apache.org/licenses/LICENSE-2.0 - *

    - * Unless required by applicable law or agreed to in writing, software + * + *

    http://www.apache.org/licenses/LICENSE-2.0 + * + *

    Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -18,8 +18,22 @@ package org.apache.storm.kafka.spout.metric2; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.mockito.ArgumentMatchers.anyMap; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.when; + import com.codahale.metrics.Gauge; import com.codahale.metrics.Metric; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ExecutionException; import org.apache.kafka.clients.admin.Admin; import org.apache.kafka.clients.admin.ListOffsetsResult; import org.apache.kafka.common.KafkaFuture; @@ -31,14 +45,6 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.junit.jupiter.MockitoExtension; -import java.util.*; -import java.util.concurrent.ExecutionException; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.mockito.ArgumentMatchers.anyMap; -import static org.mockito.Mockito.*; - @ExtendWith(MockitoExtension.class) public class KafkaOffsetTopicMetricsTest { @@ -60,16 +66,28 @@ public void registerMetricsGetSpoutLagAndPartitionRecords() throws ExecutionExce TopicPartition topicBPartition1 = new TopicPartition("topicB", 1); TopicPartition topicBPartition2 = new TopicPartition("topicB", 2); - ListOffsetsResult.ListOffsetsResultInfo topicAPartition1LatestListOffsetsResultInfo = new ListOffsetsResult.ListOffsetsResultInfo(100, System.currentTimeMillis(), Optional.empty()); - ListOffsetsResult.ListOffsetsResultInfo topicAPartition2LatestListOffsetsResultInfo = new ListOffsetsResult.ListOffsetsResultInfo(200, System.currentTimeMillis(), Optional.empty()); - ListOffsetsResult.ListOffsetsResultInfo topicBPartition1LatestListOffsetsResultInfo = new ListOffsetsResult.ListOffsetsResultInfo(300, System.currentTimeMillis(), Optional.empty()); - ListOffsetsResult.ListOffsetsResultInfo topicBPartition2LatestListOffsetsResultInfo = new ListOffsetsResult.ListOffsetsResultInfo(400, System.currentTimeMillis(), Optional.empty()); + ListOffsetsResult.ListOffsetsResultInfo topicAPartition1LatestListOffsetsResultInfo = + new ListOffsetsResult.ListOffsetsResultInfo(100, System.currentTimeMillis(), + Optional.empty()); + ListOffsetsResult.ListOffsetsResultInfo topicAPartition2LatestListOffsetsResultInfo = + new ListOffsetsResult.ListOffsetsResultInfo(200, System.currentTimeMillis(), + Optional.empty()); + ListOffsetsResult.ListOffsetsResultInfo topicBPartition1LatestListOffsetsResultInfo = + new ListOffsetsResult.ListOffsetsResultInfo(300, System.currentTimeMillis(), + Optional.empty()); + ListOffsetsResult.ListOffsetsResultInfo topicBPartition2LatestListOffsetsResultInfo = + new ListOffsetsResult.ListOffsetsResultInfo(400, System.currentTimeMillis(), + Optional.empty()); Map topicPartitionLatestListOffsetsResultInfoMap = new HashMap<>(); - topicPartitionLatestListOffsetsResultInfoMap.put(topicAPartition1, topicAPartition1LatestListOffsetsResultInfo); - topicPartitionLatestListOffsetsResultInfoMap.put(topicAPartition2, topicAPartition2LatestListOffsetsResultInfo); - topicPartitionLatestListOffsetsResultInfoMap.put(topicBPartition1, topicBPartition1LatestListOffsetsResultInfo); - topicPartitionLatestListOffsetsResultInfoMap.put(topicBPartition2, topicBPartition2LatestListOffsetsResultInfo); + topicPartitionLatestListOffsetsResultInfoMap.put(topicAPartition1, + topicAPartition1LatestListOffsetsResultInfo); + topicPartitionLatestListOffsetsResultInfoMap.put(topicAPartition2, + topicAPartition2LatestListOffsetsResultInfo); + topicPartitionLatestListOffsetsResultInfoMap.put(topicBPartition1, + topicBPartition1LatestListOffsetsResultInfo); + topicPartitionLatestListOffsetsResultInfoMap.put(topicBPartition2, + topicBPartition2LatestListOffsetsResultInfo); when(kafkaFuture.get()).thenReturn(topicPartitionLatestListOffsetsResultInfoMap); @@ -96,31 +114,45 @@ public void registerMetricsGetSpoutLagAndPartitionRecords() throws ExecutionExce assignment.add(topicBPartition2); - KafkaOffsetTopicMetrics kafkaOffsetTopicMetricsA = new KafkaOffsetTopicMetrics("topicA", () -> Collections.unmodifiableMap(offsetManagers), () -> admin, assignment); + KafkaOffsetTopicMetrics kafkaOffsetTopicMetricsA = new KafkaOffsetTopicMetrics("topicA", + () -> Collections.unmodifiableMap(offsetManagers), () -> admin, assignment); Map result = kafkaOffsetTopicMetricsA.getMetrics(); Gauge g1 = (Gauge) result.get("topicA/totalSpoutLag"); assertEquals(40L, g1.getValue()); - //get again the values from the Gauge. Values cannot change + // get again the values from the Gauge. Values cannot change g1 = (Gauge) result.get("topicA/totalSpoutLag"); assertEquals(40L, g1.getValue()); assertNull(result.get("topicB/totalSpoutLag")); - //get topic records - - ListOffsetsResult.ListOffsetsResultInfo topicAPartition1EarliestListOffsetsResultInfo = new ListOffsetsResult.ListOffsetsResultInfo(1, System.currentTimeMillis(), Optional.empty()); - ListOffsetsResult.ListOffsetsResultInfo topicAPartition2EarliestListOffsetsResultInfo = new ListOffsetsResult.ListOffsetsResultInfo(2, System.currentTimeMillis(), Optional.empty()); - ListOffsetsResult.ListOffsetsResultInfo topicBPartition1EarliestListOffsetsResultInfo = new ListOffsetsResult.ListOffsetsResultInfo(3, System.currentTimeMillis(), Optional.empty()); - ListOffsetsResult.ListOffsetsResultInfo topicBPartition2EarliestListOffsetsResultInfo = new ListOffsetsResult.ListOffsetsResultInfo(4, System.currentTimeMillis(), Optional.empty()); + // get topic records + + ListOffsetsResult.ListOffsetsResultInfo topicAPartition1EarliestListOffsetsResultInfo = + new ListOffsetsResult.ListOffsetsResultInfo(1, System.currentTimeMillis(), Optional + .empty()); + ListOffsetsResult.ListOffsetsResultInfo topicAPartition2EarliestListOffsetsResultInfo = + new ListOffsetsResult.ListOffsetsResultInfo(2, System.currentTimeMillis(), Optional + .empty()); + ListOffsetsResult.ListOffsetsResultInfo topicBPartition1EarliestListOffsetsResultInfo = + new ListOffsetsResult.ListOffsetsResultInfo(3, System.currentTimeMillis(), Optional + .empty()); + ListOffsetsResult.ListOffsetsResultInfo topicBPartition2EarliestListOffsetsResultInfo = + new ListOffsetsResult.ListOffsetsResultInfo(4, System.currentTimeMillis(), Optional + .empty()); Map topicPartitionEarliestListOffsetsResultInfoMap = new HashMap<>(); - topicPartitionEarliestListOffsetsResultInfoMap.put(topicAPartition1, topicAPartition1EarliestListOffsetsResultInfo); - topicPartitionEarliestListOffsetsResultInfoMap.put(topicAPartition2, topicAPartition2EarliestListOffsetsResultInfo); - topicPartitionEarliestListOffsetsResultInfoMap.put(topicBPartition1, topicBPartition1EarliestListOffsetsResultInfo); - topicPartitionEarliestListOffsetsResultInfoMap.put(topicBPartition2, topicBPartition2EarliestListOffsetsResultInfo); - - //mock consecutive calls. Each call to the recordsInPartition gauge will call kafkaFuture.get() twice + topicPartitionEarliestListOffsetsResultInfoMap.put(topicAPartition1, + topicAPartition1EarliestListOffsetsResultInfo); + topicPartitionEarliestListOffsetsResultInfoMap.put(topicAPartition2, + topicAPartition2EarliestListOffsetsResultInfo); + topicPartitionEarliestListOffsetsResultInfoMap.put(topicBPartition1, + topicBPartition1EarliestListOffsetsResultInfo); + topicPartitionEarliestListOffsetsResultInfoMap.put(topicBPartition2, + topicBPartition2EarliestListOffsetsResultInfo); + + // mock consecutive calls. Each call to the recordsInPartition gauge will call + // kafkaFuture.get() twice when(kafkaFuture.get()).thenReturn( topicPartitionLatestListOffsetsResultInfoMap, topicPartitionEarliestListOffsetsResultInfoMap, topicPartitionLatestListOffsetsResultInfoMap, topicPartitionEarliestListOffsetsResultInfoMap, @@ -130,8 +162,8 @@ public void registerMetricsGetSpoutLagAndPartitionRecords() throws ExecutionExce Gauge gATotal = (Gauge) result.get("topicA/totalRecordsInPartitions"); assertEquals(297L, gATotal.getValue()); - //get again the values from the Gauge. Values cannot change - gATotal = (Gauge) result.get("topicA/totalRecordsInPartitions"); + // get again the values from the Gauge. Values cannot change + gATotal = (Gauge) result.get("topicA/totalRecordsInPartitions"); assertEquals(297L, gATotal.getValue()); assertNull(result.get("topicB/totalRecordsInPartitions")); @@ -145,16 +177,28 @@ public void registerMetricsGetEarliestAndLatest() throws ExecutionException, Int TopicPartition topicBPartition1 = new TopicPartition("topicB", 1); TopicPartition topicBPartition2 = new TopicPartition("topicB", 2); - ListOffsetsResult.ListOffsetsResultInfo topicAPartition1EarliestListOffsetsResultInfo = new ListOffsetsResult.ListOffsetsResultInfo(1, System.currentTimeMillis(), Optional.empty()); - ListOffsetsResult.ListOffsetsResultInfo topicAPartition2EarliestListOffsetsResultInfo = new ListOffsetsResult.ListOffsetsResultInfo(1, System.currentTimeMillis(), Optional.empty()); - ListOffsetsResult.ListOffsetsResultInfo topicBPartition1EarliestListOffsetsResultInfo = new ListOffsetsResult.ListOffsetsResultInfo(1, System.currentTimeMillis(), Optional.empty()); - ListOffsetsResult.ListOffsetsResultInfo topicBPartition2EarliestListOffsetsResultInfo = new ListOffsetsResult.ListOffsetsResultInfo(1, System.currentTimeMillis(), Optional.empty()); + ListOffsetsResult.ListOffsetsResultInfo topicAPartition1EarliestListOffsetsResultInfo = + new ListOffsetsResult.ListOffsetsResultInfo(1, System.currentTimeMillis(), Optional + .empty()); + ListOffsetsResult.ListOffsetsResultInfo topicAPartition2EarliestListOffsetsResultInfo = + new ListOffsetsResult.ListOffsetsResultInfo(1, System.currentTimeMillis(), Optional + .empty()); + ListOffsetsResult.ListOffsetsResultInfo topicBPartition1EarliestListOffsetsResultInfo = + new ListOffsetsResult.ListOffsetsResultInfo(1, System.currentTimeMillis(), Optional + .empty()); + ListOffsetsResult.ListOffsetsResultInfo topicBPartition2EarliestListOffsetsResultInfo = + new ListOffsetsResult.ListOffsetsResultInfo(1, System.currentTimeMillis(), Optional + .empty()); Map topicPartitionEarliestListOffsetsResultInfoMap = new HashMap<>(); - topicPartitionEarliestListOffsetsResultInfoMap.put(topicAPartition1, topicAPartition1EarliestListOffsetsResultInfo); - topicPartitionEarliestListOffsetsResultInfoMap.put(topicAPartition2, topicAPartition2EarliestListOffsetsResultInfo); - topicPartitionEarliestListOffsetsResultInfoMap.put(topicBPartition1, topicBPartition1EarliestListOffsetsResultInfo); - topicPartitionEarliestListOffsetsResultInfoMap.put(topicBPartition2, topicBPartition2EarliestListOffsetsResultInfo); + topicPartitionEarliestListOffsetsResultInfoMap.put(topicAPartition1, + topicAPartition1EarliestListOffsetsResultInfo); + topicPartitionEarliestListOffsetsResultInfoMap.put(topicAPartition2, + topicAPartition2EarliestListOffsetsResultInfo); + topicPartitionEarliestListOffsetsResultInfoMap.put(topicBPartition1, + topicBPartition1EarliestListOffsetsResultInfo); + topicPartitionEarliestListOffsetsResultInfoMap.put(topicBPartition2, + topicBPartition2EarliestListOffsetsResultInfo); when(kafkaFuture.get()).thenReturn(topicPartitionEarliestListOffsetsResultInfoMap); @@ -183,31 +227,45 @@ public void registerMetricsGetEarliestAndLatest() throws ExecutionException, Int assignment.add(topicBPartition1); assignment.add(topicBPartition2); - KafkaOffsetTopicMetrics kafkaOffsetPartitionAndTopicMetrics = new KafkaOffsetTopicMetrics("topicA",() -> Collections.unmodifiableMap(offsetManagers), () -> admin, assignment); + KafkaOffsetTopicMetrics kafkaOffsetPartitionAndTopicMetrics = + new KafkaOffsetTopicMetrics("topicA", () -> Collections + .unmodifiableMap(offsetManagers), () -> admin, assignment); Map result = kafkaOffsetPartitionAndTopicMetrics.getMetrics(); Gauge gATotal = (Gauge) result.get("topicA/totalEarliestTimeOffset"); assertEquals(2L, gATotal.getValue()); assertNull(result.get("topicB/totalEarliestTimeOffset")); - //get the metrics a second time. Values should be the same + // get the metrics a second time. Values should be the same gATotal = (Gauge) result.get("topicA/totalEarliestTimeOffset"); assertEquals(2L, gATotal.getValue()); assertNull(result.get("topicB/totalEarliestTimeOffset")); - //get the latest offsets - - ListOffsetsResult.ListOffsetsResultInfo topicAPartition1LatestListOffsetsResultInfo = new ListOffsetsResult.ListOffsetsResultInfo(100, System.currentTimeMillis(), Optional.empty()); - ListOffsetsResult.ListOffsetsResultInfo topicAPartition2LatestListOffsetsResultInfo = new ListOffsetsResult.ListOffsetsResultInfo(200, System.currentTimeMillis(), Optional.empty()); - ListOffsetsResult.ListOffsetsResultInfo topicBPartition1LatestListOffsetsResultInfo = new ListOffsetsResult.ListOffsetsResultInfo(300, System.currentTimeMillis(), Optional.empty()); - ListOffsetsResult.ListOffsetsResultInfo topicBPartition2LatestListOffsetsResultInfo = new ListOffsetsResult.ListOffsetsResultInfo(400, System.currentTimeMillis(), Optional.empty()); + // get the latest offsets + + ListOffsetsResult.ListOffsetsResultInfo topicAPartition1LatestListOffsetsResultInfo = + new ListOffsetsResult.ListOffsetsResultInfo(100, System.currentTimeMillis(), + Optional.empty()); + ListOffsetsResult.ListOffsetsResultInfo topicAPartition2LatestListOffsetsResultInfo = + new ListOffsetsResult.ListOffsetsResultInfo(200, System.currentTimeMillis(), + Optional.empty()); + ListOffsetsResult.ListOffsetsResultInfo topicBPartition1LatestListOffsetsResultInfo = + new ListOffsetsResult.ListOffsetsResultInfo(300, System.currentTimeMillis(), + Optional.empty()); + ListOffsetsResult.ListOffsetsResultInfo topicBPartition2LatestListOffsetsResultInfo = + new ListOffsetsResult.ListOffsetsResultInfo(400, System.currentTimeMillis(), + Optional.empty()); Map topicPartitionLatestListOffsetsResultInfoMap = new HashMap<>(); - topicPartitionLatestListOffsetsResultInfoMap.put(topicAPartition1, topicAPartition1LatestListOffsetsResultInfo); - topicPartitionLatestListOffsetsResultInfoMap.put(topicAPartition2, topicAPartition2LatestListOffsetsResultInfo); - topicPartitionLatestListOffsetsResultInfoMap.put(topicBPartition1, topicBPartition1LatestListOffsetsResultInfo); - topicPartitionLatestListOffsetsResultInfoMap.put(topicBPartition2, topicBPartition2LatestListOffsetsResultInfo); + topicPartitionLatestListOffsetsResultInfoMap.put(topicAPartition1, + topicAPartition1LatestListOffsetsResultInfo); + topicPartitionLatestListOffsetsResultInfoMap.put(topicAPartition2, + topicAPartition2LatestListOffsetsResultInfo); + topicPartitionLatestListOffsetsResultInfoMap.put(topicBPartition1, + topicBPartition1LatestListOffsetsResultInfo); + topicPartitionLatestListOffsetsResultInfoMap.put(topicBPartition2, + topicBPartition2LatestListOffsetsResultInfo); when(kafkaFuture.get()).thenReturn(topicPartitionLatestListOffsetsResultInfoMap); @@ -217,14 +275,14 @@ public void registerMetricsGetEarliestAndLatest() throws ExecutionException, Int gATotal = (Gauge) result.get("topicA/totalLatestEmittedOffset"); assertEquals(150L, gATotal.getValue()); - assertNull( result.get("topicB/totalLatestEmittedOffset")); + assertNull(result.get("topicB/totalLatestEmittedOffset")); gATotal = (Gauge) result.get("topicA/totalLatestCompletedOffset"); assertEquals(130L, gATotal.getValue()); assertNull(result.get("topiBA/totalLatestCompletedOffset")); - //get the metrics a second time. Values should be the same + // get the metrics a second time. Values should be the same gATotal = (Gauge) result.get("topicA/totalLatestTimeOffset"); assertEquals(300L, gATotal.getValue()); @@ -232,7 +290,7 @@ public void registerMetricsGetEarliestAndLatest() throws ExecutionException, Int gATotal = (Gauge) result.get("topicA/totalLatestEmittedOffset"); assertEquals(150L, gATotal.getValue()); - assertNull( result.get("topicB/totalLatestEmittedOffset")); + assertNull(result.get("topicB/totalLatestEmittedOffset")); gATotal = (Gauge) result.get("topicA/totalLatestCompletedOffset"); diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/subscription/NamedTopicFilterTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/subscription/NamedTopicFilterTest.java index 47184a3396b..5c42710ad2e 100644 --- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/subscription/NamedTopicFilterTest.java +++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/subscription/NamedTopicFilterTest.java @@ -49,17 +49,20 @@ public void testFilter() { NamedTopicFilter filter = new NamedTopicFilter(matchingTopicOne, matchingTopicTwo); - when(consumerMock.partitionsFor(matchingTopicOne)).thenReturn(Collections.singletonList(createPartitionInfo(matchingTopicOne, 0))); + when(consumerMock.partitionsFor(matchingTopicOne)).thenReturn(Collections + .singletonList(createPartitionInfo(matchingTopicOne, 0))); List partitionTwoPartitions = new ArrayList<>(); partitionTwoPartitions.add(createPartitionInfo(matchingTopicTwo, 0)); partitionTwoPartitions.add(createPartitionInfo(matchingTopicTwo, 1)); when(consumerMock.partitionsFor(matchingTopicTwo)).thenReturn(partitionTwoPartitions); - when(consumerMock.partitionsFor(unmatchedTopic)).thenReturn(Collections.singletonList(createPartitionInfo(unmatchedTopic, 0))); + when(consumerMock.partitionsFor(unmatchedTopic)).thenReturn(Collections + .singletonList(createPartitionInfo(unmatchedTopic, 0))); Set matchedPartitions = filter.getAllSubscribedPartitions(consumerMock); assertThat("Expected filter to pass only topics with exact name matches", matchedPartitions, - containsInAnyOrder(new TopicPartition(matchingTopicOne, 0), new TopicPartition(matchingTopicTwo, 0), new TopicPartition(matchingTopicTwo, 1))); + containsInAnyOrder(new TopicPartition(matchingTopicOne, 0), + new TopicPartition(matchingTopicTwo, 0), new TopicPartition(matchingTopicTwo, 1))); } @@ -69,7 +72,8 @@ public void testFilterOnAbsentTopic() { String absentTopic = "absent"; NamedTopicFilter filter = new NamedTopicFilter(presentTopic, absentTopic); - when(consumerMock.partitionsFor(presentTopic)).thenReturn(Collections.singletonList(createPartitionInfo(presentTopic, 2))); + when(consumerMock.partitionsFor(presentTopic)).thenReturn(Collections + .singletonList(createPartitionInfo(presentTopic, 2))); when(consumerMock.partitionsFor(absentTopic)).thenReturn(null); Set presentPartitions = filter.getAllSubscribedPartitions(consumerMock); diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/subscription/PatternTopicFilterTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/subscription/PatternTopicFilterTest.java index 8ac544a737d..9c40dc58d26 100644 --- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/subscription/PatternTopicFilterTest.java +++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/subscription/PatternTopicFilterTest.java @@ -39,7 +39,7 @@ public class PatternTopicFilterTest { private KafkaConsumer consumerMock; @BeforeEach - public void setUp(){ + public void setUp() { consumerMock = mock(KafkaConsumer.class); } @@ -53,19 +53,23 @@ public void testFilter() { String unmatchedTopic = "unmatched"; Map> allTopics = new HashMap<>(); - allTopics.put(matchingTopicOne, Collections.singletonList(createPartitionInfo(matchingTopicOne, 0))); + allTopics.put(matchingTopicOne, Collections + .singletonList(createPartitionInfo(matchingTopicOne, 0))); List testTwoPartitions = new ArrayList<>(); testTwoPartitions.add(createPartitionInfo(matchingTopicTwo, 0)); testTwoPartitions.add(createPartitionInfo(matchingTopicTwo, 1)); allTopics.put(matchingTopicTwo, testTwoPartitions); - allTopics.put(unmatchedTopic, Collections.singletonList(createPartitionInfo(unmatchedTopic, 0))); + allTopics.put(unmatchedTopic, Collections.singletonList(createPartitionInfo(unmatchedTopic, + 0))); when(consumerMock.listTopics()).thenReturn(allTopics); Set matchedPartitions = filter.getAllSubscribedPartitions(consumerMock); - assertThat("Expected topic partitions matching the pattern to be passed by the filter", matchedPartitions, - containsInAnyOrder(new TopicPartition(matchingTopicOne, 0), new TopicPartition(matchingTopicTwo, 0), new TopicPartition(matchingTopicTwo, 1))); + assertThat("Expected topic partitions matching the pattern to be passed by the filter", + matchedPartitions, + containsInAnyOrder(new TopicPartition(matchingTopicOne, 0), + new TopicPartition(matchingTopicTwo, 0), new TopicPartition(matchingTopicTwo, 1))); } private PartitionInfo createPartitionInfo(String topic, int partition) { diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/subscription/RoundRobinManualPartitionerTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/subscription/RoundRobinManualPartitionerTest.java index c74daf29fd4..d3db0a8e79c 100644 --- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/subscription/RoundRobinManualPartitionerTest.java +++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/subscription/RoundRobinManualPartitionerTest.java @@ -48,13 +48,13 @@ private Set partitionsToTps(int[] expectedPartitions) { @Test public void testRoundRobinPartitioning() { List allPartitions = new ArrayList<>(); - for(int i = 0; i < 11; i++) { + for (int i = 0; i < 11; i++) { allPartitions.add(createTp(i)); } List contextMocks = new ArrayList<>(); String thisComponentId = "A spout"; List allTasks = Arrays.asList(0, 1, 2); - for(int i = 0; i < 3; i++) { + for (int i = 0; i < 3; i++) { TopologyContext contextMock = mock(TopologyContext.class); when(contextMock.getThisTaskIndex()).thenReturn(i); when(contextMock.getThisComponentId()).thenReturn(thisComponentId); @@ -63,13 +63,16 @@ public void testRoundRobinPartitioning() { } RoundRobinManualPartitioner partitioner = new RoundRobinManualPartitioner(); - Set partitionsForFirstTask = partitioner.getPartitionsForThisTask(allPartitions, contextMocks.get(0)); + Set partitionsForFirstTask = partitioner + .getPartitionsForThisTask(allPartitions, contextMocks.get(0)); assertThat(partitionsForFirstTask, is(partitionsToTps(new int[]{0, 3, 6, 9}))); - Set partitionsForSecondTask = partitioner.getPartitionsForThisTask(allPartitions, contextMocks.get(1)); + Set partitionsForSecondTask = partitioner + .getPartitionsForThisTask(allPartitions, contextMocks.get(1)); assertThat(partitionsForSecondTask, is(partitionsToTps(new int[]{1, 4, 7, 10}))); - Set partitionsForThirdTask = partitioner.getPartitionsForThisTask(allPartitions, contextMocks.get(2)); + Set partitionsForThirdTask = partitioner + .getPartitionsForThisTask(allPartitions, contextMocks.get(2)); assertThat(partitionsForThirdTask, is(partitionsToTps(new int[]{2, 5, 8}))); } diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/subscription/TopicAssignerTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/subscription/TopicAssignerTest.java index d955c491313..d08674173ba 100644 --- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/subscription/TopicAssignerTest.java +++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/subscription/TopicAssignerTest.java @@ -35,7 +35,8 @@ public class TopicAssignerTest { @Test public void testCanReassignPartitions() { - Set onePartition = Collections.singleton(new TopicPartition(SingleTopicKafkaSpoutConfiguration.TOPIC, 0)); + Set onePartition = Collections + .singleton(new TopicPartition(SingleTopicKafkaSpoutConfiguration.TOPIC, 0)); Set twoPartitions = new HashSet<>(); twoPartitions.add(new TopicPartition(SingleTopicKafkaSpoutConfiguration.TOPIC, 0)); twoPartitions.add(new TopicPartition(SingleTopicKafkaSpoutConfiguration.TOPIC, 1)); @@ -43,7 +44,7 @@ public void testCanReassignPartitions() { ConsumerRebalanceListener listenerMock = mock(ConsumerRebalanceListener.class); TopicAssigner assigner = new TopicAssigner(); - //Set the first assignment + // Set the first assignment assigner.assignPartitions(consumerMock, onePartition, listenerMock); InOrder inOrder = inOrder(consumerMock, listenerMock); @@ -55,11 +56,12 @@ public void testCanReassignPartitions() { when(consumerMock.assignment()).thenReturn(new HashSet<>(onePartition)); - //Update to set the second assignment + // Update to set the second assignment assigner.assignPartitions(consumerMock, twoPartitions, listenerMock); - //The partition revocation hook must be called before the new partitions are assigned to the consumer, - //to allow the revocation hook to commit offsets for the revoked partitions. + // The partition revocation hook must be called before the new partitions are assigned to + // the consumer, + // to allow the revocation hook to commit offsets for the revoked partitions. inOrder.verify(listenerMock).onPartitionsRevoked(new HashSet<>(onePartition)); inOrder.verify(consumerMock).assign(new HashSet<>(twoPartitions)); inOrder.verify(listenerMock).onPartitionsAssigned(new HashSet<>(twoPartitions)); diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/trident/KafkaTridentSpoutBatchMetadataTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/trident/KafkaTridentSpoutBatchMetadataTest.java index fadab02f5e1..72f35bf1d74 100644 --- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/trident/KafkaTridentSpoutBatchMetadataTest.java +++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/trident/KafkaTridentSpoutBatchMetadataTest.java @@ -26,10 +26,13 @@ public class KafkaTridentSpoutBatchMetadataTest { /** - * Tests that the metadata object can be converted to and from a Map. This is needed because Trident metadata is written to - * Zookeeper as JSON with the json-simple library, so the spout converts the metadata to Map before returning it to Trident. + * Tests that the metadata object can be converted to and from a Map. This is needed because + * Trident metadata is written to + * Zookeeper as JSON with the json-simple library, so the spout converts the metadata to Map + * before returning it to Trident. * It is important that all map entries are types json-simple knows about, - * since otherwise the library just calls toString on them which will likely produce invalid JSON. + * since otherwise the library just calls toString on them which will likely produce invalid + * JSON. */ @SuppressWarnings("rawtypes") @Test @@ -38,10 +41,13 @@ public void testMetadataIsRoundTripSerializableWithJsonSimple() throws Exception long endOffset = 20; String topologyId = "topologyId"; - KafkaTridentSpoutBatchMetadata metadata = new KafkaTridentSpoutBatchMetadata(startOffset, endOffset, topologyId); + KafkaTridentSpoutBatchMetadata metadata = new KafkaTridentSpoutBatchMetadata(startOffset, + endOffset, topologyId); Map map = metadata.toMap(); - Map deserializedMap = (Map)JSONValue.parseWithException(JSONValue.toJSONString(map)); - KafkaTridentSpoutBatchMetadata deserializedMetadata = KafkaTridentSpoutBatchMetadata.fromMap(deserializedMap); + Map deserializedMap = (Map) JSONValue.parseWithException(JSONValue + .toJSONString(map)); + KafkaTridentSpoutBatchMetadata deserializedMetadata = KafkaTridentSpoutBatchMetadata + .fromMap(deserializedMap); assertThat(deserializedMetadata.getFirstOffset(), is(metadata.getFirstOffset())); assertThat(deserializedMetadata.getLastOffset(), is(metadata.getLastOffset())); assertThat(deserializedMetadata.getTopologyId(), is(metadata.getTopologyId())); diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/trident/KafkaTridentSpoutEmitterEmitTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/trident/KafkaTridentSpoutEmitterEmitTest.java index 22a53ae4014..7e9c2cb0fe3 100644 --- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/trident/KafkaTridentSpoutEmitterEmitTest.java +++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/trident/KafkaTridentSpoutEmitterEmitTest.java @@ -30,7 +30,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; - import org.apache.kafka.clients.admin.Admin; import org.apache.kafka.clients.admin.MockAdminClient; import org.apache.kafka.clients.consumer.Consumer; @@ -75,9 +74,11 @@ public class KafkaTridentSpoutEmitterEmitTest { @Mock public TridentCollector collectorMock = mock(TridentCollector.class); - private final MockConsumer consumer = new MockConsumer<>(OffsetResetStrategy.NONE); + private final MockConsumer consumer = + new MockConsumer<>(OffsetResetStrategy.NONE); private final MockAdminClient adminClient = new MockAdminClient(); - private final TopicPartition partition = new TopicPartition(SingleTopicKafkaSpoutConfiguration.TOPIC, 0); + private final TopicPartition partition = + new TopicPartition(SingleTopicKafkaSpoutConfiguration.TOPIC, 0); private final String topologyId = "topologyId"; private final long firstOffsetInKafka = 0; private final int recordsInKafka = 100; @@ -89,12 +90,15 @@ public void setUp() { when(topologyContextMock.getStormId()).thenReturn(topologyId); consumer.assign(Collections.singleton(partition)); consumer.updateBeginningOffsets(Collections.singletonMap(partition, firstOffsetInKafka)); - consumer.updateEndOffsets(Collections.singletonMap(partition, firstOffsetInKafka + recordsInKafka)); - List> records = SpoutWithMockedConsumerSetupHelper.createRecords(partition, firstOffsetInKafka, recordsInKafka); + consumer.updateEndOffsets(Collections.singletonMap(partition, firstOffsetInKafka + + recordsInKafka)); + List> records = SpoutWithMockedConsumerSetupHelper + .createRecords(partition, firstOffsetInKafka, recordsInKafka); records.forEach(record -> consumer.addRecord(record)); } - private KafkaTridentSpoutEmitter createEmitter(Consumer kafkaConsumer, Admin adminClient, FirstPollOffsetStrategy firstPollOffsetStrategy) { + private KafkaTridentSpoutEmitter createEmitter(Consumer kafkaConsumer, Admin adminClient, FirstPollOffsetStrategy firstPollOffsetStrategy) { return new KafkaTridentSpoutEmitter<>( SingleTopicKafkaTridentSpoutConfiguration.createKafkaSpoutConfigBuilder(-1) .setRecordTranslator(r -> new Values(r.offset()), new Fields("offset")) @@ -105,7 +109,8 @@ private KafkaTridentSpoutEmitter createEmitter(Consumer() { @Override - public Consumer createConsumer(Map consumerProps) { + public Consumer createConsumer(Map consumerProps) { return kafkaConsumer; } @@ -117,103 +122,142 @@ public Admin createAdmin(Map adminProps) { } private KafkaTridentSpoutEmitter createEmitter(FirstPollOffsetStrategy firstPollOffsetStrategy) { - return createEmitter(consumer,adminClient,firstPollOffsetStrategy); + return createEmitter(consumer, adminClient, firstPollOffsetStrategy); } - private Map doEmitNewBatchTest(FirstPollOffsetStrategy firstPollOffsetStrategy, TridentCollector collectorMock, TopicPartition tp, Map previousBatchMeta) { + private Map doEmitNewBatchTest(FirstPollOffsetStrategy firstPollOffsetStrategy, + TridentCollector collectorMock, TopicPartition tp, Map previousBatchMeta) { KafkaTridentSpoutEmitter emitter = createEmitter(firstPollOffsetStrategy); TransactionAttempt txid = new TransactionAttempt(10L, 0); - return emitBatchNew(emitter,txid,collectorMock,tp, previousBatchMeta); + return emitBatchNew(emitter, txid, collectorMock, tp, previousBatchMeta); } - private Map emitBatchNew(KafkaTridentSpoutEmitter emitter, TransactionAttempt txid, TridentCollector collectorMock, TopicPartition tp, Map previousBatchMeta) { + private Map emitBatchNew(KafkaTridentSpoutEmitter emitter, + TransactionAttempt txid, TridentCollector collectorMock, TopicPartition tp, Map previousBatchMeta) { KafkaTridentSpoutTopicPartition kttp = new KafkaTridentSpoutTopicPartition(tp); - Map> lastBatchMetaMap = new HashMap<>(); + Map> lastBatchMetaMap = + new HashMap<>(); lastBatchMetaMap.put(kttp, previousBatchMeta); - return emitter.emitBatchNew(txid, collectorMock, Collections.singleton(kttp), lastBatchMetaMap).get(kttp); + return emitter.emitBatchNew(txid, collectorMock, Collections.singleton(kttp), + lastBatchMetaMap).get(kttp); } @Test public void testEmitNewBatchWithNullMetaUncommittedEarliest() { - //Check that null meta makes the spout seek to EARLIEST, and that the returned meta is correct - Map batchMeta = doEmitNewBatchTest(FirstPollOffsetStrategy.UNCOMMITTED_EARLIEST, collectorMock, partition, null); + // Check that null meta makes the spout seek to EARLIEST, and that the returned meta is + // correct + Map batchMeta = + doEmitNewBatchTest(FirstPollOffsetStrategy.UNCOMMITTED_EARLIEST, collectorMock, + partition, null); verify(collectorMock, times(recordsInKafka)).emit(emitCaptor.capture()); List> emits = emitCaptor.getAllValues(); assertThat(emits.get(0).get(0), is(firstOffsetInKafka)); assertThat(emits.get(emits.size() - 1).get(0), is(lastOffsetInKafka)); - KafkaTridentSpoutBatchMetadata deserializedMeta = KafkaTridentSpoutBatchMetadata.fromMap(batchMeta); - assertThat("The batch should start at the first offset of the polled records", deserializedMeta.getFirstOffset(), is(firstOffsetInKafka)); - assertThat("The batch should end at the last offset of the polled messages", deserializedMeta.getLastOffset(), is(lastOffsetInKafka)); + KafkaTridentSpoutBatchMetadata deserializedMeta = KafkaTridentSpoutBatchMetadata + .fromMap(batchMeta); + assertThat("The batch should start at the first offset of the polled records", + deserializedMeta.getFirstOffset(), is(firstOffsetInKafka)); + assertThat("The batch should end at the last offset of the polled messages", + deserializedMeta.getLastOffset(), is(lastOffsetInKafka)); } @Test public void testEmitNewBatchWithNullMetaUncommittedLatest() { - //Check that null meta makes the spout seek to LATEST, and that the returned meta is correct - Map batchMeta = doEmitNewBatchTest(FirstPollOffsetStrategy.UNCOMMITTED_LATEST, collectorMock, partition, null); + // Check that null meta makes the spout seek to LATEST, and that the returned meta is + // correct + Map batchMeta = + doEmitNewBatchTest(FirstPollOffsetStrategy.UNCOMMITTED_LATEST, collectorMock, + partition, null); verify(collectorMock, never()).emit(anyList()); - KafkaTridentSpoutBatchMetadata deserializedMeta = KafkaTridentSpoutBatchMetadata.fromMap(batchMeta); - assertThat("The batch should start at the first offset of the polled records", deserializedMeta.getFirstOffset(), is(lastOffsetInKafka)); - assertThat("The batch should end at the last offset of the polled messages", deserializedMeta.getLastOffset(), is(lastOffsetInKafka)); + KafkaTridentSpoutBatchMetadata deserializedMeta = KafkaTridentSpoutBatchMetadata + .fromMap(batchMeta); + assertThat("The batch should start at the first offset of the polled records", + deserializedMeta.getFirstOffset(), is(lastOffsetInKafka)); + assertThat("The batch should end at the last offset of the polled messages", + deserializedMeta.getLastOffset(), is(lastOffsetInKafka)); } @ParameterizedTest @EnumSource(value = FirstPollOffsetStrategy.class, names = {"EARLIEST", "LATEST", "TIMESTAMP"}) public void testEmitNewBatchWithPreviousMeta(FirstPollOffsetStrategy firstPollOffsetStrategy) { - //Check that non-null meta makes the spout seek according to the provided metadata, and that the returned meta is correct + // Check that non-null meta makes the spout seek according to the provided metadata, and + // that the returned meta is correct long firstExpectedEmittedOffset = 50; int expectedEmittedRecords = 50; - KafkaTridentSpoutBatchMetadata previousBatchMeta = new KafkaTridentSpoutBatchMetadata(firstOffsetInKafka, firstExpectedEmittedOffset - 1, topologyId); - Map batchMeta = doEmitNewBatchTest(firstPollOffsetStrategy, collectorMock, partition, previousBatchMeta.toMap()); + KafkaTridentSpoutBatchMetadata previousBatchMeta = + new KafkaTridentSpoutBatchMetadata(firstOffsetInKafka, + firstExpectedEmittedOffset - 1, topologyId); + Map batchMeta = doEmitNewBatchTest(firstPollOffsetStrategy, collectorMock, + partition, previousBatchMeta.toMap()); verify(collectorMock, times(expectedEmittedRecords)).emit(emitCaptor.capture()); List> emits = emitCaptor.getAllValues(); assertThat(emits.get(0).get(0), is(firstExpectedEmittedOffset)); assertThat(emits.get(emits.size() - 1).get(0), is(lastOffsetInKafka)); - KafkaTridentSpoutBatchMetadata deserializedMeta = KafkaTridentSpoutBatchMetadata.fromMap(batchMeta); - assertThat("The batch should start at the first offset of the polled records", deserializedMeta.getFirstOffset(), is(firstExpectedEmittedOffset)); - assertThat("The batch should end at the last offset of the polled messages", deserializedMeta.getLastOffset(), is(lastOffsetInKafka)); + KafkaTridentSpoutBatchMetadata deserializedMeta = KafkaTridentSpoutBatchMetadata + .fromMap(batchMeta); + assertThat("The batch should start at the first offset of the polled records", + deserializedMeta.getFirstOffset(), is(firstExpectedEmittedOffset)); + assertThat("The batch should end at the last offset of the polled messages", + deserializedMeta.getLastOffset(), is(lastOffsetInKafka)); } @Test public void testEmitEmptyBatches() { - //Check that the emitter can handle emitting empty batches on a new partition. - //If the spout is configured to seek to LATEST, or the partition is empty, the initial batches may be empty - KafkaTridentSpoutEmitter emitter = createEmitter(FirstPollOffsetStrategy.LATEST); + // Check that the emitter can handle emitting empty batches on a new partition. + // If the spout is configured to seek to LATEST, or the partition is empty, the initial + // batches may be empty + KafkaTridentSpoutEmitter emitter = + createEmitter(FirstPollOffsetStrategy.LATEST); KafkaTridentSpoutTopicPartition kttp = new KafkaTridentSpoutTopicPartition(partition); Map lastBatchMeta = null; - //Emit 10 empty batches, simulating no new records being present in Kafka + // Emit 10 empty batches, simulating no new records being present in Kafka for (int i = 0; i < 10; i++) { TransactionAttempt txid = new TransactionAttempt((long) i, 0); lastBatchMeta = emitBatchNew(emitter, txid, collectorMock, partition, lastBatchMeta); - KafkaTridentSpoutBatchMetadata deserializedMeta = KafkaTridentSpoutBatchMetadata.fromMap(lastBatchMeta); - assertThat("Since the first poll strategy is LATEST, the meta should indicate that the last message has already been emitted", deserializedMeta.getFirstOffset(), is(lastOffsetInKafka)); - assertThat("Since the first poll strategy is LATEST, the meta should indicate that the last message has already been emitted", deserializedMeta.getLastOffset(), is(lastOffsetInKafka)); + KafkaTridentSpoutBatchMetadata deserializedMeta = KafkaTridentSpoutBatchMetadata + .fromMap(lastBatchMeta); + assertThat("Since the first poll strategy is LATEST, the meta should indicate that " + + "the last message has already been emitted", deserializedMeta + .getFirstOffset(), is(lastOffsetInKafka)); + assertThat("Since the first poll strategy is LATEST, the meta should indicate that " + + "the last message has already been emitted", deserializedMeta + .getLastOffset(), is(lastOffsetInKafka)); } - //Add new records to Kafka, and check that the next batch contains these records + // Add new records to Kafka, and check that the next batch contains these records long firstNewRecordOffset = lastOffsetInKafka + 1; int numNewRecords = 10; - List> newRecords = SpoutWithMockedConsumerSetupHelper.createRecords(partition, firstNewRecordOffset, numNewRecords); + List> newRecords = SpoutWithMockedConsumerSetupHelper + .createRecords(partition, firstNewRecordOffset, numNewRecords); newRecords.forEach(consumer::addRecord); TransactionAttempt txid = new TransactionAttempt(11L, 0); - lastBatchMeta = emitBatchNew(emitter,txid, collectorMock, partition, lastBatchMeta); + lastBatchMeta = emitBatchNew(emitter, txid, collectorMock, partition, lastBatchMeta); verify(collectorMock, times(numNewRecords)).emit(emitCaptor.capture()); List> emits = emitCaptor.getAllValues(); assertThat(emits.get(0).get(0), is(firstNewRecordOffset)); - assertThat(emits.get(emits.size() - 1).get(0), is(firstNewRecordOffset + numNewRecords - 1)); - KafkaTridentSpoutBatchMetadata deserializedMeta = KafkaTridentSpoutBatchMetadata.fromMap(lastBatchMeta); - assertThat("The batch should start at the first offset of the polled records", deserializedMeta.getFirstOffset(), is(firstNewRecordOffset)); - assertThat("The batch should end at the last offset of the polled messages", deserializedMeta.getLastOffset(), is(firstNewRecordOffset + numNewRecords - 1)); + assertThat(emits.get(emits.size() - 1).get(0), is(firstNewRecordOffset + + numNewRecords - 1)); + KafkaTridentSpoutBatchMetadata deserializedMeta = KafkaTridentSpoutBatchMetadata + .fromMap(lastBatchMeta); + assertThat("The batch should start at the first offset of the polled records", + deserializedMeta.getFirstOffset(), is(firstNewRecordOffset)); + assertThat("The batch should end at the last offset of the polled messages", + deserializedMeta.getLastOffset(), is(firstNewRecordOffset + numNewRecords - 1)); } @Test public void testReEmitBatch() { - //Check that a reemit emits exactly the same tuples as the last batch, even if Kafka returns more messages + // Check that a reemit emits exactly the same tuples as the last batch, even if Kafka + // returns more messages long firstEmittedOffset = 50; int numEmittedRecords = 10; - KafkaTridentSpoutBatchMetadata batchMeta = new KafkaTridentSpoutBatchMetadata(firstEmittedOffset, firstEmittedOffset + numEmittedRecords - 1, topologyId); - KafkaTridentSpoutEmitter emitter = createEmitter(FirstPollOffsetStrategy.UNCOMMITTED_EARLIEST); + KafkaTridentSpoutBatchMetadata batchMeta = + new KafkaTridentSpoutBatchMetadata(firstEmittedOffset, firstEmittedOffset + + numEmittedRecords - 1, topologyId); + KafkaTridentSpoutEmitter emitter = + createEmitter(FirstPollOffsetStrategy.UNCOMMITTED_EARLIEST); TransactionAttempt txid = new TransactionAttempt(10L, 0); KafkaTridentSpoutTopicPartition kttp = new KafkaTridentSpoutTopicPartition(partition); emitter.reEmitPartitionBatch(txid, collectorMock, kttp, batchMeta.toMap()); @@ -221,16 +265,22 @@ public void testReEmitBatch() { verify(collectorMock, times(numEmittedRecords)).emit(emitCaptor.capture()); List> emits = emitCaptor.getAllValues(); assertThat(emits.get(0).get(0), is(firstEmittedOffset)); - assertThat(emits.get(emits.size() - 1).get(0), is(firstEmittedOffset + numEmittedRecords - 1)); + assertThat(emits.get(emits.size() - 1).get(0), is(firstEmittedOffset + + numEmittedRecords - 1)); } @Test public void testReEmitBatchForOldTopologyWhenIgnoringCommittedOffsets() { - //In some cases users will want to drop retrying old batches, e.g. if the topology should start over from scratch. - //If the FirstPollOffsetStrategy ignores committed offsets, we should not retry batches for old topologies - //The batch retry should be skipped entirely - KafkaTridentSpoutBatchMetadata batchMeta = new KafkaTridentSpoutBatchMetadata(firstOffsetInKafka, lastOffsetInKafka, "a new storm id"); - KafkaTridentSpoutEmitter emitter = createEmitter(FirstPollOffsetStrategy.EARLIEST); + // In some cases users will want to drop retrying old batches, e.g. if the topology should + // start over from scratch. + // If the FirstPollOffsetStrategy ignores committed offsets, we should not retry batches for + // old topologies + // The batch retry should be skipped entirely + KafkaTridentSpoutBatchMetadata batchMeta = + new KafkaTridentSpoutBatchMetadata(firstOffsetInKafka, lastOffsetInKafka, + "a new storm id"); + KafkaTridentSpoutEmitter emitter = + createEmitter(FirstPollOffsetStrategy.EARLIEST); TransactionAttempt txid = new TransactionAttempt(10L, 0); KafkaTridentSpoutTopicPartition kttp = new KafkaTridentSpoutTopicPartition(partition); emitter.reEmitPartitionBatch(txid, collectorMock, kttp, batchMeta.toMap()); @@ -241,49 +291,63 @@ public void testReEmitBatchForOldTopologyWhenIgnoringCommittedOffsets() { @Test public void testEmitEmptyFirstBatch() { /* - * Check that when the first batch after a redeploy is empty, the emitter does not restart at the pre-redeploy offset. STORM-3279. + * Check that when the first batch after a redeploy is empty, the emitter does not restart + * at the pre-redeploy offset. STORM-3279. */ long firstEmittedOffset = 50; int emittedRecords = 10; - KafkaTridentSpoutBatchMetadata preRedeployLastMeta = new KafkaTridentSpoutBatchMetadata(firstEmittedOffset, firstEmittedOffset + emittedRecords - 1, "an old topology"); - KafkaTridentSpoutEmitter emitter = createEmitter(FirstPollOffsetStrategy.LATEST); + KafkaTridentSpoutBatchMetadata preRedeployLastMeta = + new KafkaTridentSpoutBatchMetadata(firstEmittedOffset, firstEmittedOffset + + emittedRecords - 1, "an old topology"); + KafkaTridentSpoutEmitter emitter = + createEmitter(FirstPollOffsetStrategy.LATEST); TransactionAttempt txid = new TransactionAttempt(0L, 0); - Map meta = emitBatchNew(emitter,txid, collectorMock, partition, preRedeployLastMeta.toMap()); + Map meta = emitBatchNew(emitter, txid, collectorMock, partition, + preRedeployLastMeta.toMap()); verify(collectorMock, never()).emit(anyList()); - KafkaTridentSpoutBatchMetadata deserializedMeta = KafkaTridentSpoutBatchMetadata.fromMap(meta); + KafkaTridentSpoutBatchMetadata deserializedMeta = KafkaTridentSpoutBatchMetadata + .fromMap(meta); assertThat(deserializedMeta.getFirstOffset(), is(lastOffsetInKafka)); assertThat(deserializedMeta.getLastOffset(), is(lastOffsetInKafka)); long firstNewRecordOffset = lastOffsetInKafka + 1; int numNewRecords = 10; - List> newRecords = SpoutWithMockedConsumerSetupHelper.createRecords(partition, firstNewRecordOffset, numNewRecords); + List> newRecords = SpoutWithMockedConsumerSetupHelper + .createRecords(partition, firstNewRecordOffset, numNewRecords); newRecords.forEach(consumer::addRecord); - meta = emitBatchNew(emitter,txid, collectorMock, partition, meta); + meta = emitBatchNew(emitter, txid, collectorMock, partition, meta); verify(collectorMock, times(numNewRecords)).emit(emitCaptor.capture()); List> emits = emitCaptor.getAllValues(); assertThat(emits.get(0).get(0), is(firstNewRecordOffset)); - assertThat(emits.get(emits.size() - 1).get(0), is(firstNewRecordOffset + numNewRecords - 1)); + assertThat(emits.get(emits.size() - 1).get(0), is(firstNewRecordOffset + + numNewRecords - 1)); deserializedMeta = KafkaTridentSpoutBatchMetadata.fromMap(meta); - assertThat("The batch should start at the first offset of the polled records", deserializedMeta.getFirstOffset(), is(firstNewRecordOffset)); - assertThat("The batch should end at the last offset of the polled messages", deserializedMeta.getLastOffset(), is(firstNewRecordOffset + numNewRecords - 1)); + assertThat("The batch should start at the first offset of the polled records", + deserializedMeta.getFirstOffset(), is(firstNewRecordOffset)); + assertThat("The batch should end at the last offset of the polled messages", + deserializedMeta.getLastOffset(), is(firstNewRecordOffset + numNewRecords - 1)); } @ParameterizedTest @EnumSource(value = FirstPollOffsetStrategy.class, names = {"EARLIEST", "LATEST", "TIMESTAMP"}) public void testUnconditionalStrategyWhenSpoutWorkerIsRestarted(FirstPollOffsetStrategy firstPollOffsetStrategy) { /* - * EARLIEST/LATEST/TIMESTAMP should act like UNCOMMITTED_EARLIEST/LATEST/TIMESTAMP if the emitter is new but the + * EARLIEST/LATEST/TIMESTAMP should act like UNCOMMITTED_EARLIEST/LATEST/TIMESTAMP if the + * emitter is new but the * topology has not restarted (storm id has not changed) */ long preRestartEmittedOffset = 20; int lastBatchEmittedRecords = 10; int preRestartEmittedRecords = 30; - KafkaTridentSpoutBatchMetadata preExecutorRestartLastMeta = new KafkaTridentSpoutBatchMetadata(preRestartEmittedOffset, preRestartEmittedOffset + lastBatchEmittedRecords - 1, topologyId); + KafkaTridentSpoutBatchMetadata preExecutorRestartLastMeta = + new KafkaTridentSpoutBatchMetadata(preRestartEmittedOffset, preRestartEmittedOffset + + lastBatchEmittedRecords - 1, topologyId); KafkaTridentSpoutEmitter emitter = createEmitter(firstPollOffsetStrategy); TransactionAttempt txid = new TransactionAttempt(0L, 0); - Map meta = emitBatchNew(emitter,txid, collectorMock, partition, preExecutorRestartLastMeta.toMap()); + Map meta = emitBatchNew(emitter, txid, collectorMock, partition, + preExecutorRestartLastMeta.toMap()); long firstEmittedOffset = preRestartEmittedOffset + lastBatchEmittedRecords; int emittedRecords = recordsInKafka - preRestartEmittedRecords; @@ -291,44 +355,60 @@ public void testUnconditionalStrategyWhenSpoutWorkerIsRestarted(FirstPollOffsetS List> emits = emitCaptor.getAllValues(); assertThat(emits.get(0).get(0), is(firstEmittedOffset)); assertThat(emits.get(emits.size() - 1).get(0), is(lastOffsetInKafka)); - KafkaTridentSpoutBatchMetadata deserializedMeta = KafkaTridentSpoutBatchMetadata.fromMap(meta); - assertThat("The batch should start at the first offset of the polled records", deserializedMeta.getFirstOffset(), is(firstEmittedOffset)); - assertThat("The batch should end at the last offset of the polled messages", deserializedMeta.getLastOffset(), is(lastOffsetInKafka)); + KafkaTridentSpoutBatchMetadata deserializedMeta = KafkaTridentSpoutBatchMetadata + .fromMap(meta); + assertThat("The batch should start at the first offset of the polled records", + deserializedMeta.getFirstOffset(), is(firstEmittedOffset)); + assertThat("The batch should end at the last offset of the polled messages", + deserializedMeta.getLastOffset(), is(lastOffsetInKafka)); } @Test public void testEarliestStrategyWhenTopologyIsRedeployed() { /* - * EARLIEST should be applied if the emitter is new and the topology has been redeployed (storm id has changed) + * EARLIEST should be applied if the emitter is new and the topology has been redeployed + * (storm id has changed) */ long preRestartEmittedOffset = 20; int preRestartEmittedRecords = 10; - KafkaTridentSpoutBatchMetadata preExecutorRestartLastMeta = new KafkaTridentSpoutBatchMetadata(preRestartEmittedOffset, preRestartEmittedOffset + preRestartEmittedRecords - 1, "Some older topology"); - KafkaTridentSpoutEmitter emitter = createEmitter(FirstPollOffsetStrategy.EARLIEST); + KafkaTridentSpoutBatchMetadata preExecutorRestartLastMeta = + new KafkaTridentSpoutBatchMetadata(preRestartEmittedOffset, preRestartEmittedOffset + + preRestartEmittedRecords - 1, "Some older topology"); + KafkaTridentSpoutEmitter emitter = + createEmitter(FirstPollOffsetStrategy.EARLIEST); TransactionAttempt txid = new TransactionAttempt(0L, 0); - Map meta = emitBatchNew(emitter, txid, collectorMock, partition, preExecutorRestartLastMeta.toMap()); + Map meta = emitBatchNew(emitter, txid, collectorMock, partition, + preExecutorRestartLastMeta.toMap()); verify(collectorMock, times(recordsInKafka)).emit(emitCaptor.capture()); List> emits = emitCaptor.getAllValues(); assertThat(emits.get(0).get(0), is(firstOffsetInKafka)); assertThat(emits.get(emits.size() - 1).get(0), is(lastOffsetInKafka)); - KafkaTridentSpoutBatchMetadata deserializedMeta = KafkaTridentSpoutBatchMetadata.fromMap(meta); - assertThat("The batch should start at the first offset of the polled records", deserializedMeta.getFirstOffset(), is(firstOffsetInKafka)); - assertThat("The batch should end at the last offset of the polled messages", deserializedMeta.getLastOffset(), is(lastOffsetInKafka)); + KafkaTridentSpoutBatchMetadata deserializedMeta = KafkaTridentSpoutBatchMetadata + .fromMap(meta); + assertThat("The batch should start at the first offset of the polled records", + deserializedMeta.getFirstOffset(), is(firstOffsetInKafka)); + assertThat("The batch should end at the last offset of the polled messages", + deserializedMeta.getLastOffset(), is(lastOffsetInKafka)); } @Test public void testLatestStrategyWhenTopologyIsRedeployed() { /* - * EARLIEST should be applied if the emitter is new and the topology has been redeployed (storm id has changed) + * EARLIEST should be applied if the emitter is new and the topology has been redeployed + * (storm id has changed) */ long preRestartEmittedOffset = 20; int preRestartEmittedRecords = 10; - KafkaTridentSpoutBatchMetadata preExecutorRestartLastMeta = new KafkaTridentSpoutBatchMetadata(preRestartEmittedOffset, preRestartEmittedOffset + preRestartEmittedRecords - 1, "Some older topology"); - KafkaTridentSpoutEmitter emitter = createEmitter(FirstPollOffsetStrategy.LATEST); + KafkaTridentSpoutBatchMetadata preExecutorRestartLastMeta = + new KafkaTridentSpoutBatchMetadata(preRestartEmittedOffset, preRestartEmittedOffset + + preRestartEmittedRecords - 1, "Some older topology"); + KafkaTridentSpoutEmitter emitter = + createEmitter(FirstPollOffsetStrategy.LATEST); TransactionAttempt txid = new TransactionAttempt(0L, 0); KafkaTridentSpoutTopicPartition kttp = new KafkaTridentSpoutTopicPartition(partition); - Map meta = emitBatchNew(emitter,txid, collectorMock, partition, preExecutorRestartLastMeta.toMap()); + Map meta = emitBatchNew(emitter, txid, collectorMock, partition, + preExecutorRestartLastMeta.toMap()); verify(collectorMock, never()).emit(anyList()); } @@ -336,36 +416,48 @@ public void testLatestStrategyWhenTopologyIsRedeployed() { @Test public void testTimeStampStrategyWhenTopologyIsRedeployed() { /* - * TIMESTAMP strategy should be applied if the emitter is new and the topology has been redeployed (storm id has changed) + * TIMESTAMP strategy should be applied if the emitter is new and the topology has been + * redeployed (storm id has changed) * Offset should be reset according to the offset corresponding to startTimeStamp */ long preRestartEmittedOffset = 20; int preRestartEmittedRecords = 10; long timeStampStartOffset = 2L; long pollTimeout = 1L; - KafkaTridentSpoutBatchMetadata preExecutorRestartLastMeta = new KafkaTridentSpoutBatchMetadata(preRestartEmittedOffset, preRestartEmittedOffset + preRestartEmittedRecords - 1, "Some older topology"); + KafkaTridentSpoutBatchMetadata preExecutorRestartLastMeta = + new KafkaTridentSpoutBatchMetadata(preRestartEmittedOffset, preRestartEmittedOffset + + preRestartEmittedRecords - 1, "Some older topology"); KafkaConsumer kafkaConsumer = Mockito.mock(KafkaConsumer.class); when(kafkaConsumer.assignment()).thenReturn(Collections.singleton(partition)); - OffsetAndTimestamp offsetAndTimestamp = new OffsetAndTimestamp(timeStampStartOffset, startTimeStamp); + OffsetAndTimestamp offsetAndTimestamp = new OffsetAndTimestamp(timeStampStartOffset, + startTimeStamp); HashMap map = new HashMap<>(); map.put(partition, offsetAndTimestamp); - when(kafkaConsumer.offsetsForTimes(Collections.singletonMap(partition, startTimeStamp))).thenReturn(map); - HashMap>> topicPartitionMap = new HashMap<>(); - List> newRecords = SpoutWithMockedConsumerSetupHelper.createRecords(partition, timeStampStartOffset, recordsInKafka); + when(kafkaConsumer.offsetsForTimes(Collections.singletonMap(partition, startTimeStamp))) + .thenReturn(map); + HashMap>> topicPartitionMap = + new HashMap<>(); + List> newRecords = SpoutWithMockedConsumerSetupHelper + .createRecords(partition, timeStampStartOffset, recordsInKafka); topicPartitionMap.put(partition, newRecords); - when(kafkaConsumer.poll(Duration.ofMillis(pollTimeout))).thenReturn(new ConsumerRecords<>(topicPartitionMap)); + when(kafkaConsumer.poll(Duration.ofMillis(pollTimeout))) + .thenReturn(new ConsumerRecords<>(topicPartitionMap)); - KafkaTridentSpoutEmitter emitter = createEmitter(kafkaConsumer, adminClient, FirstPollOffsetStrategy.TIMESTAMP); + KafkaTridentSpoutEmitter emitter = createEmitter(kafkaConsumer, adminClient, + FirstPollOffsetStrategy.TIMESTAMP); TransactionAttempt txid = new TransactionAttempt(0L, 0); - Map meta = emitBatchNew(emitter,txid, collectorMock, partition, preExecutorRestartLastMeta.toMap()); + Map meta = emitBatchNew(emitter, txid, collectorMock, partition, + preExecutorRestartLastMeta.toMap()); verify(collectorMock, times(recordsInKafka)).emit(emitCaptor.capture()); verify(kafkaConsumer, times(1)).seek(partition, timeStampStartOffset); List> emits = emitCaptor.getAllValues(); assertThat(emits.get(0).get(0), is(timeStampStartOffset)); - KafkaTridentSpoutBatchMetadata deserializedMeta = KafkaTridentSpoutBatchMetadata.fromMap(meta); - assertThat("The batch should start at the first offset for startTimestamp", deserializedMeta.getFirstOffset(), is(timeStampStartOffset)); + KafkaTridentSpoutBatchMetadata deserializedMeta = KafkaTridentSpoutBatchMetadata + .fromMap(meta); + assertThat("The batch should start at the first offset for startTimestamp", deserializedMeta + .getFirstOffset(), is(timeStampStartOffset)); } } diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/trident/KafkaTridentSpoutEmitterPartitioningTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/trident/KafkaTridentSpoutEmitterPartitioningTest.java index b10f040f22b..c61d93a5129 100644 --- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/trident/KafkaTridentSpoutEmitterPartitioningTest.java +++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/trident/KafkaTridentSpoutEmitterPartitioningTest.java @@ -31,7 +31,6 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; - import org.apache.kafka.clients.admin.Admin; import org.apache.kafka.clients.admin.MockAdminClient; import org.apache.kafka.clients.consumer.Consumer; @@ -57,7 +56,8 @@ public class KafkaTridentSpoutEmitterPartitioningTest { @Mock public TopologyContext topologyContextMock; - private final MockConsumer consumer = new MockConsumer<>(OffsetResetStrategy.NONE); + private final MockConsumer consumer = + new MockConsumer<>(OffsetResetStrategy.NONE); private final MockAdminClient adminClient = new MockAdminClient(); private final TopicPartitionSerializer tpSerializer = new TopicPartitionSerializer(); @@ -67,16 +67,16 @@ public void testGetOrderedPartitionsIsConsistent() { SingleTopicKafkaTridentSpoutConfiguration.createKafkaSpoutConfigBuilder(-1) .build(), topologyContextMock, new ClientFactory() { - @Override + @Override public Consumer createConsumer(Map consumerProps) { - return consumer; - } + return consumer; + } - @Override + @Override public Admin createAdmin(Map adminProps) { - return adminClient; - } - }, new TopicAssigner()); + return adminClient; + } + }, new TopicAssigner()); Set allPartitions = new HashSet<>(); int numPartitions = 10; @@ -87,21 +87,29 @@ public Admin createAdmin(Map adminProps) { .map(tpSerializer::toMap) .collect(Collectors.toList()); - List orderedPartitions = emitter.getOrderedPartitions(serializedPartitions); - assertThat("Should contain all partitions", orderedPartitions.size(), is(allPartitions.size())); + List orderedPartitions = emitter + .getOrderedPartitions(serializedPartitions); + assertThat("Should contain all partitions", orderedPartitions.size(), is(allPartitions + .size())); Collections.shuffle(serializedPartitions); - List secondGetOrderedPartitions = emitter.getOrderedPartitions(serializedPartitions); - assertThat("Ordering must be consistent", secondGetOrderedPartitions, is(orderedPartitions)); + List secondGetOrderedPartitions = emitter + .getOrderedPartitions(serializedPartitions); + assertThat("Ordering must be consistent", secondGetOrderedPartitions, + is(orderedPartitions)); - serializedPartitions.add(tpSerializer.toMap(new TopicPartition(SingleTopicKafkaSpoutConfiguration.TOPIC, numPartitions))); - List orderedPartitionsWithNewPartition = emitter.getOrderedPartitions(serializedPartitions); + serializedPartitions.add(tpSerializer + .toMap(new TopicPartition(SingleTopicKafkaSpoutConfiguration.TOPIC, + numPartitions))); + List orderedPartitionsWithNewPartition = emitter + .getOrderedPartitions(serializedPartitions); orderedPartitionsWithNewPartition.remove(orderedPartitionsWithNewPartition.size() - 1); - assertThat("Adding new partitions should not shuffle the existing ordering", orderedPartitionsWithNewPartition, is(orderedPartitions)); + assertThat("Adding new partitions should not shuffle the existing ordering", + orderedPartitionsWithNewPartition, is(orderedPartitions)); } @Test public void testGetPartitionsForTask() { - //Verify correct wrapping/unwrapping of partition and delegation of partition assignment + // Verify correct wrapping/unwrapping of partition and delegation of partition assignment ManualPartitioner partitionerMock = mock(ManualPartitioner.class); when(partitionerMock.getPartitionsForThisTask(any(), any())) .thenAnswer(invocation -> { @@ -111,57 +119,63 @@ public void testGetPartitionsForTask() { }); KafkaTridentSpoutEmitter emitter = new KafkaTridentSpoutEmitter<>( - SingleTopicKafkaTridentSpoutConfiguration.createKafkaSpoutConfigBuilder(mock(TopicFilter.class), partitionerMock, -1) + SingleTopicKafkaTridentSpoutConfiguration + .createKafkaSpoutConfigBuilder(mock(TopicFilter.class), partitionerMock, -1) .build(), topologyContextMock, new ClientFactory() { - @Override + @Override public Consumer createConsumer(Map consumerProps) { - return consumer; - } + return consumer; + } - @Override + @Override public Admin createAdmin(Map adminProps) { - return adminClient; - } - }, new TopicAssigner()); + return adminClient; + } + }, new TopicAssigner()); List allPartitions = new ArrayList<>(); for (int i = 0; i < 10; i++) { - allPartitions.add(new KafkaTridentSpoutTopicPartition(SingleTopicKafkaSpoutConfiguration.TOPIC, i)); + allPartitions + .add(new KafkaTridentSpoutTopicPartition(SingleTopicKafkaSpoutConfiguration.TOPIC, i)); } List unwrappedPartitions = allPartitions.stream() .map(kttp -> kttp.getTopicPartition()) .collect(Collectors.toList()); - List partitionsForTask = emitter.getPartitionsForTask(0, 2, allPartitions); - verify(partitionerMock).getPartitionsForThisTask(eq(unwrappedPartitions), any(TopologyContext.class)); + List partitionsForTask = emitter.getPartitionsForTask(0, 2, + allPartitions); + verify(partitionerMock).getPartitionsForThisTask(eq(unwrappedPartitions), + any(TopologyContext.class)); allPartitions.remove(0); - assertThat("Should have assigned all except the first partition to this task", new HashSet<>(partitionsForTask), is(new HashSet<>(allPartitions))); + assertThat("Should have assigned all except the first partition to this task", + new HashSet<>(partitionsForTask), is(new HashSet<>(allPartitions))); } @Test public void testAssignPartitions() { - //Verify correct unwrapping of partitions and delegation of assignment + // Verify correct unwrapping of partitions and delegation of assignment TopicAssigner assignerMock = mock(TopicAssigner.class); KafkaTridentSpoutEmitter emitter = new KafkaTridentSpoutEmitter<>( SingleTopicKafkaTridentSpoutConfiguration.createKafkaSpoutConfigBuilder(-1) .build(), topologyContextMock, new ClientFactory() { - @Override + @Override public Consumer createConsumer(Map consumerProps) { - return consumer; - } + return consumer; + } - @Override + @Override public Admin createAdmin(Map adminProps) { - return adminClient; - } - }, assignerMock); + return adminClient; + } + }, assignerMock); List allPartitions = new ArrayList<>(); for (int i = 0; i < 10; i++) { - allPartitions.add(new KafkaTridentSpoutTopicPartition(SingleTopicKafkaSpoutConfiguration.TOPIC, i)); + allPartitions + .add(new KafkaTridentSpoutTopicPartition(SingleTopicKafkaSpoutConfiguration.TOPIC, i)); } Set unwrappedPartitions = allPartitions.stream() .map(kttp -> kttp.getTopicPartition()) @@ -169,7 +183,8 @@ public Admin createAdmin(Map adminProps) { emitter.refreshPartitions(allPartitions); - verify(assignerMock).assignPartitions(eq(consumer), eq(unwrappedPartitions), any(ConsumerRebalanceListener.class)); + verify(assignerMock).assignPartitions(eq(consumer), eq(unwrappedPartitions), + any(ConsumerRebalanceListener.class)); } } diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/trident/KafkaTridentSpoutOpaqueCoordinatorTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/trident/KafkaTridentSpoutOpaqueCoordinatorTest.java index 07dfc40815e..e6fe73ef9d4 100644 --- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/trident/KafkaTridentSpoutOpaqueCoordinatorTest.java +++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/trident/KafkaTridentSpoutOpaqueCoordinatorTest.java @@ -16,7 +16,6 @@ package org.apache.storm.kafka.spout.trident; - import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.is; @@ -31,7 +30,6 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; - import org.apache.kafka.clients.admin.Admin; import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.consumer.KafkaConsumer; @@ -40,8 +38,8 @@ import org.apache.storm.kafka.spout.subscription.ManualPartitioner; import org.apache.storm.kafka.spout.subscription.TopicFilter; import org.apache.storm.kafka.spout.trident.config.builder.SingleTopicKafkaTridentSpoutConfiguration; -import org.apache.storm.utils.Time; import org.apache.storm.utils.Time.SimulatedTime; +import org.apache.storm.utils.Time; import org.junit.jupiter.api.Test; public class KafkaTridentSpoutOpaqueCoordinatorTest { @@ -54,24 +52,28 @@ public void testCanGetPartitions() { Admin mockAdmin = mock(Admin.class); TopicPartition expectedPartition = new TopicPartition("test", 0); TopicFilter mockFilter = mock(TopicFilter.class); - when(mockFilter.getAllSubscribedPartitions(any())).thenReturn(Collections.singleton(expectedPartition)); + when(mockFilter.getAllSubscribedPartitions(any())).thenReturn(Collections + .singleton(expectedPartition)); KafkaTridentSpoutConfig spoutConfig = - SingleTopicKafkaTridentSpoutConfiguration.createKafkaSpoutConfigBuilder(mockFilter, mock(ManualPartitioner.class), -1) + SingleTopicKafkaTridentSpoutConfiguration.createKafkaSpoutConfigBuilder(mockFilter, + mock(ManualPartitioner.class), -1) .build(); - KafkaTridentSpoutCoordinator coordinator = new KafkaTridentSpoutCoordinator<>(spoutConfig, new ClientFactory() { - @Override + KafkaTridentSpoutCoordinator coordinator = + new KafkaTridentSpoutCoordinator<>(spoutConfig, new ClientFactory() { + @Override public Consumer createConsumer(Map consumerProps) { - return mockConsumer; - } + return mockConsumer; + } - @Override + @Override public Admin createAdmin(Map adminProps) { - return mockAdmin; - } - }); + return mockAdmin; + } + }); - List < Map < String, Object >> partitionsForBatch = coordinator.getPartitionsForBatch(); + List> partitionsForBatch = coordinator.getPartitionsForBatch(); List tps = deserializePartitions(partitionsForBatch); @@ -96,30 +98,35 @@ public void testCanUpdatePartitions() { .thenReturn(allPartitions); KafkaTridentSpoutConfig spoutConfig = - SingleTopicKafkaTridentSpoutConfiguration.createKafkaSpoutConfigBuilder(mockFilter, mock(ManualPartitioner.class), -1) + SingleTopicKafkaTridentSpoutConfiguration.createKafkaSpoutConfigBuilder(mockFilter, + mock(ManualPartitioner.class), -1) .build(); - KafkaTridentSpoutCoordinator coordinator = new KafkaTridentSpoutCoordinator<>(spoutConfig, new ClientFactory() { - @Override + KafkaTridentSpoutCoordinator coordinator = + new KafkaTridentSpoutCoordinator<>(spoutConfig, new ClientFactory() { + @Override public Consumer createConsumer(Map consumerProps) { - return mockConsumer; - } + return mockConsumer; + } - @Override + @Override public Admin createAdmin(Map adminProps) { - return mockAdmin; - } - }); + return mockAdmin; + } + }); - List < Map < String, Object >> partitionsForBatch = coordinator.getPartitionsForBatch(); + List> partitionsForBatch = coordinator.getPartitionsForBatch(); List firstBatchTps = deserializePartitions(partitionsForBatch); verify(mockFilter).getAllSubscribedPartitions(mockConsumer); assertThat(firstBatchTps, contains(expectedPartition)); - Time.advanceTime(KafkaTridentSpoutCoordinator.TIMER_DELAY_MS + spoutConfig.getPartitionRefreshPeriodMs()); + Time.advanceTime(KafkaTridentSpoutCoordinator.TIMER_DELAY_MS + spoutConfig + .getPartitionRefreshPeriodMs()); - List> partitionsForSecondBatch = coordinator.getPartitionsForBatch(); + List> partitionsForSecondBatch = coordinator + .getPartitionsForBatch(); List secondBatchTps = deserializePartitions(partitionsForSecondBatch); verify(mockFilter, times(2)).getAllSubscribedPartitions(mockConsumer); diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/trident/config/builder/SingleTopicKafkaTridentSpoutConfiguration.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/trident/config/builder/SingleTopicKafkaTridentSpoutConfiguration.java index 5adeb1e807c..a0e7bd10351 100644 --- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/trident/config/builder/SingleTopicKafkaTridentSpoutConfiguration.java +++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/trident/config/builder/SingleTopicKafkaTridentSpoutConfiguration.java @@ -34,7 +34,8 @@ public static KafkaTridentSpoutConfig.Builder createKafkaSpoutCo } public static KafkaTridentSpoutConfig.Builder createKafkaSpoutConfigBuilder(TopicFilter topicFilter, ManualPartitioner topicPartitioner, int port) { - return setCommonSpoutConfig(new KafkaTridentSpoutConfig.Builder<>("127.0.0.1:" + port, topicFilter, topicPartitioner)); + return setCommonSpoutConfig(new KafkaTridentSpoutConfig.Builder<>("127.0.0.1:" + port, + topicFilter, topicPartitioner)); } public static KafkaTridentSpoutConfig.Builder setCommonSpoutConfig(KafkaTridentSpoutConfig.Builder config) { diff --git a/external/storm-kafka-migration/pom.xml b/external/storm-kafka-migration/pom.xml index 1a0f1e580bd..f2bcde4ba9c 100644 --- a/external/storm-kafka-migration/pom.xml +++ b/external/storm-kafka-migration/pom.xml @@ -81,6 +81,16 @@ + + org.openrewrite.maven + rewrite-maven-plugin + + + + org.codehaus.gmavenplus + gmavenplus-plugin + + org.apache.maven.plugins maven-checkstyle-plugin diff --git a/external/storm-kafka-migration/src/main/java/org/apache/storm/kafka/migration/KafkaSpoutMigration.java b/external/storm-kafka-migration/src/main/java/org/apache/storm/kafka/migration/KafkaSpoutMigration.java index bdd77eebce8..717f3c5b8de 100644 --- a/external/storm-kafka-migration/src/main/java/org/apache/storm/kafka/migration/KafkaSpoutMigration.java +++ b/external/storm-kafka-migration/src/main/java/org/apache/storm/kafka/migration/KafkaSpoutMigration.java @@ -54,7 +54,8 @@ private static class Configuration { } /** - * Migrates offsets from the Zookeeper store used by the storm-kafka non-Trident spouts, to Kafka's offset store used by the + * Migrates offsets from the Zookeeper store used by the storm-kafka non-Trident spouts, to + * Kafka's offset store used by the * storm-kafka-client non-Trident spout. */ public static void main(String[] args) throws Exception { @@ -74,7 +75,8 @@ public static void main(String[] args) throws Exception { configuration.kafkaBootstrapServers = MapUtil.getOrError(conf, "kafka.bootstrap.servers"); configuration.newSpoutConsumerGroup = MapUtil.getOrError(conf, "new.spout.consumer.group"); configuration.zkSessionTimeoutMs = MapUtil.getOrError(conf, "zookeeper.session.timeout.ms"); - configuration.zkConnectionTimeoutMs = MapUtil.getOrError(conf, "zookeeper.connection.timeout.ms"); + configuration.zkConnectionTimeoutMs = MapUtil.getOrError(conf, + "zookeeper.connection.timeout.ms"); configuration.zkRetryTimes = MapUtil.getOrError(conf, "zookeeper.retry.times"); configuration.zkRetryIntervalMs = MapUtil.getOrError(conf, "zookeeper.retry.interval.ms"); @@ -93,7 +95,8 @@ public static void main(String[] args) throws Exception { consumer.commitSync(offsetsToCommit); } - LOG.info("Migrated offsets {} to consumer group {}", offsetsToCommit, configuration.newSpoutConsumerGroup); + LOG.info("Migrated offsets {} to consumer group {}", offsetsToCommit, + configuration.newSpoutConsumerGroup); } private static Map getOffsetsAtPath( @@ -108,8 +111,9 @@ private static Map getOffsetsAtPath( String absPartitionPath = partitionsRoot + "/" + partitionPath; LOG.info("Reading offset data from path {}", absPartitionPath); byte[] partitionBytes = curator.getData().forPath(absPartitionPath); - Map partitionMetadata = objectMapper.readValue(partitionBytes, new TypeReference>() { - }); + Map partitionMetadata = objectMapper.readValue(partitionBytes, + new TypeReference>() { + }); String topic = (String) partitionMetadata.get("topic"); int partition = ((Number) partitionMetadata.get("partition")).intValue(); long offset = ((Number) partitionMetadata.get("offset")).longValue(); @@ -135,11 +139,13 @@ private static Map getOffsetsToCommit(Configu List topicPaths = curator.getChildren().forPath(spoutRoot); for (String topicPath : topicPaths) { if (!topicPath.matches(configuration.topic)) { - LOG.info("Skipping directory {} because it doesn't match the topic pattern {}", topicPath, configuration.topic); + LOG.info("Skipping directory {} because it doesn't match the topic " + + "pattern {}", topicPath, configuration.topic); } else { String absTopicPath = spoutRoot + "/" + topicPath; LOG.info("Looking for partitions in {}", absTopicPath); - offsetsToCommit.putAll(getOffsetsAtPath(curator, objectMapper, absTopicPath)); + offsetsToCommit.putAll(getOffsetsAtPath(curator, objectMapper, + absTopicPath)); } } } else { diff --git a/external/storm-kafka-migration/src/main/java/org/apache/storm/kafka/migration/KafkaTridentSpoutMigration.java b/external/storm-kafka-migration/src/main/java/org/apache/storm/kafka/migration/KafkaTridentSpoutMigration.java index f27ecef03be..d1535208160 100644 --- a/external/storm-kafka-migration/src/main/java/org/apache/storm/kafka/migration/KafkaTridentSpoutMigration.java +++ b/external/storm-kafka-migration/src/main/java/org/apache/storm/kafka/migration/KafkaTridentSpoutMigration.java @@ -21,8 +21,8 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Map.Entry; +import java.util.Map; import org.apache.commons.lang3.builder.ToStringBuilder; import org.apache.commons.lang3.builder.ToStringStyle; import org.apache.curator.framework.CuratorFramework; @@ -30,7 +30,6 @@ import org.apache.curator.framework.api.PathAndBytesable; import org.apache.curator.retry.RetryNTimes; import org.apache.kafka.common.TopicPartition; - import org.apache.storm.utils.Utils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -71,7 +70,8 @@ public String toString() { } /** - * Migrates offsets from the Zookeeper store used by the storm-kafka Trident spouts, to the Zookeeper store used by the + * Migrates offsets from the Zookeeper store used by the storm-kafka Trident spouts, to the + * Zookeeper store used by the * storm-kafka-clients Trident spout. */ public static void main(String[] args) throws Exception { @@ -90,14 +90,16 @@ public static void main(String[] args) throws Exception { configuration.isWildcardTopic = MapUtil.getOrError(conf, "is.wildcard.topic"); configuration.newTopologyTxId = MapUtil.getOrError(conf, "new.topology.txid"); configuration.zkSessionTimeoutMs = MapUtil.getOrError(conf, "zookeeper.session.timeout.ms"); - configuration.zkConnectionTimeoutMs = MapUtil.getOrError(conf, "zookeeper.connection.timeout.ms"); + configuration.zkConnectionTimeoutMs = MapUtil.getOrError(conf, + "zookeeper.connection.timeout.ms"); configuration.zkRetryTimes = MapUtil.getOrError(conf, "zookeeper.retry.times"); configuration.zkRetryIntervalMs = MapUtil.getOrError(conf, "zookeeper.retry.interval.ms"); try (CuratorFramework curator = newCurator(configuration)) { curator.start(); - Map> offsetsToMigrate = getOffsetsToMigrate(curator, configuration); + Map> offsetsToMigrate = + getOffsetsToMigrate(curator, configuration); LOG.info("Migrating offsets {}", offsetsToMigrate); @@ -125,12 +127,15 @@ private static Map> getOffsetsAtPat String absTransactionPath = absPartitionPath + "/" + transaction; LOG.info("Reading offset data from path {}", absTransactionPath); byte[] partitionBytes = curator.getData().forPath(absTransactionPath); - Map partitionMetadata = objectMapper.readValue(partitionBytes, new TypeReference>() { - }); - tp = new TopicPartition((String) partitionMetadata.get("topic"), ((Number) partitionMetadata.get("partition")).intValue()); + Map partitionMetadata = objectMapper.readValue(partitionBytes, + new TypeReference>() { + }); + tp = new TopicPartition((String) partitionMetadata.get("topic"), + ((Number) partitionMetadata.get("partition")).intValue()); PartitionMetadata meta = new PartitionMetadata( ((Number) partitionMetadata.get("offset")).longValue(), - ((Number) partitionMetadata.get("nextOffset")).longValue() - 1); //nextOffset is the last offset from last batch + 1 + ((Number) partitionMetadata.get("nextOffset")) + .longValue() - 1); // nextOffset is the last offset from last batch + 1 partitionMeta.put(Long.parseLong(transaction), meta); } if (tp != null) { @@ -142,7 +147,7 @@ private static Map> getOffsetsAtPat private static Map> getOffsetsToMigrate( CuratorFramework curator, Configuration configuration) throws Exception { - //Read the partitions, transaction ids and offsets from the old storm-kafka /user path + // Read the partitions, transaction ids and offsets from the old storm-kafka /user path Map> offsetsToMigrate = new HashMap<>(); String streamRoot = configuration.zkRoot + "/" + configuration.txId + "/user"; @@ -155,11 +160,13 @@ private static Map> getOffsetsToMig List topics = curator.getChildren().forPath(streamRoot); for (String topic : topics) { if (!topic.matches(configuration.topic)) { - LOG.info("Skipping directory {} because it does not match topic pattern {}", topic, configuration.topic); + LOG.info("Skipping directory {} because it does not match topic pattern {}", + topic, configuration.topic); } else { String partitionsRoot = streamRoot + "/" + topic; LOG.info("Looking for partitions in {}", partitionsRoot); - offsetsToMigrate.putAll(getOffsetsAtPath(curator, objectMapper, partitionsRoot)); + offsetsToMigrate.putAll(getOffsetsAtPath(curator, objectMapper, + partitionsRoot)); } } } else { @@ -175,8 +182,8 @@ private static String coordinatorPath(Configuration configuration, String txid) private static void migrateCoordinator( CuratorFramework curator, Configuration configuration, List topics) throws Exception { - //Migrate the /coordinator currtx, currattempts and meta directories. - //The new spout expects the list of topic partitions as coordinator meta. + // Migrate the /coordinator currtx, currattempts and meta directories. + // The new spout expects the list of topic partitions as coordinator meta. String oldCoordinatorRoot = coordinatorPath(configuration, configuration.txId); String newCoordinatorRoot = coordinatorPath(configuration, configuration.newTopologyTxId); @@ -186,7 +193,8 @@ private static void migrateCoordinator( String oldAttemptsPath = oldCoordinatorRoot + "/currattempts"; String newAttemptsPath = newCoordinatorRoot + "/currattempts"; - createOrUpdate(curator, newAttemptsPath).forPath(newAttemptsPath, curator.getData().forPath(oldAttemptsPath)); + createOrUpdate(curator, newAttemptsPath).forPath(newAttemptsPath, curator.getData() + .forPath(oldAttemptsPath)); List transactions = curator.getChildren().forPath(oldCoordinatorRoot + "/meta"); List> coordinatorMeta = new ArrayList<>(); @@ -195,12 +203,14 @@ private static void migrateCoordinator( } for (String transaction : transactions) { String newMetaPath = newCoordinatorRoot + "/meta/" + transaction; - createOrUpdate(curator, newMetaPath).forPath(newMetaPath, objectMapper.writeValueAsBytes(coordinatorMeta)); + createOrUpdate(curator, newMetaPath).forPath(newMetaPath, objectMapper + .writeValueAsBytes(coordinatorMeta)); } LOG.info("Migrated coordinator data to new path {}", newCoordinatorRoot); } - private static PathAndBytesable createOrUpdate(CuratorFramework curator, String path) throws Exception { + private static PathAndBytesable createOrUpdate(CuratorFramework curator, + String path) throws Exception { if (curator.checkExists().forPath(path) == null) { return curator.create().creatingParentsIfNeeded(); } else { @@ -217,7 +227,7 @@ private static Map tpMeta(TopicPartition tp) { private static void migrateOffsets( CuratorFramework curator, Configuration configuration, Map> offsets) throws Exception { - //Writes the offsets in the new format to the /user partitions paths + // Writes the offsets in the new format to the /user partitions paths String streamRoot = configuration.zkRoot + "/" + configuration.newTopologyTxId + "/user"; for (Entry> offset : offsets.entrySet()) { @@ -228,9 +238,11 @@ private static void migrateOffsets( metadataToWrite.put("firstOffset", meta.firstOffset); metadataToWrite.put("lastOffset", meta.lastOffset); metadataToWrite.put("tp", tpMeta(tp)); - String partitionPath = streamRoot + "/" + tp.topic() + "@" + tp.partition() + "/" + transaction.getKey(); + String partitionPath = streamRoot + "/" + tp.topic() + "@" + tp.partition() + "/" + + transaction.getKey(); LOG.info("Writing {} to path {}", metadataToWrite, partitionPath); - createOrUpdate(curator, partitionPath).forPath(partitionPath, objectMapper.writeValueAsBytes(metadataToWrite)); + createOrUpdate(curator, partitionPath).forPath(partitionPath, objectMapper + .writeValueAsBytes(metadataToWrite)); } } diff --git a/external/storm-kafka-monitor/pom.xml b/external/storm-kafka-monitor/pom.xml index 326e00a34ef..0aa34b473e5 100644 --- a/external/storm-kafka-monitor/pom.xml +++ b/external/storm-kafka-monitor/pom.xml @@ -81,6 +81,16 @@ + + org.openrewrite.maven + rewrite-maven-plugin + + + + org.codehaus.gmavenplus + gmavenplus-plugin + + org.apache.maven.plugins maven-checkstyle-plugin diff --git a/external/storm-kafka-monitor/src/main/java/org/apache/storm/kafka/monitor/KafkaOffsetLagResult.java b/external/storm-kafka-monitor/src/main/java/org/apache/storm/kafka/monitor/KafkaOffsetLagResult.java index d028a54e465..b77f456dce6 100644 --- a/external/storm-kafka-monitor/src/main/java/org/apache/storm/kafka/monitor/KafkaOffsetLagResult.java +++ b/external/storm-kafka-monitor/src/main/java/org/apache/storm/kafka/monitor/KafkaOffsetLagResult.java @@ -30,7 +30,8 @@ public class KafkaOffsetLagResult implements JSONAware { private long logHeadOffset; private long lag; - public KafkaOffsetLagResult(String topic, int parition, long consumerCommittedOffset, long logHeadOffset) { + public KafkaOffsetLagResult(String topic, int parition, long consumerCommittedOffset, + long logHeadOffset) { this.topic = topic; this.partition = parition; this.consumerCommittedOffset = consumerCommittedOffset; diff --git a/external/storm-kafka-monitor/src/main/java/org/apache/storm/kafka/monitor/KafkaOffsetLagUtil.java b/external/storm-kafka-monitor/src/main/java/org/apache/storm/kafka/monitor/KafkaOffsetLagUtil.java index d5918f78443..44e352f1a39 100644 --- a/external/storm-kafka-monitor/src/main/java/org/apache/storm/kafka/monitor/KafkaOffsetLagUtil.java +++ b/external/storm-kafka-monitor/src/main/java/org/apache/storm/kafka/monitor/KafkaOffsetLagUtil.java @@ -24,7 +24,6 @@ import java.util.List; import java.util.Map; import java.util.Properties; - import net.minidev.json.JSONValue; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; @@ -64,17 +63,21 @@ public static void main(String[] args) { } String securityProtocol = commandLine.getOptionValue(OPTION_SECURITY_PROTOCOL_LONG); String saslMechanism = commandLine.getOptionValue(OPTION_SASL_MECHANISM_LONG); - if (!commandLine.hasOption(OPTION_GROUP_ID_LONG) || !commandLine.hasOption(OPTION_BOOTSTRAP_BROKERS_LONG)) { - printUsageAndExit(options, OPTION_GROUP_ID_LONG + " and " + OPTION_BOOTSTRAP_BROKERS_LONG + " are required"); + if (!commandLine.hasOption(OPTION_GROUP_ID_LONG) || !commandLine + .hasOption(OPTION_BOOTSTRAP_BROKERS_LONG)) { + printUsageAndExit(options, OPTION_GROUP_ID_LONG + " and " + + OPTION_BOOTSTRAP_BROKERS_LONG + " are required"); } NewKafkaSpoutOffsetQuery newKafkaSpoutOffsetQuery = new NewKafkaSpoutOffsetQuery(commandLine.getOptionValue(OPTION_TOPIC_LONG), commandLine.getOptionValue(OPTION_BOOTSTRAP_BROKERS_LONG), - commandLine.getOptionValue(OPTION_GROUP_ID_LONG), securityProtocol, saslMechanism, + commandLine + .getOptionValue(OPTION_GROUP_ID_LONG), securityProtocol, saslMechanism, commandLine.getOptionValue(OPTION_CONSUMER_CONFIG_LONG)); List results = getOffsetLags(newKafkaSpoutOffsetQuery); - Map> keyedResult = keyByTopicAndPartition(results); + Map> keyedResult = + keyByTopicAndPartition(results); System.out.print(JSONValue.toJSONString(keyedResult)); } catch (Exception ex) { System.out.print("Unable to get offset lags for kafka. Reason: "); @@ -88,14 +91,17 @@ private static Map> keyByTopicAndP for (KafkaOffsetLagResult result : results) { String topic = result.getTopic(); - Map topicResultKeyedByPartition = resultKeyedByTopic.get(topic); + Map topicResultKeyedByPartition = resultKeyedByTopic + .get(topic); if (topicResultKeyedByPartition == null) { topicResultKeyedByPartition = new HashMap<>(); resultKeyedByTopic.put(topic, topicResultKeyedByPartition); } topicResultKeyedByPartition.put(result.getPartition(), - new KafkaPartitionOffsetLag(result.getConsumerCommittedOffset(), result.getLogHeadOffset())); + new KafkaPartitionOffsetLag(result + .getConsumerCommittedOffset(), result + .getLogHeadOffset())); } return resultKeyedByTopic; @@ -118,7 +124,8 @@ private static Options buildOptions() { true, "Comma separated list of bootstrap broker hosts for new " + "consumer/spout e.g. hostname1:9092,hostname2:9092"); - options.addOption(OPTION_GROUP_ID_SHORT, OPTION_GROUP_ID_LONG, true, "Group id of consumer"); + options.addOption(OPTION_GROUP_ID_SHORT, OPTION_GROUP_ID_LONG, true, + "Group id of consumer"); options.addOption(OPTION_SECURITY_PROTOCOL_SHORT, OPTION_SECURITY_PROTOCOL_LONG, true, @@ -136,7 +143,9 @@ private static Options buildOptions() { /** * Get offset lags. - * @param newKafkaSpoutOffsetQuery represents the information needed to query kafka for log head and spout offsets + * + * @param newKafkaSpoutOffsetQuery represents the information needed to query kafka for log head + * and spout offsets * @return log head offset, spout offset and lag for each partition */ public static List getOffsetLags(NewKafkaSpoutOffsetQuery newKafkaSpoutOffsetQuery) throws Exception { @@ -148,8 +157,10 @@ public static List getOffsetLags(NewKafkaSpoutOffsetQuery props.put("group.id", newKafkaSpoutOffsetQuery.getConsumerGroupId()); props.put("enable.auto.commit", "false"); props.put("session.timeout.ms", "30000"); - props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); - props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); + props.put("key.deserializer", + "org.apache.kafka.common.serialization.StringDeserializer"); + props.put("value.deserializer", + "org.apache.kafka.common.serialization.StringDeserializer"); if (newKafkaSpoutOffsetQuery.getSecurityProtocol() != null) { props.put("security.protocol", newKafkaSpoutOffsetQuery.getSecurityProtocol()); } @@ -158,7 +169,8 @@ public static List getOffsetLags(NewKafkaSpoutOffsetQuery } // Read property file for extra consumer properties if (newKafkaSpoutOffsetQuery.getConsumerPropertiesFileName() != null) { - props.putAll(Utils.loadProps(newKafkaSpoutOffsetQuery.getConsumerPropertiesFileName())); + props.putAll(Utils.loadProps(newKafkaSpoutOffsetQuery + .getConsumerPropertiesFileName())); } List topicPartitionList = new ArrayList<>(); consumer = new KafkaConsumer<>(props); @@ -166,17 +178,20 @@ public static List getOffsetLags(NewKafkaSpoutOffsetQuery List partitionInfoList = consumer.partitionsFor(topic); if (partitionInfoList != null) { for (PartitionInfo partitionInfo : partitionInfoList) { - topicPartitionList.add(new TopicPartition(partitionInfo.topic(), partitionInfo.partition())); + topicPartitionList.add(new TopicPartition(partitionInfo.topic(), + partitionInfo.partition())); } } } consumer.assign(topicPartitionList); - Map committedOffsets = consumer.committed(new HashSet<>(topicPartitionList)); + Map committedOffsets = consumer + .committed(new HashSet<>(topicPartitionList)); consumer.seekToEnd(topicPartitionList); for (TopicPartition topicPartition : topicPartitionList) { OffsetAndMetadata partitionOffset = committedOffsets.get(topicPartition); long committedOffset = partitionOffset != null ? partitionOffset.offset() : -1; - result.add(new KafkaOffsetLagResult(topicPartition.topic(), topicPartition.partition(), committedOffset, + result.add(new KafkaOffsetLagResult(topicPartition.topic(), topicPartition + .partition(), committedOffset, consumer.position(topicPartition))); } } finally { diff --git a/external/storm-kafka-monitor/src/main/java/org/apache/storm/kafka/monitor/NewKafkaSpoutOffsetQuery.java b/external/storm-kafka-monitor/src/main/java/org/apache/storm/kafka/monitor/NewKafkaSpoutOffsetQuery.java index c771a00bea0..4d9bc827a40 100644 --- a/external/storm-kafka-monitor/src/main/java/org/apache/storm/kafka/monitor/NewKafkaSpoutOffsetQuery.java +++ b/external/storm-kafka-monitor/src/main/java/org/apache/storm/kafka/monitor/NewKafkaSpoutOffsetQuery.java @@ -19,7 +19,8 @@ package org.apache.storm.kafka.monitor; /** - * Class representing information for querying kafka for log head offsets, consumer offsets and the difference for new + * Class representing information for querying kafka for log head offsets, consumer offsets and the + * difference for new * kafka spout using new consumer api. */ public class NewKafkaSpoutOffsetQuery { @@ -30,7 +31,8 @@ public class NewKafkaSpoutOffsetQuery { private final String saslMechanism; // Sasl mechanism to connect to kafka, default is GSSAPI private final String consumerPropertiesFileName; // properties file containing additional kafka consumer configs - public NewKafkaSpoutOffsetQuery(String topics, String bootstrapBrokers, String consumerGroupId, String securityProtocol, + public NewKafkaSpoutOffsetQuery(String topics, String bootstrapBrokers, String consumerGroupId, + String securityProtocol, String saslMechanism, String consumerPropertiesFileName) { this.topics = topics; this.bootStrapBrokers = bootstrapBrokers; @@ -117,7 +119,8 @@ public int hashCode() { result = 31 * result + (bootStrapBrokers != null ? bootStrapBrokers.hashCode() : 0); result = 31 * result + (securityProtocol != null ? securityProtocol.hashCode() : 0); result = 31 * result + (saslMechanism != null ? saslMechanism.hashCode() : 0); - result = 31 * result + (consumerPropertiesFileName != null ? consumerPropertiesFileName.hashCode() : 0); + result = 31 * result + (consumerPropertiesFileName != null ? consumerPropertiesFileName + .hashCode() : 0); return result; } } diff --git a/external/storm-kafka-monitor/src/test/java/org/apache/storm/kafka/monitor/KafkaOffsetLagUtilTest.java b/external/storm-kafka-monitor/src/test/java/org/apache/storm/kafka/monitor/KafkaOffsetLagUtilTest.java index c70a2db4f09..d9c1b49198c 100644 --- a/external/storm-kafka-monitor/src/test/java/org/apache/storm/kafka/monitor/KafkaOffsetLagUtilTest.java +++ b/external/storm-kafka-monitor/src/test/java/org/apache/storm/kafka/monitor/KafkaOffsetLagUtilTest.java @@ -9,8 +9,10 @@ * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -54,14 +56,16 @@ class KafkaOffsetLagUtilTest { private static final long COMMITTED_OFFSET_PARTITION_0 = 7L; @Container - private static final KafkaContainer KAFKA = new KafkaContainer(DockerImageName.parse("apache/kafka:4.0.0")); + private static final KafkaContainer KAFKA = new KafkaContainer(DockerImageName + .parse("apache/kafka:4.0.0")); @BeforeAll static void seedKafka() throws Exception { Properties adminProps = new Properties(); adminProps.put("bootstrap.servers", KAFKA.getBootstrapServers()); try (Admin admin = Admin.create(adminProps)) { - admin.createTopics(Collections.singletonList(new NewTopic(TOPIC, PARTITIONS, (short) 1))).all().get(); + admin.createTopics(Collections.singletonList(new NewTopic(TOPIC, PARTITIONS, + (short) 1))).all().get(); } produceOneRecordToEachPartition(); commitOffsetForPartitionZeroOnly(); @@ -82,7 +86,8 @@ void getOffsetLagsReportsCommittedOffsetForCommittedPartitionsAndMinusOneForUnco assertNotNull(committedByPartition.get(0)); assertEquals(COMMITTED_OFFSET_PARTITION_0, committedByPartition.get(0), "Partition with a committed offset should report it"); - // Regression assertion: pre-fix this NPE'd inside the monitor and surfaced as ClassCastException upstream. + // Regression assertion: pre-fix this NPE'd inside the monitor and surfaced as + // ClassCastException upstream. assertEquals(-1L, committedByPartition.get(1), "Partition with no committed offset should report -1, not throw"); } @@ -90,8 +95,10 @@ void getOffsetLagsReportsCommittedOffsetForCommittedPartitionsAndMinusOneForUnco private static void produceOneRecordToEachPartition() { Properties producerProps = new Properties(); producerProps.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, KAFKA.getBootstrapServers()); - producerProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); - producerProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); + producerProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class + .getName()); + producerProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class + .getName()); try (KafkaProducer producer = new KafkaProducer<>(producerProps)) { for (int p = 0; p < PARTITIONS; p++) { producer.send(new ProducerRecord<>(TOPIC, p, "k", "v")); @@ -105,11 +112,14 @@ private static void commitOffsetForPartitionZeroOnly() { consumerProps.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, KAFKA.getBootstrapServers()); consumerProps.put(ConsumerConfig.GROUP_ID_CONFIG, GROUP_ID); consumerProps.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false"); - consumerProps.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); - consumerProps.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); + consumerProps.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class + .getName()); + consumerProps.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class + .getName()); try (KafkaConsumer consumer = new KafkaConsumer<>(consumerProps)) { TopicPartition p0 = new TopicPartition(TOPIC, 0); - consumer.commitSync(Collections.singletonMap(p0, new OffsetAndMetadata(COMMITTED_OFFSET_PARTITION_0))); + consumer.commitSync(Collections.singletonMap(p0, + new OffsetAndMetadata(COMMITTED_OFFSET_PARTITION_0))); } } } diff --git a/external/storm-metrics-prometheus/src/main/java/org/apache/storm/metrics/prometheus/PrometheusPreparableReporter.java b/external/storm-metrics-prometheus/src/main/java/org/apache/storm/metrics/prometheus/PrometheusPreparableReporter.java index d268528e6b5..213fcd9a08d 100644 --- a/external/storm-metrics-prometheus/src/main/java/org/apache/storm/metrics/prometheus/PrometheusPreparableReporter.java +++ b/external/storm-metrics-prometheus/src/main/java/org/apache/storm/metrics/prometheus/PrometheusPreparableReporter.java @@ -1,37 +1,40 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.metrics.prometheus; +import com.codahale.metrics.MetricRegistry; +import io.prometheus.metrics.exporter.pushgateway.HttpConnectionFactory; +import io.prometheus.metrics.exporter.pushgateway.PushGateway; +import io.prometheus.metrics.exporter.pushgateway.Scheme; import java.security.KeyManagementException; import java.security.NoSuchAlgorithmException; import java.security.cert.X509Certificate; import java.util.Map; import java.util.concurrent.TimeUnit; - -import com.codahale.metrics.MetricRegistry; -import io.prometheus.metrics.exporter.pushgateway.HttpConnectionFactory; -import io.prometheus.metrics.exporter.pushgateway.PushGateway; -import io.prometheus.metrics.exporter.pushgateway.Scheme; +import javax.net.ssl.HttpsURLConnection; +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManager; +import javax.net.ssl.X509TrustManager; import org.apache.storm.DaemonConfig; import org.apache.storm.daemon.metrics.reporters.PreparableReporter; import org.apache.storm.utils.ObjectReader; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.net.ssl.HttpsURLConnection; -import javax.net.ssl.SSLContext; -import javax.net.ssl.TrustManager; -import javax.net.ssl.X509TrustManager; - public class PrometheusPreparableReporter implements PreparableReporter { private static final Logger LOG = LoggerFactory.getLogger(PrometheusPreparableReporter.class); @@ -77,21 +80,32 @@ protected PrometheusReporterClient getReporter() { public void prepare(MetricRegistry metricsRegistry, Map daemonConf) { if (daemonConf != null) { - final String jobName = (String) daemonConf.getOrDefault("storm.daemon.metrics.reporter.plugin.prometheus.job", "storm"); - final String endpoint = (String) daemonConf.getOrDefault("storm.daemon.metrics.reporter.plugin.prometheus.endpoint", "localhost:9091"); - final String schemeAsString = (String) daemonConf.getOrDefault("storm.daemon.metrics.reporter.plugin.prometheus.scheme", "http"); + final String jobName = (String) daemonConf + .getOrDefault("storm.daemon.metrics.reporter.plugin.prometheus.job", "storm"); + final String endpoint = (String) daemonConf + .getOrDefault("storm.daemon.metrics.reporter.plugin.prometheus.endpoint", + "localhost:9091"); + final String schemeAsString = (String) daemonConf + .getOrDefault("storm.daemon.metrics.reporter.plugin.prometheus.scheme", "http"); Scheme scheme = Scheme.HTTP; try { scheme = Scheme.fromString(schemeAsString); } catch (IllegalArgumentException iae) { - LOG.warn("Unsupported scheme. Expecting 'http' or 'https'. Was: {}", schemeAsString); + LOG.warn("Unsupported scheme. Expecting 'http' or 'https'. Was: {}", + schemeAsString); } - final String basicAuthUser = (String) daemonConf.getOrDefault("storm.daemon.metrics.reporter.plugin.prometheus.basic_auth_user", ""); - final String basicAuthPassword = (String) daemonConf.getOrDefault("storm.daemon.metrics.reporter.plugin.prometheus.basic_auth_password", ""); - final boolean skipTlsValidation = (boolean) daemonConf.getOrDefault("storm.daemon.metrics.reporter.plugin.prometheus.skip_tls_validation", false); + final String basicAuthUser = (String) daemonConf + .getOrDefault("storm.daemon.metrics.reporter.plugin.prometheus.basic_auth_user", + ""); + final String basicAuthPassword = (String) daemonConf + .getOrDefault("storm.daemon.metrics.reporter.plugin.prometheus.basic_auth_pass" + + "word", ""); + final boolean skipTlsValidation = (boolean) daemonConf + .getOrDefault("storm.daemon.metrics.reporter.plugin.prometheus.skip_tls_valida" + + "tion", false); final PushGateway.Builder builder = PushGateway.builder(); @@ -105,7 +119,8 @@ public void prepare(MetricRegistry metricsRegistry, Map daemonCo if (scheme == Scheme.HTTPS && skipTlsValidation) { LOG.warn("TLS validation is DISABLED for the Prometheus PushGateway connection " - + "(storm.daemon.metrics.reporter.plugin.prometheus.skip_tls_validation=true). " + + "(storm.daemon.metrics.reporter.plugin.prometheus.skip_tls_validation=tr" + + "ue). " + "This is insecure and must not be used in production."); builder.connectionFactory(INSECURE_CONNECTION_FACTORY); } @@ -113,20 +128,21 @@ public void prepare(MetricRegistry metricsRegistry, Map daemonCo final PushGateway pushGateway = builder.build(); reporter = new PrometheusReporterClient(metricsRegistry, pushGateway); - reportingIntervalSecs = ObjectReader.getInt(daemonConf.get(DaemonConfig.STORM_DAEMON_METRICS_REPORTER_INTERVAL_SECS), 10); + reportingIntervalSecs = ObjectReader.getInt(daemonConf + .get(DaemonConfig.STORM_DAEMON_METRICS_REPORTER_INTERVAL_SECS), 10); } else { LOG.warn("No daemonConfiguration was supplied. Don't initialize."); } } - @Override public void start() { if (reporter != null) { LOG.debug("Starting..."); reporter.start(reportingIntervalSecs, TimeUnit.SECONDS); } else { - throw new IllegalStateException("Attempt to start without preparing " + getClass().getSimpleName()); + throw new IllegalStateException("Attempt to start without preparing " + getClass() + .getSimpleName()); } } @@ -137,7 +153,8 @@ public void stop() { reporter.report(); reporter.stop(); } else { - throw new IllegalStateException("Attempt to stop without preparing " + getClass().getSimpleName()); + throw new IllegalStateException("Attempt to stop without preparing " + getClass() + .getSimpleName()); } } } diff --git a/external/storm-metrics-prometheus/src/main/java/org/apache/storm/metrics/prometheus/PrometheusReporterClient.java b/external/storm-metrics-prometheus/src/main/java/org/apache/storm/metrics/prometheus/PrometheusReporterClient.java index 20a8ddd20b4..01734fa5315 100644 --- a/external/storm-metrics-prometheus/src/main/java/org/apache/storm/metrics/prometheus/PrometheusReporterClient.java +++ b/external/storm-metrics-prometheus/src/main/java/org/apache/storm/metrics/prometheus/PrometheusReporterClient.java @@ -1,12 +1,17 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.metrics.prometheus; @@ -21,15 +26,14 @@ import com.codahale.metrics.Snapshot; import com.codahale.metrics.Timer; import io.prometheus.metrics.exporter.pushgateway.PushGateway; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.IOException; import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.SortedMap; import java.util.concurrent.TimeUnit; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * This reporter pushes common cluster metrics towards a Prometheus Pushgateway. @@ -53,12 +57,14 @@ public class PrometheusReporterClient extends ScheduledReporter { * via a transport protocol */ protected PrometheusReporterClient(MetricRegistry registry, PushGateway prometheus) { - super(registry, "prometheus-reporter", MetricFilter.ALL, RATE_UNIT, DURATION_UNIT, null, true, Collections.emptySet()); + super(registry, "prometheus-reporter", MetricFilter.ALL, RATE_UNIT, DURATION_UNIT, null, + true, Collections.emptySet()); this.prometheus = prometheus; } @Override - public void report(SortedMap gauges, SortedMap counters, SortedMap histograms, SortedMap meters, SortedMap timers) { + public void report(SortedMap gauges, SortedMap counters, + SortedMap histograms, SortedMap meters, SortedMap timers) { try { if (CLUSTER_SUMMARY_METRICS.isEmpty()) { initClusterMetrics(); @@ -66,18 +72,23 @@ public void report(SortedMap gauges, SortedMap c for (Map.Entry e : gauges.entrySet()) { - final io.prometheus.metrics.core.metrics.Gauge pGauge = (io.prometheus.metrics.core.metrics.Gauge) CLUSTER_SUMMARY_METRICS.get(e.getKey()); + final io.prometheus.metrics.core.metrics.Gauge pGauge = + (io.prometheus.metrics.core.metrics.Gauge) CLUSTER_SUMMARY_METRICS.get(e + .getKey()); if (pGauge != null) { try { pGauge.set(toDouble(e.getValue().getValue())); } catch (NumberFormatException ignored) { - LOG.warn("Invalid type for Gauge {}: {}", e.getKey(), e.getValue().getClass().getName()); + LOG.warn("Invalid type for Gauge {}: {}", e.getKey(), e.getValue() + .getClass().getName()); } } } for (Map.Entry e : histograms.entrySet()) { - final io.prometheus.metrics.core.metrics.Histogram pHisto = (io.prometheus.metrics.core.metrics.Histogram) CLUSTER_SUMMARY_METRICS.get(e.getKey()); + final io.prometheus.metrics.core.metrics.Histogram pHisto = + (io.prometheus.metrics.core.metrics.Histogram) CLUSTER_SUMMARY_METRICS.get(e + .getKey()); if (pHisto != null) { final Snapshot s = e.getValue().getSnapshot(); for (double d : s.getValues()) { @@ -109,147 +120,180 @@ private double toDouble(Object obj) { } private static void initClusterMetrics() { - CLUSTER_SUMMARY_METRICS.put("summary.cluster:num-nimbus-leaders", io.prometheus.metrics.core.metrics.Gauge.builder() + CLUSTER_SUMMARY_METRICS.put("summary.cluster:num-nimbus-leaders", + io.prometheus.metrics.core.metrics.Gauge.builder() .name("summary_cluster_num_nimbus_leaders") - .help("Number of nimbuses marked as a leader. This should really only ever be 1 in a healthy cluster, or 0 for a short period of time while a fail over happens.") + .help("Number of nimbuses marked as a leader. This should really only ever be 1 " + + "in a healthy cluster, or 0 for a short period of time while a fail over " + + "happens.") .register()); - CLUSTER_SUMMARY_METRICS.put("summary.cluster:num-nimbuses", io.prometheus.metrics.core.metrics.Gauge.builder() + CLUSTER_SUMMARY_METRICS.put("summary.cluster:num-nimbuses", + io.prometheus.metrics.core.metrics.Gauge.builder() .name("summary_cluster_num_nimbuses") .help("Number of nimbuses, leader or standby.") .register()); - CLUSTER_SUMMARY_METRICS.put("summary.cluster:num-supervisors", io.prometheus.metrics.core.metrics.Gauge.builder() + CLUSTER_SUMMARY_METRICS.put("summary.cluster:num-supervisors", + io.prometheus.metrics.core.metrics.Gauge.builder() .name("summary_cluster_num_supervisors") .help("Number of supervisors.") .register()); - CLUSTER_SUMMARY_METRICS.put("summary.cluster:num-topologies", io.prometheus.metrics.core.metrics.Gauge.builder() + CLUSTER_SUMMARY_METRICS.put("summary.cluster:num-topologies", + io.prometheus.metrics.core.metrics.Gauge.builder() .name("summary_cluster_num_topologies") .help("Number of topologies.") .register()); - CLUSTER_SUMMARY_METRICS.put("summary.cluster:num-total-used-workers", io.prometheus.metrics.core.metrics.Gauge.builder() + CLUSTER_SUMMARY_METRICS.put("summary.cluster:num-total-used-workers", + io.prometheus.metrics.core.metrics.Gauge.builder() .name("summary_cluster_num_total_used_workers") .help("Number of used workers/slots.") .register()); - CLUSTER_SUMMARY_METRICS.put("summary.cluster:num-total-workers", io.prometheus.metrics.core.metrics.Gauge.builder() + CLUSTER_SUMMARY_METRICS.put("summary.cluster:num-total-workers", + io.prometheus.metrics.core.metrics.Gauge.builder() .name("summary_cluster_num_total_workers") .help("Number of workers/slots.") .register()); - CLUSTER_SUMMARY_METRICS.put("summary.cluster:total-fragmented-cpu-non-negative", io.prometheus.metrics.core.metrics.Gauge.builder() + CLUSTER_SUMMARY_METRICS.put("summary.cluster:total-fragmented-cpu-non-negative", + io.prometheus.metrics.core.metrics.Gauge.builder() .name("summary_cluster_total_fragmented_cpu_non_negative") - .help("Total fragmented CPU (% of core). This is CPU that the system thinks it cannot use because other resources on the node are used up.") + .help("Total fragmented CPU (% of core). This is CPU that the system thinks it " + + "cannot use because other resources on the node are used up.") .register()); - CLUSTER_SUMMARY_METRICS.put("summary.cluster:total-fragmented-memory-non-negative", io.prometheus.metrics.core.metrics.Gauge.builder() + CLUSTER_SUMMARY_METRICS.put("summary.cluster:total-fragmented-memory-non-negative", + io.prometheus.metrics.core.metrics.Gauge.builder() .name("summary_cluster_total_fragmented_memory_non_negative") - .help("Total fragmented memory (MB). This is memory that the system thinks it cannot use because other resources on the node are used up.") + .help("Total fragmented memory (MB). This is memory that the system thinks it " + + "cannot use because other resources on the node are used up.") .register()); - CLUSTER_SUMMARY_METRICS.put("nimbus:available-cpu-non-negative", io.prometheus.metrics.core.metrics.Gauge.builder() + CLUSTER_SUMMARY_METRICS.put("nimbus:available-cpu-non-negative", + io.prometheus.metrics.core.metrics.Gauge.builder() .name("nimbus_available_cpu_non_negative") .help("Available cpu on the cluster (% of a core).") .register()); - CLUSTER_SUMMARY_METRICS.put("nimbus:total-cpu", io.prometheus.metrics.core.metrics.Gauge.builder() + CLUSTER_SUMMARY_METRICS.put("nimbus:total-cpu", io.prometheus.metrics.core.metrics.Gauge + .builder() .name("nimbus_total_cpu") .help("total CPU on the cluster (% of a core)") .register()); - CLUSTER_SUMMARY_METRICS.put("nimbus:total-memory", io.prometheus.metrics.core.metrics.Gauge.builder() + CLUSTER_SUMMARY_METRICS.put("nimbus:total-memory", io.prometheus.metrics.core.metrics.Gauge + .builder() .name("nimbus_total_memory") .help("total memory on the cluster MB") .register()); - CLUSTER_SUMMARY_METRICS.put("summary.topologies:assigned-cpu", io.prometheus.metrics.core.metrics.Histogram.builder() + CLUSTER_SUMMARY_METRICS.put("summary.topologies:assigned-cpu", + io.prometheus.metrics.core.metrics.Histogram.builder() .name("summary_topologies_assigned_cpu") .help("CPU scheduled per topology (% of a core)") .register()); - CLUSTER_SUMMARY_METRICS.put("summary.topologies:assigned-mem-off-heap", io.prometheus.metrics.core.metrics.Histogram.builder() + CLUSTER_SUMMARY_METRICS.put("summary.topologies:assigned-mem-off-heap", + io.prometheus.metrics.core.metrics.Histogram.builder() .name("summary_topologies_assigned_mem_off_heap") .help("Off heap memory scheduled per topology (MB)") .register()); - CLUSTER_SUMMARY_METRICS.put("summary.topologies:assigned-mem-on-heap", io.prometheus.metrics.core.metrics.Histogram.builder() + CLUSTER_SUMMARY_METRICS.put("summary.topologies:assigned-mem-on-heap", + io.prometheus.metrics.core.metrics.Histogram.builder() .name("summary_topologies_assigned_mem_on_heap") .help("On heap memory scheduled per topology (MB)") .register()); - CLUSTER_SUMMARY_METRICS.put("summary.topologies:num-executors", io.prometheus.metrics.core.metrics.Histogram.builder() + CLUSTER_SUMMARY_METRICS.put("summary.topologies:num-executors", + io.prometheus.metrics.core.metrics.Histogram.builder() .name("summary_topologies_num_executors") .help("Number of executors per topology") .register()); - CLUSTER_SUMMARY_METRICS.put("summary.topologies:num-tasks", io.prometheus.metrics.core.metrics.Histogram.builder() + CLUSTER_SUMMARY_METRICS.put("summary.topologies:num-tasks", + io.prometheus.metrics.core.metrics.Histogram.builder() .name("summary_topologies_num_tasks") .help("Number of tasks per topology") .register()); - CLUSTER_SUMMARY_METRICS.put("summary.topologies:num-workers", io.prometheus.metrics.core.metrics.Histogram.builder() + CLUSTER_SUMMARY_METRICS.put("summary.topologies:num-workers", + io.prometheus.metrics.core.metrics.Histogram.builder() .name("summary_topologies_num_workers") .help("Number of workers per topology") .register()); - CLUSTER_SUMMARY_METRICS.put("summary.topologies:replication-count", io.prometheus.metrics.core.metrics.Histogram.builder() + CLUSTER_SUMMARY_METRICS.put("summary.topologies:replication-count", + io.prometheus.metrics.core.metrics.Histogram.builder() .name("summary_topologies_replication_count") .help("Replication count per topology") .register()); - CLUSTER_SUMMARY_METRICS.put("summary.topologies:requested-cpu", io.prometheus.metrics.core.metrics.Histogram.builder() + CLUSTER_SUMMARY_METRICS.put("summary.topologies:requested-cpu", + io.prometheus.metrics.core.metrics.Histogram.builder() .name("summary_topologies_requested_cpu") .help("CPU requested per topology (% of a core)") .register()); - CLUSTER_SUMMARY_METRICS.put("summary.topologies:requested-mem-off-heap", io.prometheus.metrics.core.metrics.Histogram.builder() + CLUSTER_SUMMARY_METRICS.put("summary.topologies:requested-mem-off-heap", + io.prometheus.metrics.core.metrics.Histogram.builder() .name("summary_topologies_requested_mem_off_heap") .help("Off heap memory requested per topology (MB)") .register()); - CLUSTER_SUMMARY_METRICS.put("summary.topologies:requested-mem-on-heap", io.prometheus.metrics.core.metrics.Histogram.builder() + CLUSTER_SUMMARY_METRICS.put("summary.topologies:requested-mem-on-heap", + io.prometheus.metrics.core.metrics.Histogram.builder() .name("summary_topologies_requested_mem_on_heap") .help("On heap memory requested per topology (MB)") .register()); - CLUSTER_SUMMARY_METRICS.put("summary.topologies:uptime-secs", io.prometheus.metrics.core.metrics.Histogram.builder() + CLUSTER_SUMMARY_METRICS.put("summary.topologies:uptime-secs", + io.prometheus.metrics.core.metrics.Histogram.builder() .name("summary_topologies_uptime_secs") .help("Uptime per topology (seconds)") .register()); - CLUSTER_SUMMARY_METRICS.put("summary.supervisors:fragmented-cpu", io.prometheus.metrics.core.metrics.Histogram.builder() + CLUSTER_SUMMARY_METRICS.put("summary.supervisors:fragmented-cpu", + io.prometheus.metrics.core.metrics.Histogram.builder() .name("supervisors_fragmented_cpu") .help("fragmented CPU per supervisor (% of a core)") .register()); - CLUSTER_SUMMARY_METRICS.put("summary.supervisors:fragmented-mem", io.prometheus.metrics.core.metrics.Histogram.builder() + CLUSTER_SUMMARY_METRICS.put("summary.supervisors:fragmented-mem", + io.prometheus.metrics.core.metrics.Histogram.builder() .name("supervisors_fragmented_mem") .help("fragmented memory per supervisor (MB)") .register()); - CLUSTER_SUMMARY_METRICS.put("summary.supervisors:num-used-workers", io.prometheus.metrics.core.metrics.Histogram.builder() + CLUSTER_SUMMARY_METRICS.put("summary.supervisors:num-used-workers", + io.prometheus.metrics.core.metrics.Histogram.builder() .name("supervisors_num_used_workers") .help("workers used per supervisor") .register()); - CLUSTER_SUMMARY_METRICS.put("summary.supervisors:num-workers", io.prometheus.metrics.core.metrics.Histogram.builder() + CLUSTER_SUMMARY_METRICS.put("summary.supervisors:num-workers", + io.prometheus.metrics.core.metrics.Histogram.builder() .name("supervisors_num_workers") .help("number of workers per supervisor") .register()); - CLUSTER_SUMMARY_METRICS.put("summary.supervisors:uptime-secs", io.prometheus.metrics.core.metrics.Histogram.builder() + CLUSTER_SUMMARY_METRICS.put("summary.supervisors:uptime-secs", + io.prometheus.metrics.core.metrics.Histogram.builder() .name("supervisors_uptime_secs") .help("uptime of supervisors") .register()); - CLUSTER_SUMMARY_METRICS.put("summary.supervisors:used-cpu", io.prometheus.metrics.core.metrics.Histogram.builder() + CLUSTER_SUMMARY_METRICS.put("summary.supervisors:used-cpu", + io.prometheus.metrics.core.metrics.Histogram.builder() .name("supervisors_used_cpu") .help("CPU used per supervisor (% of a core)") .register()); - CLUSTER_SUMMARY_METRICS.put("summary.supervisors:used-mem", io.prometheus.metrics.core.metrics.Histogram.builder() + CLUSTER_SUMMARY_METRICS.put("summary.supervisors:used-mem", + io.prometheus.metrics.core.metrics.Histogram.builder() .name("supervisors_used_mem") .help("memory used per supervisor (MB)") .register()); diff --git a/external/storm-metrics-prometheus/src/test/java/org/apache/storm/metrics/prometheus/PrometheusPreparableReporterTest.java b/external/storm-metrics-prometheus/src/test/java/org/apache/storm/metrics/prometheus/PrometheusPreparableReporterTest.java index 9e33be80fde..21e32deccb1 100644 --- a/external/storm-metrics-prometheus/src/test/java/org/apache/storm/metrics/prometheus/PrometheusPreparableReporterTest.java +++ b/external/storm-metrics-prometheus/src/test/java/org/apache/storm/metrics/prometheus/PrometheusPreparableReporterTest.java @@ -1,31 +1,23 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

    - * http://www.apache.org/licenses/LICENSE-2.0 - *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

    http://www.apache.org/licenses/LICENSE-2.0 + * + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.metrics.prometheus; import com.codahale.metrics.MetricRegistry; -import org.apache.storm.metrics2.SimpleGauge; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.testcontainers.containers.GenericContainer; -import org.testcontainers.containers.wait.strategy.Wait; -import org.testcontainers.junit.jupiter.Testcontainers; -import org.testcontainers.utility.MountableFile; - -import javax.net.ssl.HttpsURLConnection; -import javax.net.ssl.SSLContext; -import javax.net.ssl.TrustManager; -import javax.net.ssl.X509TrustManager; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; @@ -39,6 +31,19 @@ import java.util.List; import java.util.Map; import java.util.Set; +import javax.net.ssl.HttpsURLConnection; +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManager; +import javax.net.ssl.X509TrustManager; +import org.apache.storm.metrics2.SimpleGauge; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.utility.MountableFile; @Testcontainers(disabledWithoutDocker = true) public class PrometheusPreparableReporterTest { @@ -65,7 +70,8 @@ public void testSimple() throws IOException { final Map daemonConf = Map.of( "storm.daemon.metrics.reporter.plugin.prometheus.job", "test_simple", - "storm.daemon.metrics.reporter.plugin.prometheus.endpoint", "localhost:" + pushGatewayContainer.getMappedPort(9091), + "storm.daemon.metrics.reporter.plugin.prometheus.endpoint", "localhost:" + + pushGatewayContainer.getMappedPort(9091), "storm.daemon.metrics.reporter.plugin.prometheus.scheme", "http" ); @@ -76,7 +82,9 @@ public void testSimple() throws IOException { @Test public void testBasicAuth() throws IOException { pushGatewayContainer - .withCopyFileToContainer(MountableFile.forClasspathResource("/pushgateway-basicauth.yaml"), "/pushgateway/pushgateway-basicauth.yaml") + .withCopyFileToContainer(MountableFile + .forClasspathResource("/pushgateway-basicauth.yaml"), "/pushgateway/pushgateway-ba" + + "sicauth.yaml") .withCommand("--web.config.file", "pushgateway-basicauth.yaml") .start(); @@ -84,10 +92,12 @@ public void testBasicAuth() throws IOException { final Map daemonConf = Map.of( "storm.daemon.metrics.reporter.plugin.prometheus.job", "test_simple", - "storm.daemon.metrics.reporter.plugin.prometheus.endpoint", "localhost:" + pushGatewayContainer.getMappedPort(9091), + "storm.daemon.metrics.reporter.plugin.prometheus.endpoint", "localhost:" + + pushGatewayContainer.getMappedPort(9091), "storm.daemon.metrics.reporter.plugin.prometheus.scheme", "http", "storm.daemon.metrics.reporter.plugin.prometheus.basic_auth_user", "my_user", - "storm.daemon.metrics.reporter.plugin.prometheus.basic_auth_password", "secret_password" + "storm.daemon.metrics.reporter.plugin.prometheus.basic_auth_password", "secret_pas" + + "sword" ); runTest(sut, daemonConf); @@ -96,7 +106,8 @@ public void testBasicAuth() throws IOException { @Test public void testTls() throws IOException { pushGatewayContainer - .withCopyFileToContainer(MountableFile.forClasspathResource("/pushgateway-ssl.yaml"), "/pushgateway/pushgateway-ssl.yaml") + .withCopyFileToContainer(MountableFile + .forClasspathResource("/pushgateway-ssl.yaml"), "/pushgateway/pushgateway-ssl.yaml") .withCommand("--web.config.file", "pushgateway-ssl.yaml") .start(); @@ -104,7 +115,8 @@ public void testTls() throws IOException { final Map daemonConf = Map.of( "storm.daemon.metrics.reporter.plugin.prometheus.job", "test_simple", - "storm.daemon.metrics.reporter.plugin.prometheus.endpoint", "localhost:" + pushGatewayContainer.getMappedPort(9091), + "storm.daemon.metrics.reporter.plugin.prometheus.endpoint", "localhost:" + + pushGatewayContainer.getMappedPort(9091), "storm.daemon.metrics.reporter.plugin.prometheus.scheme", "https", "storm.daemon.metrics.reporter.plugin.prometheus.skip_tls_validation", true ); @@ -112,8 +124,8 @@ public void testTls() throws IOException { runTest(sut, daemonConf); } - - private void runTest(PrometheusPreparableReporter sut, Map daemonConf) throws IOException { + private void runTest(PrometheusPreparableReporter sut, Map daemonConf) throws IOException { // We fake the metrics here. In a real Storm environment, these metrics are generated. final MetricRegistry r = new MetricRegistry(); final SimpleGauge supervisor = new SimpleGauge<>(5); @@ -123,7 +135,8 @@ private void runTest(PrometheusPreparableReporter sut, Map daemo sut.prepare(r, daemonConf); - //manually trigger a reporting here, in a real Storm environment, this is called by a scheduled executor. + // manually trigger a reporting here, in a real Storm environment, this is called by a + // scheduled executor. sut.getReporter().report(); assertMetrics( @@ -137,12 +150,15 @@ private void runTest(PrometheusPreparableReporter sut, Map daemo "# HELP nimbus_total_cpu total CPU on the cluster (% of a core)", "# TYPE nimbus_total_cpu gauge", "nimbus_total_cpu{instance=\"\",job=\"test_simple\"} 500" - ), daemonConf.get("storm.daemon.metrics.reporter.plugin.prometheus.scheme") + "://" + daemonConf.get("storm.daemon.metrics.reporter.plugin.prometheus.endpoint") + "/metrics", daemonConf); + ), daemonConf.get("storm.daemon.metrics.reporter.plugin.prometheus.scheme") + "://" + + daemonConf.get("storm.daemon.metrics.reporter.plugin.prometheus.endpoint") + + "/metrics", daemonConf); - //update a metric + // update a metric supervisor.set(100); - //manually trigger a reporting here, in a real Storm environment, this is called by a scheduled executor. + // manually trigger a reporting here, in a real Storm environment, this is called by a + // scheduled executor. sut.getReporter().report(); assertMetrics( @@ -156,14 +172,18 @@ private void runTest(PrometheusPreparableReporter sut, Map daemo "# HELP nimbus_total_cpu total CPU on the cluster (% of a core)", "# TYPE nimbus_total_cpu gauge", "nimbus_total_cpu{instance=\"\",job=\"test_simple\"} 500" - ), daemonConf.get("storm.daemon.metrics.reporter.plugin.prometheus.scheme") + "://" + daemonConf.get("storm.daemon.metrics.reporter.plugin.prometheus.endpoint") + "/metrics", daemonConf); + ), daemonConf.get("storm.daemon.metrics.reporter.plugin.prometheus.scheme") + "://" + + daemonConf.get("storm.daemon.metrics.reporter.plugin.prometheus.endpoint") + + "/metrics", daemonConf); } - private void assertMetrics(List elements, String endpoint, Map conf) throws IOException { + private void assertMetrics(List elements, String endpoint, Map conf) throws IOException { final String content = readContent(endpoint, conf); Assertions.assertNotNull(content); final Set contentLinesSet = new HashSet<>(Arrays.asList(content.split("\n"))); - elements.forEach(find -> Assertions.assertTrue(contentLinesSet.contains(find), "Did not find: " + find)); + elements.forEach(find -> Assertions.assertTrue(contentLinesSet.contains(find), + "Did not find: " + find)); } private String readContent(String url, Map conf) throws IOException { @@ -187,7 +207,7 @@ public void checkClientTrusted(X509Certificate[] chain, String authType) { @Override public void checkServerTrusted(X509Certificate[] chain, String authType) { } - }}, null); + } }, null); ((HttpsURLConnection) con).setSSLSocketFactory(sslContext.getSocketFactory()); ((HttpsURLConnection) con).setHostnameVerifier((h, s) -> true); } catch (GeneralSecurityException e) { @@ -198,7 +218,10 @@ public void checkServerTrusted(X509Certificate[] chain, String authType) { if (conf.containsKey("storm.daemon.metrics.reporter.plugin.prometheus.basic_auth_user")) { - String auth = conf.get("storm.daemon.metrics.reporter.plugin.prometheus.basic_auth_user") + ":" + conf.get("storm.daemon.metrics.reporter.plugin.prometheus.basic_auth_password"); + String auth = conf + .get("storm.daemon.metrics.reporter.plugin.prometheus.basic_auth_user") + ":" + + conf.get("storm.daemon.metrics.reporter.plugin.prometheus.basic_auth_passwor" + + "d"); String encodedAuth = Base64.getEncoder().encodeToString(auth.getBytes()); String authHeaderValue = "Basic " + encodedAuth; con.setRequestProperty("Authorization", authHeaderValue); diff --git a/external/storm-metrics/pom.xml b/external/storm-metrics/pom.xml index 31197865bdd..da14dbce237 100644 --- a/external/storm-metrics/pom.xml +++ b/external/storm-metrics/pom.xml @@ -99,6 +99,16 @@ + + org.openrewrite.maven + rewrite-maven-plugin + + + + org.codehaus.gmavenplus + gmavenplus-plugin + + org.apache.maven.plugins maven-checkstyle-plugin diff --git a/external/storm-metrics/src/main/java/org/apache/storm/metrics/hdrhistogram/HistogramMetric.java b/external/storm-metrics/src/main/java/org/apache/storm/metrics/hdrhistogram/HistogramMetric.java index 99b21c6e80c..ea577f7ddfe 100644 --- a/external/storm-metrics/src/main/java/org/apache/storm/metrics/hdrhistogram/HistogramMetric.java +++ b/external/storm-metrics/src/main/java/org/apache/storm/metrics/hdrhistogram/HistogramMetric.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

    Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -37,20 +37,27 @@ public HistogramMetric(Long highestTrackableValue, final int numberOfSignificant /** * (From the Constructor of Histogram) - * Construct a Histogram given the Lowest and Highest values to be tracked and a number of significant - * decimal digits. Providing a lowestDiscernibleValue is useful is situations where the units used - * for the histogram's values are much smaller that the minimal accuracy required. E.g. when tracking - * time values stated in nanosecond units, where the minimal accuracy required is a microsecond, the + * Construct a Histogram given the Lowest and Highest values to be tracked and a number of + * significant + * decimal digits. Providing a lowestDiscernibleValue is useful is situations where the units + * used + * for the histogram's values are much smaller that the minimal accuracy required. E.g. when + * tracking + * time values stated in nanosecond units, where the minimal accuracy required is a microsecond, + * the * proper value for lowestDiscernibleValue would be 1000. * - * @param lowestDiscernibleValue The lowest value that can be discerned (distinguished from 0) by the + * @param lowestDiscernibleValue The lowest value that can be discerned (distinguished from 0) + * by the * histogram. Must be a positive integer that is {@literal >=} 1. May be * internally rounded down to nearest power of 2 * (if null 1 is used). - * @param highestTrackableValue The highest value to be tracked by the histogram. Must be a positive + * @param highestTrackableValue The highest value to be tracked by the histogram. Must be a + * positive * integer that is {@literal >=} (2 * lowestDiscernibleValue). * (if null 2 * lowestDiscernibleValue is used and auto-resize is enabled) - * @param numberOfSignificantValueDigits Specifies the precision to use. This is the number of significant + * @param numberOfSignificantValueDigits Specifies the precision to use. This is the number of + * significant * decimal digits to which the histogram will maintain value resolution * and separation. Must be a non-negative integer between 0 and 5. */ @@ -64,7 +71,8 @@ public HistogramMetric(Long lowestDiscernibleValue, Long highestTrackableValue, highestTrackableValue = 2 * lowestDiscernibleValue; autoResize = true; } - histo = new Histogram(lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits); + histo = new Histogram(lowestDiscernibleValue, highestTrackableValue, + numberOfSignificantValueDigits); if (autoResize) { histo.setAutoResize(true); } diff --git a/external/storm-metrics/src/main/java/org/apache/storm/metrics/sigar/CPUMetric.java b/external/storm-metrics/src/main/java/org/apache/storm/metrics/sigar/CPUMetric.java index 59dac4216c1..477275b97f9 100644 --- a/external/storm-metrics/src/main/java/org/apache/storm/metrics/sigar/CPUMetric.java +++ b/external/storm-metrics/src/main/java/org/apache/storm/metrics/sigar/CPUMetric.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

    Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -19,9 +19,7 @@ package org.apache.storm.metrics.sigar; import java.util.HashMap; - import org.apache.storm.metric.api.IMetric; - import org.hyperic.sigar.ProcCpu; import org.hyperic.sigar.Sigar; diff --git a/external/storm-redis/pom.xml b/external/storm-redis/pom.xml index 0c240cea90b..9a380b24b48 100644 --- a/external/storm-redis/pom.xml +++ b/external/storm-redis/pom.xml @@ -100,6 +100,16 @@ + + org.openrewrite.maven + rewrite-maven-plugin + + + + org.codehaus.gmavenplus + gmavenplus-plugin + + org.apache.maven.plugins maven-checkstyle-plugin diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/bolt/AbstractRedisBolt.java b/external/storm-redis/src/main/java/org/apache/storm/redis/bolt/AbstractRedisBolt.java index 0a7d293d1ce..28153a4de51 100644 --- a/external/storm-redis/src/main/java/org/apache/storm/redis/bolt/AbstractRedisBolt.java +++ b/external/storm-redis/src/main/java/org/apache/storm/redis/bolt/AbstractRedisBolt.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -22,9 +28,11 @@ import org.apache.storm.topology.base.BaseTickTupleAwareRichBolt; /** - * AbstractRedisBolt class is for users to implement custom bolts which makes interaction with Redis. + * AbstractRedisBolt class is for users to implement custom bolts which makes interaction with + * Redis. * - *

    Due to environment abstraction, AbstractRedisBolt provides JedisCommands which contains only single key operations. + *

    Due to environment abstraction, AbstractRedisBolt provides JedisCommands which contains only + * single key operations. * *

    Custom Bolts may want to follow this pattern: * @@ -71,7 +79,8 @@ public AbstractRedisBolt(JedisClusterConfig config) { * {@inheritDoc} */ @Override - public void prepare(Map map, TopologyContext topologyContext, OutputCollector collector) { + public void prepare(Map map, TopologyContext topologyContext, + OutputCollector collector) { // FIXME: stores map (topoConf), topologyContext and expose these to derived classes this.collector = collector; @@ -87,6 +96,7 @@ public void prepare(Map map, TopologyContext topologyContext, Ou /** * Borrow JedisCommands instance from container.

    * JedisCommands is an interface which contains single key operations. + * * @return implementation of JedisCommands * @see JedisCommandsContainer */ diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/bolt/RedisFilterBolt.java b/external/storm-redis/src/main/java/org/apache/storm/redis/bolt/RedisFilterBolt.java index d9228d32d2e..d7fc23e8f67 100644 --- a/external/storm-redis/src/main/java/org/apache/storm/redis/bolt/RedisFilterBolt.java +++ b/external/storm-redis/src/main/java/org/apache/storm/redis/bolt/RedisFilterBolt.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -14,7 +20,6 @@ import java.util.List; import java.util.Objects; - import org.apache.storm.redis.common.config.JedisClusterConfig; import org.apache.storm.redis.common.config.JedisPoolConfig; import org.apache.storm.redis.common.container.JedisCommandsContainer; @@ -46,6 +51,7 @@ public class RedisFilterBolt extends AbstractRedisBolt { /** * Constructor for single Redis environment (JedisPool). + * * @param config configuration for initializing JedisPool * @param filterMapper mapper containing which datatype, query key that Bolt uses */ @@ -66,6 +72,7 @@ public RedisFilterBolt(JedisPoolConfig config, RedisFilterMapper filterMapper) { /** * Constructor for Redis Cluster environment (JedisCluster). + * * @param config configuration for initializing JedisCluster * @param filterMapper mapper containing which datatype, query key that Bolt uses */ @@ -124,7 +131,8 @@ public void process(Tuple input) { break; default: - throw new IllegalArgumentException("Cannot process such data type: " + dataType); + throw new IllegalArgumentException("Cannot process such data type: " + + dataType); } if (found) { diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/bolt/RedisLookupBolt.java b/external/storm-redis/src/main/java/org/apache/storm/redis/bolt/RedisLookupBolt.java index 07c79773cac..76d8040664a 100644 --- a/external/storm-redis/src/main/java/org/apache/storm/redis/bolt/RedisLookupBolt.java +++ b/external/storm-redis/src/main/java/org/apache/storm/redis/bolt/RedisLookupBolt.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -34,6 +40,7 @@ public class RedisLookupBolt extends AbstractRedisBolt { /** * Constructor for single Redis environment (JedisPool). + * * @param config configuration for initializing JedisPool * @param lookupMapper mapper containing which datatype, query key, output key that Bolt uses */ @@ -49,6 +56,7 @@ public RedisLookupBolt(JedisPoolConfig config, RedisLookupMapper lookupMapper) { /** * Constructor for Redis Cluster environment (JedisCluster). + * * @param config configuration for initializing JedisCluster * @param lookupMapper mapper containing which datatype, query key, output key that Bolt uses */ @@ -104,7 +112,8 @@ public void process(Tuple input) { break; default: - throw new IllegalArgumentException("Cannot process such data type: " + dataType); + throw new IllegalArgumentException("Cannot process such data type: " + + dataType); } List values = lookupMapper.toTuple(input, lookupValue); diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/bolt/RedisStoreBolt.java b/external/storm-redis/src/main/java/org/apache/storm/redis/bolt/RedisStoreBolt.java index 065d9cbea3b..95f5b3c80a7 100644 --- a/external/storm-redis/src/main/java/org/apache/storm/redis/bolt/RedisStoreBolt.java +++ b/external/storm-redis/src/main/java/org/apache/storm/redis/bolt/RedisStoreBolt.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -32,6 +38,7 @@ public class RedisStoreBolt extends AbstractRedisBolt { /** * Constructor for single Redis environment (JedisPool). + * * @param config configuration for initializing JedisPool * @param storeMapper mapper containing which datatype, storing value's key that Bolt uses */ @@ -46,6 +53,7 @@ public RedisStoreBolt(JedisPoolConfig config, RedisStoreMapper storeMapper) { /** * Constructor for Redis Cluster environment (JedisCluster). + * * @param config configuration for initializing JedisCluster * @param storeMapper mapper containing which datatype, storing value's key that Bolt uses */ @@ -98,7 +106,8 @@ public void process(Tuple input) { case GEO: String[] array = value.split(":"); if (array.length != 2) { - throw new IllegalArgumentException("value structure should be longitude:latitude"); + throw new IllegalArgumentException("value structure should be " + + "longitude:latitude"); } double longitude = Double.valueOf(array[0]); @@ -107,7 +116,8 @@ public void process(Tuple input) { break; default: - throw new IllegalArgumentException("Cannot process such data type: " + dataType); + throw new IllegalArgumentException("Cannot process such data type: " + + dataType); } collector.ack(input); diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/common/adapter/RedisCommandsAdapterJedis.java b/external/storm-redis/src/main/java/org/apache/storm/redis/common/adapter/RedisCommandsAdapterJedis.java index a3f1f5a6b66..d685c980429 100644 --- a/external/storm-redis/src/main/java/org/apache/storm/redis/common/adapter/RedisCommandsAdapterJedis.java +++ b/external/storm-redis/src/main/java/org/apache/storm/redis/common/adapter/RedisCommandsAdapterJedis.java @@ -97,7 +97,8 @@ public String rename(String oldkey, String newkey) { } @Override - public ScanResult> hscan(byte[] key, byte[] cursor, ScanParams params) { + public ScanResult> hscan(byte[] key, byte[] cursor, + ScanParams params) { return jedis.hscan(key, cursor, params); } diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/common/adapter/RedisCommandsAdapterJedisCluster.java b/external/storm-redis/src/main/java/org/apache/storm/redis/common/adapter/RedisCommandsAdapterJedisCluster.java index 9d2d96b199a..8ad96345247 100644 --- a/external/storm-redis/src/main/java/org/apache/storm/redis/common/adapter/RedisCommandsAdapterJedisCluster.java +++ b/external/storm-redis/src/main/java/org/apache/storm/redis/common/adapter/RedisCommandsAdapterJedisCluster.java @@ -26,7 +26,6 @@ import redis.clients.jedis.params.ScanParams; import redis.clients.jedis.resps.ScanResult; - /** * Adapter class to make JedisCluster instance play with BinaryRedisCommands interface. */ @@ -98,7 +97,8 @@ public String rename(String oldkey, String newkey) { } @Override - public ScanResult> hscan(byte[] key, byte[] cursor, ScanParams params) { + public ScanResult> hscan(byte[] key, byte[] cursor, + ScanParams params) { return jedisCluster.hscan(key, cursor, params); } diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/common/commands/RedisCommands.java b/external/storm-redis/src/main/java/org/apache/storm/redis/common/commands/RedisCommands.java index 5a7170eb2a6..2397bde9c1c 100644 --- a/external/storm-redis/src/main/java/org/apache/storm/redis/common/commands/RedisCommands.java +++ b/external/storm-redis/src/main/java/org/apache/storm/redis/common/commands/RedisCommands.java @@ -19,14 +19,14 @@ package org.apache.storm.redis.common.commands; import java.util.Map; - import redis.clients.jedis.params.ScanParams; import redis.clients.jedis.resps.ScanResult; /** * This interface represents Jedis methods exhaustively which are used on storm-redis. * - *

    This is a workaround since Jedis and JedisCluster doesn't implement same interface for binary type of methods, and + *

    This is a workaround since Jedis and JedisCluster doesn't implement same interface for binary + * type of methods, and * unify binary methods and string methods into one interface. */ public interface RedisCommands { diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/common/config/JedisClusterConfig.java b/external/storm-redis/src/main/java/org/apache/storm/redis/common/config/JedisClusterConfig.java index e9b355d6cd5..89c82143886 100644 --- a/external/storm-redis/src/main/java/org/apache/storm/redis/common/config/JedisClusterConfig.java +++ b/external/storm-redis/src/main/java/org/apache/storm/redis/common/config/JedisClusterConfig.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -38,7 +44,8 @@ public JedisClusterConfig() { *

    * You can use JedisClusterConfig.Builder() for leaving some fields to apply default value. *

    - * Note that list of node is mandatory, and when you didn't set nodes, it throws NullPointerException. + * Note that list of node is mandatory, and when you didn't set nodes, it throws + * NullPointerException. * * @param nodes list of node information for JedisCluster * @param timeout socket / connection timeout @@ -54,7 +61,8 @@ public JedisClusterConfig(Set nodes, int timeout, int maxRedi *

    * You can use JedisClusterConfig.Builder() for leaving some fields to apply default value. *

    - * Note that list of node is mandatory, and when you didn't set nodes, it throws NullPointerException. + * Note that list of node is mandatory, and when you didn't set nodes, it throws + * NullPointerException. * * @param nodes list of node information for JedisCluster * @param timeout socket / connection timeout @@ -62,7 +70,8 @@ public JedisClusterConfig(Set nodes, int timeout, int maxRedi * @param password password, if any * @throws NullPointerException when you didn't set nodes */ - public JedisClusterConfig(Set nodes, int timeout, int maxRedirections, String password) { + public JedisClusterConfig(Set nodes, int timeout, int maxRedirections, + String password) { Preconditions.checkNotNull(nodes, "Node information should be presented"); this.nodes = nodes; @@ -73,6 +82,7 @@ public JedisClusterConfig(Set nodes, int timeout, int maxRedi /** * Returns nodes. + * * @return list of node information */ public Set getNodes() { @@ -85,6 +95,7 @@ public Set getNodes() { /** * Returns socket / connection timeout. + * * @return socket / connection timeout */ public int getTimeout() { @@ -93,6 +104,7 @@ public int getTimeout() { /** * Returns limit of redirection. + * * @return limit of redirection */ public int getMaxRedirections() { @@ -101,6 +113,7 @@ public int getMaxRedirections() { /** * Returns password. + * * @return password */ public String getPassword() { @@ -118,6 +131,7 @@ public static class Builder { /** * Sets list of node. + * * @param nodes list of node * @return Builder itself */ @@ -128,6 +142,7 @@ public Builder setNodes(Set nodes) { /** * Sets socket / connection timeout. + * * @param timeout socket / connection timeout * @return Builder itself */ @@ -138,6 +153,7 @@ public Builder setTimeout(int timeout) { /** * Sets limit of redirection. + * * @param maxRedirections limit of redirection * @return Builder itself */ @@ -148,6 +164,7 @@ public Builder setMaxRedirections(int maxRedirections) { /** * Sets password. + * * @param password password, if any * @return Builder itself */ @@ -158,6 +175,7 @@ public Builder setPassword(String password) { /** * Builds JedisClusterConfig. + * * @return JedisClusterConfig */ public JedisClusterConfig build() { diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/common/config/JedisPoolConfig.java b/external/storm-redis/src/main/java/org/apache/storm/redis/common/config/JedisPoolConfig.java index 3e53cacab37..0f1d668e84d 100644 --- a/external/storm-redis/src/main/java/org/apache/storm/redis/common/config/JedisPoolConfig.java +++ b/external/storm-redis/src/main/java/org/apache/storm/redis/common/config/JedisPoolConfig.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -51,6 +57,7 @@ public JedisPoolConfig(String host, int port, int timeout, String password, int /** * Returns host. + * * @return hostname or IP */ public String getHost() { @@ -59,6 +66,7 @@ public String getHost() { /** * Returns port. + * * @return port */ public int getPort() { @@ -67,6 +75,7 @@ public int getPort() { /** * Returns timeout. + * * @return socket / connection timeout */ public int getTimeout() { @@ -75,6 +84,7 @@ public int getTimeout() { /** * Returns database index. + * * @return database index */ public int getDatabase() { @@ -83,6 +93,7 @@ public int getDatabase() { /** * Returns password. + * * @return password */ public String getPassword() { @@ -112,6 +123,7 @@ public static class Builder { /** * Sets host. + * * @param host host * @return Builder itself */ @@ -122,6 +134,7 @@ public Builder setHost(String host) { /** * Sets port. + * * @param port port * @return Builder itself */ @@ -132,6 +145,7 @@ public Builder setPort(int port) { /** * Sets timeout. + * * @param timeout timeout * @return Builder itself */ @@ -142,6 +156,7 @@ public Builder setTimeout(int timeout) { /** * Sets database index. + * * @param database database index * @return Builder itself */ @@ -152,6 +167,7 @@ public Builder setDatabase(int database) { /** * Sets password. + * * @param password password, if any * @return Builder itself */ @@ -162,6 +178,7 @@ public Builder setPassword(String password) { /** * Builds JedisPoolConfig. + * * @return JedisPoolConfig */ public JedisPoolConfig build() { diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/common/container/JedisClusterContainer.java b/external/storm-redis/src/main/java/org/apache/storm/redis/common/container/JedisClusterContainer.java index 5220c1c7ae9..cfd8b611433 100644 --- a/external/storm-redis/src/main/java/org/apache/storm/redis/common/container/JedisClusterContainer.java +++ b/external/storm-redis/src/main/java/org/apache/storm/redis/common/container/JedisClusterContainer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -19,7 +25,8 @@ /** * Container for managing JedisCluster. *

    - * Note that JedisCluster doesn't need to be pooled since it's thread-safe and it stores pools internally. + * Note that JedisCluster doesn't need to be pooled since it's thread-safe and it stores pools + * internally. */ public class JedisClusterContainer implements JedisCommandsContainer { @@ -27,6 +34,7 @@ public class JedisClusterContainer implements JedisCommandsContainer { /** * Constructor. + * * @param jedisCluster JedisCluster instance */ public JedisClusterContainer(JedisCluster jedisCluster) { @@ -49,7 +57,8 @@ public String hget(final String key, final String field) { } @Override - public Long geoadd(final String key, final double longitude, final double latitude, final String member) { + public Long geoadd(final String key, final double longitude, final double latitude, + final String member) { return jedisCluster.geoadd(key, longitude, latitude, member); } diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/common/container/JedisCommandsContainer.java b/external/storm-redis/src/main/java/org/apache/storm/redis/common/container/JedisCommandsContainer.java index 38b1321b7b1..e721113990d 100644 --- a/external/storm-redis/src/main/java/org/apache/storm/redis/common/container/JedisCommandsContainer.java +++ b/external/storm-redis/src/main/java/org/apache/storm/redis/common/container/JedisCommandsContainer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/common/container/JedisCommandsContainerBuilder.java b/external/storm-redis/src/main/java/org/apache/storm/redis/common/container/JedisCommandsContainerBuilder.java index 9370e3421a7..d5f2f7158f4 100644 --- a/external/storm-redis/src/main/java/org/apache/storm/redis/common/container/JedisCommandsContainerBuilder.java +++ b/external/storm-redis/src/main/java/org/apache/storm/redis/common/container/JedisCommandsContainerBuilder.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -26,6 +32,7 @@ public class JedisCommandsContainerBuilder { /** * Builds container for single Redis environment. + * * @param config configuration for JedisPool * @return container for single Redis environment */ @@ -33,7 +40,8 @@ public static JedisCommandsContainer build(JedisPoolConfig config) { // FIXME: We're using default config since it cannot be serialized // We still needs to provide some options externally JedisPool jedisPool = - new JedisPool(new redis.clients.jedis.JedisPoolConfig(), config.getHost(), config.getPort(), + new JedisPool(new redis.clients.jedis.JedisPoolConfig(), config.getHost(), config + .getPort(), config.getTimeout(), config.getPassword(), config.getDatabase()); return new JedisContainer(jedisPool); @@ -41,6 +49,7 @@ public static JedisCommandsContainer build(JedisPoolConfig config) { /** * Builds container for Redis Cluster environment. + * * @param config configuration for JedisCluster * @return container for Redis Cluster environment */ @@ -48,7 +57,8 @@ public static JedisCommandsContainer build(JedisClusterConfig config) { // FIXME: We're using default config since it cannot be serialized // We still needs to provide some options externally JedisCluster jedisCluster = - new JedisCluster(config.getNodes(), config.getTimeout(), config.getTimeout(), config.getMaxRedirections(), config.getPassword(), + new JedisCluster(config.getNodes(), config.getTimeout(), config.getTimeout(), config + .getMaxRedirections(), config.getPassword(), new GenericObjectPoolConfig<>()); return new JedisClusterContainer(jedisCluster); } diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/common/container/JedisContainer.java b/external/storm-redis/src/main/java/org/apache/storm/redis/common/container/JedisContainer.java index 1eed610dc6e..eea89ee16ec 100644 --- a/external/storm-redis/src/main/java/org/apache/storm/redis/common/container/JedisContainer.java +++ b/external/storm-redis/src/main/java/org/apache/storm/redis/common/container/JedisContainer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -16,7 +22,6 @@ import java.io.IOException; import java.util.List; import java.util.function.Function; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; import redis.clients.jedis.GeoCoordinate; @@ -24,7 +29,8 @@ import redis.clients.jedis.commands.JedisCommands; /** - * Adapter for providing a unified interface for running commands over both Jedis and JedisCluster instances. + * Adapter for providing a unified interface for running commands over both Jedis and JedisCluster + * instances. */ public class JedisContainer implements JedisCommandsContainer { private static final Logger LOG = LoggerFactory.getLogger(JedisContainer.class); @@ -33,6 +39,7 @@ public class JedisContainer implements JedisCommandsContainer { /** * Constructor. + * * @param jedisPool JedisPool which actually manages Jedis instances */ public JedisContainer(JedisPool jedisPool) { @@ -76,8 +83,10 @@ public String hget(final String key, final String field) { } @Override - public Long geoadd(final String key, final double longitude, final double latitude, final String member) { - return runCommand((jedisCommands) -> jedisCommands.geoadd(key, longitude, latitude, member)); + public Long geoadd(final String key, final double longitude, final double latitude, + final String member) { + return runCommand((jedisCommands) -> jedisCommands.geoadd(key, longitude, latitude, + member)); } @Override diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/common/container/RedisClusterContainer.java b/external/storm-redis/src/main/java/org/apache/storm/redis/common/container/RedisClusterContainer.java index 600d4fec683..0556d26fa99 100644 --- a/external/storm-redis/src/main/java/org/apache/storm/redis/common/container/RedisClusterContainer.java +++ b/external/storm-redis/src/main/java/org/apache/storm/redis/common/container/RedisClusterContainer.java @@ -25,7 +25,8 @@ /** * Container for managing JedisCluster. *

    - * Note that JedisCluster doesn't need to be pooled since it's thread-safe and it stores pools internally. + * Note that JedisCluster doesn't need to be pooled since it's thread-safe and it stores pools + * internally. */ public class RedisClusterContainer implements RedisCommandsInstanceContainer { private JedisCluster jedisCluster; diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/common/container/RedisCommandsContainerBuilder.java b/external/storm-redis/src/main/java/org/apache/storm/redis/common/container/RedisCommandsContainerBuilder.java index b369e1ca0ee..ddfef782eb5 100644 --- a/external/storm-redis/src/main/java/org/apache/storm/redis/common/container/RedisCommandsContainerBuilder.java +++ b/external/storm-redis/src/main/java/org/apache/storm/redis/common/container/RedisCommandsContainerBuilder.java @@ -25,7 +25,8 @@ import redis.clients.jedis.JedisPool; /** - * The binary version of container builder which helps abstraction of two env. - single instance or Redis Cluster. + * The binary version of container builder which helps abstraction of two env. - single instance or + * Redis Cluster. */ public class RedisCommandsContainerBuilder { @@ -39,7 +40,8 @@ public static RedisCommandsInstanceContainer build(JedisPoolConfig config) { // FIXME: We're using default config since it cannot be serialized // We still needs to provide some options externally JedisPool jedisPool = - new JedisPool(new redis.clients.jedis.JedisPoolConfig(), config.getHost(), config.getPort(), + new JedisPool(new redis.clients.jedis.JedisPoolConfig(), config.getHost(), config + .getPort(), config.getTimeout(), config.getPassword(), config.getDatabase()); return new RedisContainer(jedisPool); @@ -55,7 +57,8 @@ public static RedisCommandsInstanceContainer build(JedisClusterConfig config) { // FIXME: We're using default config since it cannot be serialized // We still needs to provide some options externally JedisCluster jedisCluster = - new JedisCluster(config.getNodes(), config.getTimeout(), config.getMaxRedirections(), new GenericObjectPoolConfig<>()); + new JedisCluster(config.getNodes(), config.getTimeout(), config.getMaxRedirections(), + new GenericObjectPoolConfig<>()); return new RedisClusterContainer(jedisCluster); } } diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/common/mapper/RedisDataTypeDescription.java b/external/storm-redis/src/main/java/org/apache/storm/redis/common/mapper/RedisDataTypeDescription.java index b896bdb3d12..e3fe6523ade 100644 --- a/external/storm-redis/src/main/java/org/apache/storm/redis/common/mapper/RedisDataTypeDescription.java +++ b/external/storm-redis/src/main/java/org/apache/storm/redis/common/mapper/RedisDataTypeDescription.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -15,7 +21,8 @@ import java.io.Serializable; /** - * RedisDataTypeDescription defines data type and additional key if needed for lookup / store tuples. + * RedisDataTypeDescription defines data type and additional key if needed for lookup / store + * tuples. */ public class RedisDataTypeDescription implements Serializable { private RedisDataType dataType; @@ -23,6 +30,7 @@ public class RedisDataTypeDescription implements Serializable { /** * Constructor. + * * @param dataType data type */ public RedisDataTypeDescription(RedisDataType dataType) { @@ -31,6 +39,7 @@ public RedisDataTypeDescription(RedisDataType dataType) { /** * Constructor. + * * @param dataType data type * @param additionalKey additional key for hash and sorted set */ @@ -42,13 +51,15 @@ public RedisDataTypeDescription(RedisDataType dataType, String additionalKey) { || dataType == RedisDataType.SORTED_SET || dataType == RedisDataType.GEO) { if (additionalKey == null) { - throw new IllegalArgumentException("Hash, Sorted Set and GEO should have additional key"); + throw new IllegalArgumentException("Hash, Sorted Set and GEO should have " + + "additional key"); } } } /** * Returns defined data type. + * * @return data type */ public RedisDataType getDataType() { @@ -57,6 +68,7 @@ public RedisDataType getDataType() { /** * Returns defined additional key. + * * @return additional key */ public String getAdditionalKey() { diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/common/mapper/RedisFilterMapper.java b/external/storm-redis/src/main/java/org/apache/storm/redis/common/mapper/RedisFilterMapper.java index f1d650691d9..856cd66dd16 100644 --- a/external/storm-redis/src/main/java/org/apache/storm/redis/common/mapper/RedisFilterMapper.java +++ b/external/storm-redis/src/main/java/org/apache/storm/redis/common/mapper/RedisFilterMapper.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -15,12 +21,14 @@ import org.apache.storm.topology.OutputFieldsDeclarer; /** - * RedisFilterMapper is for defining spec. which is used for querying value from Redis and filtering. + * RedisFilterMapper is for defining spec. which is used for querying value from Redis and + * filtering. */ public interface RedisFilterMapper extends TupleMapper, RedisMapper { /** - * declare what are the fields that this code will output. + * Declare what are the fields that this code will output. + * * @param declarer OutputFieldsDeclarer */ void declareOutputFields(OutputFieldsDeclarer declarer); diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/common/mapper/RedisLookupMapper.java b/external/storm-redis/src/main/java/org/apache/storm/redis/common/mapper/RedisLookupMapper.java index 0e3b1c532c2..304d18c905d 100644 --- a/external/storm-redis/src/main/java/org/apache/storm/redis/common/mapper/RedisLookupMapper.java +++ b/external/storm-redis/src/main/java/org/apache/storm/redis/common/mapper/RedisLookupMapper.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -18,19 +24,23 @@ import org.apache.storm.tuple.Values; /** - * RedisLookupMapper is for defining spec. which is used for querying value from Redis and converting response to tuple. + * RedisLookupMapper is for defining spec. which is used for querying value from Redis and + * converting response to tuple. */ public interface RedisLookupMapper extends TupleMapper, RedisMapper { /** * Converts return value from Redis to a list of storm values that can be emitted. + * * @param input the input tuple. * @param value Redis query response value. Can be String, Boolean, Long regarding of data type. - * @return a List of storm values that can be emitted. Each item in list is emitted as an output tuple. + * @return a List of storm values that can be emitted. Each item in list is emitted as an output + * tuple. */ List toTuple(ITuple input, Object value); /** - * declare what are the fields that this code will output. + * Declare what are the fields that this code will output. + * * @param declarer OutputFieldsDeclarer */ void declareOutputFields(OutputFieldsDeclarer declarer); diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/common/mapper/RedisMapper.java b/external/storm-redis/src/main/java/org/apache/storm/redis/common/mapper/RedisMapper.java index b4ea07d7f94..e11bbfde038 100644 --- a/external/storm-redis/src/main/java/org/apache/storm/redis/common/mapper/RedisMapper.java +++ b/external/storm-redis/src/main/java/org/apache/storm/redis/common/mapper/RedisMapper.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -18,6 +24,7 @@ public interface RedisMapper { /** * Returns descriptor which defines data type. + * * @return data type descriptor */ RedisDataTypeDescription getDataTypeDescription(); diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/common/mapper/RedisStoreMapper.java b/external/storm-redis/src/main/java/org/apache/storm/redis/common/mapper/RedisStoreMapper.java index f745d5d431f..570bc737d2c 100644 --- a/external/storm-redis/src/main/java/org/apache/storm/redis/common/mapper/RedisStoreMapper.java +++ b/external/storm-redis/src/main/java/org/apache/storm/redis/common/mapper/RedisStoreMapper.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/common/mapper/TupleMapper.java b/external/storm-redis/src/main/java/org/apache/storm/redis/common/mapper/TupleMapper.java index ffe0d710c68..ce932972e33 100644 --- a/external/storm-redis/src/main/java/org/apache/storm/redis/common/mapper/TupleMapper.java +++ b/external/storm-redis/src/main/java/org/apache/storm/redis/common/mapper/TupleMapper.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -21,6 +27,7 @@ public interface TupleMapper extends Serializable { /** * Extracts key from tuple. + * * @param tuple source tuple * @return key */ @@ -28,6 +35,7 @@ public interface TupleMapper extends Serializable { /** * Extracts value from tuple. + * * @param tuple source tuple * @return value */ diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/state/RedisKeyValueState.java b/external/storm-redis/src/main/java/org/apache/storm/redis/state/RedisKeyValueState.java index 880aa5373d1..660dad452aa 100644 --- a/external/storm-redis/src/main/java/org/apache/storm/redis/state/RedisKeyValueState.java +++ b/external/storm-redis/src/main/java/org/apache/storm/redis/state/RedisKeyValueState.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -42,8 +48,9 @@ */ public class RedisKeyValueState implements KeyValueState { public static final int ITERATOR_CHUNK_SIZE = 100; - public static final NavigableMap EMPTY_PENDING_COMMIT_MAP = Maps.unmodifiableNavigableMap( - new TreeMap(UnsignedBytes.lexicographicalComparator())); + public static final NavigableMap EMPTY_PENDING_COMMIT_MAP = Maps + .unmodifiableNavigableMap( + new TreeMap(UnsignedBytes.lexicographicalComparator())); private static final Logger LOG = LoggerFactory.getLogger(RedisKeyValueState.class); private static final String COMMIT_TXID_KEY = "commit"; private static final String PREPARE_TXID_KEY = "prepare"; @@ -65,16 +72,21 @@ public RedisKeyValueState(String namespace) { } public RedisKeyValueState(String namespace, JedisPoolConfig poolConfig) { - this(namespace, poolConfig, new DefaultStateSerializer(), new DefaultStateSerializer()); + this(namespace, poolConfig, new DefaultStateSerializer(), + new DefaultStateSerializer()); } - public RedisKeyValueState(String namespace, JedisPoolConfig poolConfig, Serializer keySerializer, Serializer valueSerializer) { - this(namespace, RedisCommandsContainerBuilder.build(poolConfig), keySerializer, valueSerializer); + public RedisKeyValueState(String namespace, JedisPoolConfig poolConfig, + Serializer keySerializer, Serializer valueSerializer) { + this(namespace, RedisCommandsContainerBuilder.build(poolConfig), keySerializer, + valueSerializer); } - public RedisKeyValueState(String namespace, JedisClusterConfig jedisClusterConfig, Serializer keySerializer, + public RedisKeyValueState(String namespace, JedisClusterConfig jedisClusterConfig, + Serializer keySerializer, Serializer valueSerializer) { - this(namespace, RedisCommandsContainerBuilder.build(jedisClusterConfig), keySerializer, valueSerializer); + this(namespace, RedisCommandsContainerBuilder.build(jedisClusterConfig), keySerializer, + valueSerializer); } public RedisKeyValueState(String namespace, RedisCommandsInstanceContainer container, @@ -110,7 +122,8 @@ private void initPendingCommit() { commands = container.getInstance(); if (commands.exists(prepareNamespace)) { LOG.debug("Loading previously prepared commit from {}", prepareNamespace); - NavigableMap pendingCommitMap = new TreeMap<>(UnsignedBytes.lexicographicalComparator()); + NavigableMap pendingCommitMap = new TreeMap<>(UnsignedBytes + .lexicographicalComparator()); pendingCommitMap.putAll(commands.hgetAll(prepareNamespace)); pendingCommit = Maps.unmodifiableNavigableMap(pendingCommitMap); } else { @@ -174,9 +187,12 @@ public V delete(K key) { @Override public Iterator> iterator() { - return new RedisKeyValueStateIterator(namespace, container, pendingPrepare.entrySet().iterator(), + return new RedisKeyValueStateIterator(namespace, container, pendingPrepare.entrySet() + .iterator(), pendingCommit.entrySet().iterator(), - ITERATOR_CHUNK_SIZE, encoder.getKeySerializer(), encoder.getValueSerializer()); + ITERATOR_CHUNK_SIZE, encoder + .getKeySerializer(), encoder + .getValueSerializer()); } @Override @@ -299,7 +315,8 @@ private void validatePrepareTxid(long txid) { Long committedTxid = lastCommittedTxid(); if (committedTxid != null) { if (txid <= committedTxid) { - throw new RuntimeException("Invalid txid '" + txid + "' for prepare. Txid '" + committedTxid + throw new RuntimeException("Invalid txid '" + txid + "' for prepare. Txid '" + + committedTxid + "' is already committed"); } } @@ -313,13 +330,15 @@ private void validateCommitTxid(long txid) { Long committedTxid = lastCommittedTxid(); if (committedTxid != null) { if (txid < committedTxid) { - throw new RuntimeException("Invalid txid '" + txid + "' txid '" + committedTxid + "' is already committed"); + throw new RuntimeException("Invalid txid '" + txid + "' txid '" + committedTxid + + "' is already committed"); } } Long preparedTxid = lastPreparedTxid(); if (preparedTxid != null) { if (txid != preparedTxid) { - throw new RuntimeException("Invalid txid '" + txid + "' not same as prepared txid '" + preparedTxid + "'"); + throw new RuntimeException("Invalid txid '" + txid + + "' not same as prepared txid '" + preparedTxid + "'"); } } } diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/state/RedisKeyValueStateIterator.java b/external/storm-redis/src/main/java/org/apache/storm/redis/state/RedisKeyValueStateIterator.java index e6b282714d2..cc29e06eb3e 100644 --- a/external/storm-redis/src/main/java/org/apache/storm/redis/state/RedisKeyValueStateIterator.java +++ b/external/storm-redis/src/main/java/org/apache/storm/redis/state/RedisKeyValueStateIterator.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -93,7 +99,8 @@ private void loadChunkFromRedis() { RedisCommands commands = null; try { commands = container.getInstance(); - ScanResult> scanResult = commands.hscan(namespace, cursor, scanParams); + ScanResult> scanResult = commands.hscan(namespace, cursor, + scanParams); List> result = scanResult.getResult(); if (result != null) { cachedResultIterator = result.iterator(); diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/state/RedisKeyValueStateProvider.java b/external/storm-redis/src/main/java/org/apache/storm/redis/state/RedisKeyValueStateProvider.java index b8cd377c144..8a576f47a24 100644 --- a/external/storm-redis/src/main/java/org/apache/storm/redis/state/RedisKeyValueStateProvider.java +++ b/external/storm-redis/src/main/java/org/apache/storm/redis/state/RedisKeyValueStateProvider.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -40,7 +46,8 @@ public State newState(String namespace, Map topoConf, TopologyCo try { return getRedisKeyValueState(namespace, topoConf, context, getStateConfig(topoConf)); } catch (Exception ex) { - LOG.error("Error loading config from storm conf {}", ConfigUtils.maskCredentials(topoConf)); + LOG.error("Error loading config from storm conf {}", ConfigUtils + .maskCredentials(topoConf)); throw new RuntimeException(ex); } } @@ -59,7 +66,8 @@ StateConfig getStateConfig(Map topoConf) throws Exception { return stateConfig; } - private RedisKeyValueState getRedisKeyValueState(String namespace, Map topoConf, TopologyContext context, + private RedisKeyValueState getRedisKeyValueState(String namespace, Map topoConf, + TopologyContext context, StateConfig config) throws Exception { JedisPoolConfig jedisPoolConfig = getJedisPoolConfig(config); JedisClusterConfig jedisClusterConfig = getJedisClusterConfig(config); @@ -70,33 +78,39 @@ private RedisKeyValueState getRedisKeyValueState(String namespace, Map topoConf, TopologyContext context, StateConfig config) throws Exception { + private Serializer getKeySerializer(Map topoConf, TopologyContext context, + StateConfig config) throws Exception { Serializer serializer; if (config.keySerializerClass != null) { Class klass = (Class) Class.forName(config.keySerializerClass); serializer = (Serializer) klass.newInstance(); } else if (config.keyClass != null) { - serializer = new DefaultStateSerializer(topoConf, context, Collections.singletonList(Class.forName(config.keyClass))); + serializer = new DefaultStateSerializer(topoConf, context, Collections + .singletonList(Class.forName(config.keyClass))); } else { serializer = new DefaultStateSerializer(topoConf, context); } return serializer; } - private Serializer getValueSerializer(Map topoConf, TopologyContext context, StateConfig config) throws Exception { + private Serializer getValueSerializer(Map topoConf, TopologyContext context, + StateConfig config) throws Exception { Serializer serializer; if (config.valueSerializerClass != null) { Class klass = (Class) Class.forName(config.valueSerializerClass); serializer = (Serializer) klass.newInstance(); } else if (config.valueClass != null) { - serializer = new DefaultStateSerializer(topoConf, context, Collections.singletonList(Class.forName(config.valueClass))); + serializer = new DefaultStateSerializer(topoConf, context, Collections + .singletonList(Class.forName(config.valueClass))); } else { serializer = new DefaultStateSerializer(topoConf, context); } diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/AbstractRedisMapState.java b/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/AbstractRedisMapState.java index 6058b9711e1..484ae575d27 100644 --- a/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/AbstractRedisMapState.java +++ b/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/AbstractRedisMapState.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

    - * http://www.apache.org/licenses/LICENSE-2.0 - *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

    http://www.apache.org/licenses/LICENSE-2.0 + * + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -40,7 +46,8 @@ * @param value's type class */ public abstract class AbstractRedisMapState implements IBackingMap { - public static final EnumMap DEFAULT_SERIALIZERS = Maps.newEnumMap(ImmutableMap.of( + public static final EnumMap DEFAULT_SERIALIZERS = Maps + .newEnumMap(ImmutableMap.of( StateType.NON_TRANSACTIONAL, new JSONNonTransactionalSerializer(), StateType.TRANSACTIONAL, new JSONTransactionalSerializer(), StateType.OPAQUE, new JSONOpaqueSerializer() @@ -111,6 +118,7 @@ private List deserializeValues(List> keys, List values) /** * Returns KeyFactory which is used for converting state key -> Redis key. + * * @return key factory */ protected abstract KeyFactory getKeyFactory(); diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/AbstractRedisStateQuerier.java b/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/AbstractRedisStateQuerier.java index 4378f8ff43f..70d363de4d5 100644 --- a/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/AbstractRedisStateQuerier.java +++ b/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/AbstractRedisStateQuerier.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

    - * http://www.apache.org/licenses/LICENSE-2.0 - *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

    http://www.apache.org/licenses/LICENSE-2.0 + * + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -23,7 +29,8 @@ import org.apache.storm.tuple.Values; /** - * AbstractRedisStateQuerier is base class of any RedisStateQuerier, which implements BaseQueryFunction. + * AbstractRedisStateQuerier is base class of any RedisStateQuerier, which implements + * BaseQueryFunction. *

    * Derived classes should provide how to retrieve values from Redis, * and AbstractRedisStateQuerier takes care of rest things. diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/AbstractRedisStateUpdater.java b/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/AbstractRedisStateUpdater.java index 5663e18487d..ba8ff425add 100644 --- a/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/AbstractRedisStateUpdater.java +++ b/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/AbstractRedisStateUpdater.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

    - * http://www.apache.org/licenses/LICENSE-2.0 - *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

    http://www.apache.org/licenses/LICENSE-2.0 + * + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -23,7 +29,8 @@ import org.apache.storm.trident.tuple.TridentTuple; /** - * AbstractRedisStateUpdater is base class of any RedisStateUpdater, which implements BaseStateUpdater. + * AbstractRedisStateUpdater is base class of any RedisStateUpdater, which implements + * BaseStateUpdater. * *

    Derived classes should provide how to update (key, value) pairs to Redis, * and AbstractRedisStateUpdater takes care of rest things. diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/KeyFactory.java b/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/KeyFactory.java index 88243c98685..385179bc0c0 100644 --- a/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/KeyFactory.java +++ b/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/KeyFactory.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

    - * http://www.apache.org/licenses/LICENSE-2.0 - *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

    http://www.apache.org/licenses/LICENSE-2.0 + * + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -21,6 +27,7 @@ public interface KeyFactory extends Serializable { /** * Converts state key to Redis key. + * * @param key state key * @return Redis key */ diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/Options.java b/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/Options.java index a65790a42d8..f5aaf86334e 100644 --- a/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/Options.java +++ b/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/Options.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

    - * http://www.apache.org/licenses/LICENSE-2.0 - *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

    http://www.apache.org/licenses/LICENSE-2.0 + * + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/RedisClusterMapState.java b/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/RedisClusterMapState.java index ae634d686f3..71e578e5dd0 100644 --- a/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/RedisClusterMapState.java +++ b/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/RedisClusterMapState.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

    - * http://www.apache.org/licenses/LICENSE-2.0 - *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

    http://www.apache.org/licenses/LICENSE-2.0 + * + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -15,7 +21,6 @@ import com.google.common.collect.Lists; import java.util.List; import java.util.Map; - import org.apache.commons.pool2.impl.GenericObjectPoolConfig; import org.apache.storm.redis.common.config.JedisClusterConfig; import org.apache.storm.redis.common.mapper.RedisDataTypeDescription; @@ -81,7 +86,8 @@ public static StateFactory opaque(JedisClusterConfig jedisClusterConfig) { * @param dataTypeDescription definition of data type * @return StateFactory */ - public static StateFactory opaque(JedisClusterConfig jedisClusterConfig, RedisDataTypeDescription dataTypeDescription) { + public static StateFactory opaque(JedisClusterConfig jedisClusterConfig, + RedisDataTypeDescription dataTypeDescription) { Options opts = new Options(); opts.dataTypeDescription = dataTypeDescription; return opaque(jedisClusterConfig, opts); @@ -107,7 +113,8 @@ public static StateFactory opaque(JedisClusterConfig jedisClusterConfig, KeyFact * @param opts options of State * @return StateFactory */ - public static StateFactory opaque(JedisClusterConfig jedisClusterConfig, Options opts) { + public static StateFactory opaque(JedisClusterConfig jedisClusterConfig, + Options opts) { return new Factory(jedisClusterConfig, StateType.OPAQUE, opts); } @@ -128,7 +135,8 @@ public static StateFactory transactional(JedisClusterConfig jedisClusterConfig) * @param dataTypeDescription definition of data type * @return StateFactory */ - public static StateFactory transactional(JedisClusterConfig jedisClusterConfig, RedisDataTypeDescription dataTypeDescription) { + public static StateFactory transactional(JedisClusterConfig jedisClusterConfig, + RedisDataTypeDescription dataTypeDescription) { Options opts = new Options(); opts.dataTypeDescription = dataTypeDescription; return transactional(jedisClusterConfig, opts); @@ -141,7 +149,8 @@ public static StateFactory transactional(JedisClusterConfig jedisClusterConfig, * @param factory key factory * @return StateFactory */ - public static StateFactory transactional(JedisClusterConfig jedisClusterConfig, KeyFactory factory) { + public static StateFactory transactional(JedisClusterConfig jedisClusterConfig, + KeyFactory factory) { Options opts = new Options(); opts.keyFactory = factory; return transactional(jedisClusterConfig, opts); @@ -154,7 +163,8 @@ public static StateFactory transactional(JedisClusterConfig jedisClusterConfig, * @param opts options of State * @return StateFactory */ - public static StateFactory transactional(JedisClusterConfig jedisClusterConfig, Options opts) { + public static StateFactory transactional(JedisClusterConfig jedisClusterConfig, + Options opts) { return new Factory(jedisClusterConfig, StateType.TRANSACTIONAL, opts); } @@ -175,7 +185,8 @@ public static StateFactory nonTransactional(JedisClusterConfig jedisClusterConfi * @param dataTypeDescription definition of data type * @return StateFactory */ - public static StateFactory nonTransactional(JedisClusterConfig jedisClusterConfig, RedisDataTypeDescription dataTypeDescription) { + public static StateFactory nonTransactional(JedisClusterConfig jedisClusterConfig, + RedisDataTypeDescription dataTypeDescription) { Options opts = new Options(); opts.dataTypeDescription = dataTypeDescription; return nonTransactional(jedisClusterConfig, opts); @@ -188,7 +199,8 @@ public static StateFactory nonTransactional(JedisClusterConfig jedisClusterConfi * @param factory key factory * @return StateFactory */ - public static StateFactory nonTransactional(JedisClusterConfig jedisClusterConfig, KeyFactory factory) { + public static StateFactory nonTransactional(JedisClusterConfig jedisClusterConfig, + KeyFactory factory) { Options opts = new Options(); opts.keyFactory = factory; return nonTransactional(jedisClusterConfig, opts); @@ -201,7 +213,8 @@ public static StateFactory nonTransactional(JedisClusterConfig jedisClusterConfi * @param opts options of State * @return StateFactory */ - public static StateFactory nonTransactional(JedisClusterConfig jedisClusterConfig, Options opts) { + public static StateFactory nonTransactional(JedisClusterConfig jedisClusterConfig, + Options opts) { return new Factory(jedisClusterConfig, StateType.NON_TRANSACTIONAL, opts); } @@ -243,7 +256,8 @@ protected List retrieveValuesFromRedis(List keys) { return jedisCluster.hmget(description.getAdditionalKey(), stringKeys); default: - throw new IllegalArgumentException("Cannot process such data type: " + description.getDataType()); + throw new IllegalArgumentException("Cannot process such data type: " + description + .getDataType()); } } @@ -257,7 +271,8 @@ protected void updateStatesToRedis(Map keyValues) { case STRING: for (Map.Entry kvEntry : keyValues.entrySet()) { if (this.options.expireIntervalSec > 0) { - jedisCluster.setex(kvEntry.getKey(), this.options.expireIntervalSec, kvEntry.getValue()); + jedisCluster.setex(kvEntry.getKey(), this.options.expireIntervalSec, kvEntry + .getValue()); } else { jedisCluster.set(kvEntry.getKey(), kvEntry.getValue()); } @@ -267,12 +282,14 @@ protected void updateStatesToRedis(Map keyValues) { case HASH: jedisCluster.hmset(description.getAdditionalKey(), keyValues); if (this.options.expireIntervalSec > 0) { - jedisCluster.expire(description.getAdditionalKey(), this.options.expireIntervalSec); + jedisCluster.expire(description.getAdditionalKey(), + this.options.expireIntervalSec); } break; default: - throw new IllegalArgumentException("Cannot process such data type: " + description.getDataType()); + throw new IllegalArgumentException("Cannot process such data type: " + description + .getDataType()); } } @@ -280,7 +297,8 @@ protected void updateStatesToRedis(Map keyValues) { * RedisClusterMapState.Factory provides Redis Cluster environment version of StateFactory. */ protected static class Factory implements StateFactory { - public static final GenericObjectPoolConfig DEFAULT_POOL_CONFIG = new GenericObjectPoolConfig<>(); + public static final GenericObjectPoolConfig DEFAULT_POOL_CONFIG = + new GenericObjectPoolConfig<>(); JedisClusterConfig jedisClusterConfig; @@ -318,7 +336,8 @@ public Factory(JedisClusterConfig jedisClusterConfig, StateType type, Options op * {@inheritDoc} */ @Override - public State makeState(Map conf, IMetricsContext metrics, int partitionIndex, int numPartitions) { + public State makeState(Map conf, IMetricsContext metrics, + int partitionIndex, int numPartitions) { final JedisCluster jedisCluster = new JedisCluster(jedisClusterConfig.getNodes(), jedisClusterConfig.getTimeout(), jedisClusterConfig.getTimeout(), @@ -326,7 +345,8 @@ public State makeState(Map conf, IMetricsContext metrics, int pa jedisClusterConfig.getPassword(), DEFAULT_POOL_CONFIG); - RedisClusterMapState state = new RedisClusterMapState(jedisCluster, options, serializer, keyFactory); + RedisClusterMapState state = new RedisClusterMapState(jedisCluster, options, serializer, + keyFactory); CachedMap c = new CachedMap(state, options.localCacheSize); MapState ms; diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/RedisClusterState.java b/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/RedisClusterState.java index e052dd1b66b..d26912e08cc 100644 --- a/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/RedisClusterState.java +++ b/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/RedisClusterState.java @@ -1,19 +1,24 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.redis.trident.state; import java.util.Map; - import org.apache.commons.pool2.impl.GenericObjectPoolConfig; import org.apache.storm.redis.common.config.JedisClusterConfig; import org.apache.storm.task.IMetricsContext; @@ -68,7 +73,7 @@ public JedisCluster getJedisCluster() { * @param jedisCluster JedisCluster instance to return to pool */ public void returnJedisCluster(JedisCluster jedisCluster) { - //do nothing + // do nothing } /** @@ -77,7 +82,8 @@ public void returnJedisCluster(JedisCluster jedisCluster) { * @see StateFactory */ public static class Factory implements StateFactory { - public static final GenericObjectPoolConfig DEFAULT_POOL_CONFIG = new GenericObjectPoolConfig<>(); + public static final GenericObjectPoolConfig DEFAULT_POOL_CONFIG = + new GenericObjectPoolConfig<>(); private final JedisClusterConfig jedisClusterConfig; @@ -94,7 +100,8 @@ public Factory(JedisClusterConfig config) { * {@inheritDoc} */ @Override - public State makeState(Map conf, IMetricsContext metrics, int partitionIndex, int numPartitions) { + public State makeState(Map conf, IMetricsContext metrics, + int partitionIndex, int numPartitions) { final JedisCluster jedisCluster = new JedisCluster(jedisClusterConfig.getNodes(), jedisClusterConfig.getTimeout(), jedisClusterConfig.getTimeout(), diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/RedisClusterStateQuerier.java b/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/RedisClusterStateQuerier.java index 88f61a183a5..ee402937978 100644 --- a/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/RedisClusterStateQuerier.java +++ b/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/RedisClusterStateQuerier.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

    - * http://www.apache.org/licenses/LICENSE-2.0 - *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

    http://www.apache.org/licenses/LICENSE-2.0 + * + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -51,7 +57,8 @@ protected List retrieveValuesFromRedis(RedisClusterState state, Listhttp://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -47,7 +53,8 @@ public RedisClusterStateUpdater withExpire(int expireIntervalSec) { * {@inheritDoc} */ @Override - protected void updateStatesToRedis(RedisClusterState redisClusterState, Map keyToValue) { + protected void updateStatesToRedis(RedisClusterState redisClusterState, Map keyToValue) { JedisCluster jedisCluster = null; try { jedisCluster = redisClusterState.getJedisCluster(); @@ -68,7 +75,8 @@ protected void updateStatesToRedis(RedisClusterState redisClusterState, Maphttp://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -79,7 +85,8 @@ public static StateFactory opaque(JedisPoolConfig jedisPoolConfig) { * @param dataTypeDescription definition of data type * @return StateFactory */ - public static StateFactory opaque(JedisPoolConfig jedisPoolConfig, RedisDataTypeDescription dataTypeDescription) { + public static StateFactory opaque(JedisPoolConfig jedisPoolConfig, + RedisDataTypeDescription dataTypeDescription) { Options opts = new Options(); opts.dataTypeDescription = dataTypeDescription; return opaque(jedisPoolConfig, opts); @@ -126,7 +133,8 @@ public static StateFactory transactional(JedisPoolConfig jedisPoolConfig) { * @param dataTypeDescription definition of data type * @return StateFactory */ - public static StateFactory transactional(JedisPoolConfig jedisPoolConfig, RedisDataTypeDescription dataTypeDescription) { + public static StateFactory transactional(JedisPoolConfig jedisPoolConfig, + RedisDataTypeDescription dataTypeDescription) { Options opts = new Options(); opts.dataTypeDescription = dataTypeDescription; return transactional(jedisPoolConfig, opts); @@ -152,7 +160,8 @@ public static StateFactory transactional(JedisPoolConfig jedisPoolConfig, KeyFac * @param opts options of State * @return StateFactory */ - public static StateFactory transactional(JedisPoolConfig jedisPoolConfig, Options opts) { + public static StateFactory transactional(JedisPoolConfig jedisPoolConfig, + Options opts) { return new Factory(jedisPoolConfig, StateType.TRANSACTIONAL, opts); } @@ -173,7 +182,8 @@ public static StateFactory nonTransactional(JedisPoolConfig jedisPoolConfig) { * @param dataTypeDescription definition of data type * @return StateFactory */ - public static StateFactory nonTransactional(JedisPoolConfig jedisPoolConfig, RedisDataTypeDescription dataTypeDescription) { + public static StateFactory nonTransactional(JedisPoolConfig jedisPoolConfig, + RedisDataTypeDescription dataTypeDescription) { Options opts = new Options(); opts.dataTypeDescription = dataTypeDescription; return nonTransactional(jedisPoolConfig, opts); @@ -186,7 +196,8 @@ public static StateFactory nonTransactional(JedisPoolConfig jedisPoolConfig, Red * @param factory key factory * @return StateFactory */ - public static StateFactory nonTransactional(JedisPoolConfig jedisPoolConfig, KeyFactory factory) { + public static StateFactory nonTransactional(JedisPoolConfig jedisPoolConfig, + KeyFactory factory) { Options opts = new Options(); opts.keyFactory = factory; return nonTransactional(jedisPoolConfig, opts); @@ -199,7 +210,8 @@ public static StateFactory nonTransactional(JedisPoolConfig jedisPoolConfig, Key * @param opts options of State * @return StateFactory */ - public static StateFactory nonTransactional(JedisPoolConfig jedisPoolConfig, Options opts) { + public static StateFactory nonTransactional(JedisPoolConfig jedisPoolConfig, + Options opts) { return new Factory(jedisPoolConfig, StateType.NON_TRANSACTIONAL, opts); } @@ -239,7 +251,8 @@ protected List retrieveValuesFromRedis(List keys) { return jedis.hmget(description.getAdditionalKey(), stringKeys); default: - throw new IllegalArgumentException("Cannot process such data type: " + description.getDataType()); + throw new IllegalArgumentException("Cannot process such data type: " + + description.getDataType()); } } finally { @@ -276,12 +289,14 @@ protected void updateStatesToRedis(Map keyValues) { case HASH: jedis.hmset(description.getAdditionalKey(), keyValues); if (this.options.expireIntervalSec > 0) { - jedis.expire(description.getAdditionalKey(), this.options.expireIntervalSec); + jedis.expire(description.getAdditionalKey(), + this.options.expireIntervalSec); } break; default: - throw new IllegalArgumentException("Cannot process such data type: " + description.getDataType()); + throw new IllegalArgumentException("Cannot process such data type: " + + description.getDataType()); } } finally { @@ -307,7 +322,8 @@ private String[] buildKeyValuesList(Map keyValues) { * RedisMapState.Factory provides single Redis environment version of StateFactory. */ protected static class Factory implements StateFactory { - public static final redis.clients.jedis.JedisPoolConfig DEFAULT_POOL_CONFIG = new redis.clients.jedis.JedisPoolConfig(); + public static final redis.clients.jedis.JedisPoolConfig DEFAULT_POOL_CONFIG = + new redis.clients.jedis.JedisPoolConfig(); JedisPoolConfig jedisPoolConfig; @@ -345,7 +361,8 @@ public Factory(JedisPoolConfig jedisPoolConfig, StateType type, Options options) * {@inheritDoc} */ @Override - public State makeState(Map conf, IMetricsContext metrics, int partitionIndex, int numPartitions) { + public State makeState(Map conf, IMetricsContext metrics, + int partitionIndex, int numPartitions) { JedisPool jedisPool = new JedisPool(DEFAULT_POOL_CONFIG, jedisPoolConfig.getHost(), jedisPoolConfig.getPort(), diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/RedisState.java b/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/RedisState.java index 88089d9dbef..f47e7aaddf1 100644 --- a/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/RedisState.java +++ b/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/RedisState.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -75,7 +81,8 @@ public void returnJedis(Jedis jedis) { * @see StateFactory */ public static class Factory implements StateFactory { - public static final redis.clients.jedis.JedisPoolConfig DEFAULT_POOL_CONFIG = new redis.clients.jedis.JedisPoolConfig(); + public static final redis.clients.jedis.JedisPoolConfig DEFAULT_POOL_CONFIG = + new redis.clients.jedis.JedisPoolConfig(); private JedisPoolConfig jedisPoolConfig; @@ -92,7 +99,8 @@ public Factory(JedisPoolConfig config) { * {@inheritDoc} */ @Override - public State makeState(Map conf, IMetricsContext metrics, int partitionIndex, int numPartitions) { + public State makeState(Map conf, IMetricsContext metrics, + int partitionIndex, int numPartitions) { JedisPool jedisPool = new JedisPool(DEFAULT_POOL_CONFIG, jedisPoolConfig.getHost(), jedisPoolConfig.getPort(), diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/RedisStateQuerier.java b/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/RedisStateQuerier.java index eea76c3ad3b..d4a4ac60551 100644 --- a/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/RedisStateQuerier.java +++ b/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/RedisStateQuerier.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -50,7 +56,8 @@ protected List retrieveValuesFromRedis(RedisState state, List ke redisVals = jedis.hmget(additionalKey, keysForRedis); break; default: - throw new IllegalArgumentException("Cannot process such data type: " + dataType); + throw new IllegalArgumentException("Cannot process such data type: " + + dataType); } return redisVals; diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/RedisStateUpdater.java b/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/RedisStateUpdater.java index 8984468d4ab..cc512976a89 100644 --- a/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/RedisStateUpdater.java +++ b/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/RedisStateUpdater.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -70,7 +76,8 @@ protected void updateStatesToRedis(RedisState redisState, Map ke pipeline.hset(additionalKey, key, value); break; default: - throw new IllegalArgumentException("Cannot process such data type: " + dataType); + throw new IllegalArgumentException("Cannot process such data type: " + + dataType); } } diff --git a/external/storm-redis/src/test/java/org/apache/storm/redis/bolt/RedisFilterBoltTest.java b/external/storm-redis/src/test/java/org/apache/storm/redis/bolt/RedisFilterBoltTest.java index dc80361a600..543ef4384f1 100644 --- a/external/storm-redis/src/test/java/org/apache/storm/redis/bolt/RedisFilterBoltTest.java +++ b/external/storm-redis/src/test/java/org/apache/storm/redis/bolt/RedisFilterBoltTest.java @@ -18,6 +18,20 @@ package org.apache.storm.redis.bolt; +import static org.apache.storm.redis.common.mapper.RedisDataTypeDescription.RedisDataType.GEO; +import static org.apache.storm.redis.common.mapper.RedisDataTypeDescription.RedisDataType.HASH; +import static org.apache.storm.redis.common.mapper.RedisDataTypeDescription.RedisDataType.HYPER_LOG_LOG; +import static org.apache.storm.redis.common.mapper.RedisDataTypeDescription.RedisDataType.SET; +import static org.apache.storm.redis.common.mapper.RedisDataTypeDescription.RedisDataType.SORTED_SET; +import static org.apache.storm.redis.common.mapper.RedisDataTypeDescription.RedisDataType.STRING; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verifyNoMoreInteractions; + +import java.util.HashMap; +import java.util.Map; import org.apache.storm.redis.common.config.JedisPoolConfig; import org.apache.storm.redis.common.mapper.RedisDataTypeDescription; import org.apache.storm.redis.common.mapper.RedisFilterMapper; @@ -40,21 +54,6 @@ import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; -import java.util.HashMap; -import java.util.Map; - -import static org.apache.storm.redis.common.mapper.RedisDataTypeDescription.RedisDataType.GEO; -import static org.apache.storm.redis.common.mapper.RedisDataTypeDescription.RedisDataType.HASH; -import static org.apache.storm.redis.common.mapper.RedisDataTypeDescription.RedisDataType.HYPER_LOG_LOG; -import static org.apache.storm.redis.common.mapper.RedisDataTypeDescription.RedisDataType.SET; -import static org.apache.storm.redis.common.mapper.RedisDataTypeDescription.RedisDataType.SORTED_SET; -import static org.apache.storm.redis.common.mapper.RedisDataTypeDescription.RedisDataType.STRING; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verifyNoMoreInteractions; - @Testcontainers class RedisFilterBoltTest { @@ -435,6 +434,7 @@ void smokeTest_geopos_isMember() { /** * Utility method to help verify that a tuple passed throught hte RedisFilterBolt properly. + * * @param expectedTuple The tuple we expected to pass through the bolt. */ private void verifyTuplePassed(final Tuple expectedTuple) { @@ -443,10 +443,12 @@ private void verifyTuplePassed(final Tuple expectedTuple) { assertTrue(outputCollector.getFailedTuples().isEmpty(), "Should have no failed tuples"); // We should have a single acked tuple - assertEquals(1, outputCollector.getAckedTuples().size(), "Should have a single acked tuple"); + assertEquals(1, outputCollector.getAckedTuples().size(), + "Should have a single acked tuple"); // We should have a single emitted tuple. - assertEquals(1, outputCollector.getEmittedTuples().size(), "Should have a single emitted tuple"); + assertEquals(1, outputCollector.getEmittedTuples().size(), + "Should have a single emitted tuple"); // Verify the tuple is what we expected final EmittedTuple emittedTuple = outputCollector.getEmittedTuples().get(0); @@ -464,7 +466,8 @@ private void verifyTupleFiltered() { assertTrue(outputCollector.getFailedTuples().isEmpty(), "Should have no failed tuples"); // We should have a single acked tuple - assertEquals(1, outputCollector.getAckedTuples().size(), "Should have a single acked tuple"); + assertEquals(1, outputCollector.getAckedTuples().size(), + "Should have a single acked tuple"); // We should have no emitted tuple. assertTrue(outputCollector.getEmittedTuples().isEmpty(), "Should have no emitted tuples"); @@ -481,7 +484,8 @@ private TestMapper(final RedisDataTypeDescription.RedisDataType dataType) { this(dataType, null); } - private TestMapper(final RedisDataTypeDescription.RedisDataType dataType, final String additionalKey) { + private TestMapper(final RedisDataTypeDescription.RedisDataType dataType, + final String additionalKey) { this.dataType = dataType; this.additionalKey = additionalKey; } @@ -506,4 +510,4 @@ public String getValueFromTuple(final ITuple tuple) { return tuple.getStringByField("value"); } } -} \ No newline at end of file +} diff --git a/external/storm-redis/src/test/java/org/apache/storm/redis/state/RedisKeyValueStateIteratorTest.java b/external/storm-redis/src/test/java/org/apache/storm/redis/state/RedisKeyValueStateIteratorTest.java index 00f4a0eb498..08731b181ff 100644 --- a/external/storm-redis/src/test/java/org/apache/storm/redis/state/RedisKeyValueStateIteratorTest.java +++ b/external/storm-redis/src/test/java/org/apache/storm/redis/state/RedisKeyValueStateIteratorTest.java @@ -1,17 +1,30 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.redis.state; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + import com.google.common.primitives.UnsignedBytes; import java.util.ArrayList; import java.util.Map; @@ -27,14 +40,6 @@ import redis.clients.jedis.params.ScanParams; import redis.clients.jedis.resps.ScanResult; -import static org.junit.jupiter.api.Assertions.assertArrayEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - /** * Test for RedisKeyValueStateIterator. */ @@ -79,8 +84,10 @@ public void testGetEntriesFromFirstPartOfChunkInRedis() { .thenReturn(scanResultFirst, scanResultSecond); RedisKeyValueStateIterator kvIterator = - new RedisKeyValueStateIterator<>(namespace, mockContainer, pendingPrepare.entrySet().iterator(), - pendingCommit.entrySet().iterator(), chunkSize, keySerializer, valueSerializer); + new RedisKeyValueStateIterator<>(namespace, mockContainer, pendingPrepare.entrySet() + .iterator(), + pendingCommit.entrySet() + .iterator(), chunkSize, keySerializer, valueSerializer); assertNextEntry(kvIterator, "key0".getBytes(), "value0".getBytes()); @@ -114,8 +121,10 @@ public void testGetEntriesFromThirdPartOfChunkInRedis() { .thenReturn(scanResultFirst, scanResultSecond, scanResultThird); RedisKeyValueStateIterator kvIterator = - new RedisKeyValueStateIterator<>(namespace, mockContainer, pendingPrepare.entrySet().iterator(), - pendingCommit.entrySet().iterator(), chunkSize, keySerializer, valueSerializer); + new RedisKeyValueStateIterator<>(namespace, mockContainer, pendingPrepare.entrySet() + .iterator(), + pendingCommit.entrySet() + .iterator(), chunkSize, keySerializer, valueSerializer); assertNextEntry(kvIterator, "key0".getBytes(), "value0".getBytes()); @@ -154,8 +163,10 @@ public void testGetEntriesRemovingDuplicationKeys() { .thenReturn(scanResultFirst, scanResultSecond, scanResultThird); RedisKeyValueStateIterator kvIterator = - new RedisKeyValueStateIterator<>(namespace, mockContainer, pendingPrepare.entrySet().iterator(), - pendingCommit.entrySet().iterator(), chunkSize, keySerializer, valueSerializer); + new RedisKeyValueStateIterator<>(namespace, mockContainer, pendingPrepare.entrySet() + .iterator(), + pendingCommit.entrySet() + .iterator(), chunkSize, keySerializer, valueSerializer); // keys shouldn't appear twice @@ -182,13 +193,16 @@ public void testGetEntryNotAvailable() { .thenReturn(scanResult); RedisKeyValueStateIterator kvIterator = - new RedisKeyValueStateIterator<>(namespace, mockContainer, pendingPrepare.entrySet().iterator(), - pendingCommit.entrySet().iterator(), chunkSize, keySerializer, valueSerializer); + new RedisKeyValueStateIterator<>(namespace, mockContainer, pendingPrepare.entrySet() + .iterator(), + pendingCommit.entrySet() + .iterator(), chunkSize, keySerializer, valueSerializer); assertFalse(kvIterator.hasNext()); } - private void assertNextEntry(RedisKeyValueStateIterator kvIterator, byte[] expectedKey, + private void assertNextEntry(RedisKeyValueStateIterator kvIterator, + byte[] expectedKey, byte[] expectedValue) { assertTrue(kvIterator.hasNext()); Map.Entry entry = kvIterator.next(); @@ -196,7 +210,8 @@ private void assertNextEntry(RedisKeyValueStateIterator kvIterat assertArrayEquals(expectedValue, entry.getValue()); } - private void putEncodedKeyValueToMap(NavigableMap map, byte[] key, byte[] value) { + private void putEncodedKeyValueToMap(NavigableMap map, byte[] key, + byte[] value) { map.put(encoder.encodeKey(key), encoder.encodeValue(value)); } @@ -207,4 +222,4 @@ private void putTombstoneToMap(NavigableMap map, byte[] key) { private TreeMap getBinaryTreeMap() { return new TreeMap<>(UnsignedBytes.lexicographicalComparator()); } -} \ No newline at end of file +} diff --git a/external/storm-redis/src/test/java/org/apache/storm/redis/state/RedisKeyValueStateProviderTest.java b/external/storm-redis/src/test/java/org/apache/storm/redis/state/RedisKeyValueStateProviderTest.java index 318ec30923d..2930ef9913f 100644 --- a/external/storm-redis/src/test/java/org/apache/storm/redis/state/RedisKeyValueStateProviderTest.java +++ b/external/storm-redis/src/test/java/org/apache/storm/redis/state/RedisKeyValueStateProviderTest.java @@ -1,27 +1,33 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.redis.state; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + import java.util.HashMap; import java.util.Map; import org.apache.storm.Config; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; - /** - * Unit tests for {@link RedisKeyValueStateProvider} + * Unit tests for {@link RedisKeyValueStateProvider}. */ public class RedisKeyValueStateProviderTest { @@ -30,22 +36,23 @@ public void testgetDefaultConfig() throws Exception { RedisKeyValueStateProvider provider = new RedisKeyValueStateProvider(); Map topoConf = new HashMap<>(); - //topoConf.put(Config.TOPOLOGY_STATE_PROVIDER_CONFIG, "{\"keyClass\":\"String\"}"); + // topoConf.put(Config.TOPOLOGY_STATE_PROVIDER_CONFIG, "{\"keyClass\":\"String\"}"); RedisKeyValueStateProvider.StateConfig config = provider.getStateConfig(topoConf); assertNotNull(config); } - @Test public void testgetConfigWithProviderConfig() throws Exception { RedisKeyValueStateProvider provider = new RedisKeyValueStateProvider(); Map topoConf = new HashMap<>(); - topoConf.put(Config.TOPOLOGY_STATE_PROVIDER_CONFIG, "{\"keyClass\":\"String\", \"valueClass\":\"String\"," + - " \"jedisPoolConfig\":" + - "{\"host\":\"localhost\", \"port\":1000}}"); + topoConf.put(Config.TOPOLOGY_STATE_PROVIDER_CONFIG, + "{\"keyClass\":\"String\", \"valueClass\":\"String\"," + + " \"jedisPoolConfig\":" + + "{\"host\":\"localhost\", " + + "\"port\":1000}}"); RedisKeyValueStateProvider.StateConfig config = provider.getStateConfig(topoConf); - //System.out.println(config); + // System.out.println(config); assertEquals("String", config.keyClass); assertEquals("String", config.valueClass); assertEquals("localhost", config.jedisPoolConfig.getHost()); diff --git a/external/storm-redis/src/test/java/org/apache/storm/redis/state/RedisKeyValueStateTest.java b/external/storm-redis/src/test/java/org/apache/storm/redis/state/RedisKeyValueStateTest.java index f82ef6fa4df..6c649960ad0 100644 --- a/external/storm-redis/src/test/java/org/apache/storm/redis/state/RedisKeyValueStateTest.java +++ b/external/storm-redis/src/test/java/org/apache/storm/redis/state/RedisKeyValueStateTest.java @@ -1,17 +1,26 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.redis.state; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + import com.google.common.primitives.UnsignedBytes; import java.util.Arrays; import java.util.HashMap; @@ -29,14 +38,10 @@ import org.mockito.stubbing.Answer; import redis.clients.jedis.util.SafeEncoder; -import static org.junit.jupiter.api.Assertions.assertArrayEquals; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; - /** - * Unit tests for {@link RedisKeyValueState} + * Unit tests for {@link RedisKeyValueState}. * - * NOTE: The type of key for mockMap is String, which should be byte[], + *

    NOTE: The type of key for mockMap is String, which should be byte[], * since but byte[] doesn't implement equals() so taking workaround to make life happier. * It shouldn't make issues on Redis side, since raw type of Redis is binary. */ @@ -84,7 +89,8 @@ public void setUp() { .thenAnswer((Answer) invocation -> { Object[] args = invocation.getArguments(); int argsSize = args.length; - byte[][] fields = Arrays.asList(args).subList(1, argsSize).toArray(new byte[argsSize - 1][]); + byte[][] fields = Arrays.asList(args).subList(1, argsSize) + .toArray(new byte[argsSize - 1][]); return hdel(mockMap, (byte[]) args[0], fields); }); @@ -106,7 +112,8 @@ public void setUp() { return hgetAll(mockMap, (String) args[0]); }); - keyValueState = new RedisKeyValueState<>("test", mockContainer, new DefaultStateSerializer<>(), + keyValueState = new RedisKeyValueState<>("test", mockContainer, + new DefaultStateSerializer<>(), new DefaultStateSerializer<>()); } @@ -177,7 +184,12 @@ private Boolean exists(NavigableMap> mockMa return mockMap.containsKey(key); } - private String hmset(NavigableMap> mockMap, byte[] key, Map value) { + private Boolean exists(NavigableMap> mockMap, String key) { + return mockMap.containsKey(SafeEncoder.encode(key)); + } + + private String hmset(NavigableMap> mockMap, byte[] key, + Map value) { NavigableMap currentValue = mockMap.get(key); if (currentValue == null) { currentValue = new TreeMap<>(UnsignedBytes.lexicographicalComparator()); @@ -189,6 +201,22 @@ private String hmset(NavigableMap> mockMap, return ""; } + private String hmset(NavigableMap> mockMap, String key, + Map value) { + NavigableMap currentValue = mockMap.get(SafeEncoder.encode(key)); + if (currentValue == null) { + currentValue = new TreeMap<>(UnsignedBytes.lexicographicalComparator()); + } + + for (Map.Entry entry : value.entrySet()) { + currentValue.put(SafeEncoder.encode(entry.getKey()), SafeEncoder.encode(entry + .getValue())); + } + + mockMap.put(SafeEncoder.encode(key), currentValue); + return ""; + } + private Long del(NavigableMap> mockMap, byte[] key) { if (mockMap.remove(key) == null) { return 0L; @@ -197,41 +225,27 @@ private Long del(NavigableMap> mockMap, byt } } - private byte[] hget(NavigableMap> mockMap, byte[] namespace, byte[] key) { + private byte[] hget(NavigableMap> mockMap, + byte[] namespace, byte[] key) { if (mockMap.containsKey(namespace)) { return mockMap.get(namespace).get(key); } return null; } - private Long hdel(NavigableMap> mockMap, byte[] namespace, byte[]... keys) { + private Long hdel(NavigableMap> mockMap, byte[] namespace, + byte[]... keys) { Long count = 0L; for (byte[] key : keys) { - if (mockMap.get(namespace).remove(key) != null) count++; + if (mockMap.get(namespace).remove(key) != null) { + count++; + } } return count; } - private Boolean exists(NavigableMap> mockMap, String key) { - return mockMap.containsKey(SafeEncoder.encode(key)); - } - - - private String hmset(NavigableMap> mockMap, String key, Map value) { - NavigableMap currentValue = mockMap.get(SafeEncoder.encode(key)); - if (currentValue == null) { - currentValue = new TreeMap<>(UnsignedBytes.lexicographicalComparator()); - } - - for (Map.Entry entry : value.entrySet()) { - currentValue.put(SafeEncoder.encode(entry.getKey()), SafeEncoder.encode(entry.getValue())); - } - - mockMap.put(SafeEncoder.encode(key), currentValue); - return ""; - } - - private Map hgetAll(NavigableMap> mockMap, String key) { + private Map hgetAll(NavigableMap> mockMap, + String key) { Map currentValue = mockMap.get(SafeEncoder.encode(key)); Map converted = new HashMap<>(currentValue.size()); diff --git a/external/storm-redis/src/test/java/org/apache/storm/redis/util/JedisTestHelper.java b/external/storm-redis/src/test/java/org/apache/storm/redis/util/JedisTestHelper.java index 9c357493f83..8be896f9d07 100644 --- a/external/storm-redis/src/test/java/org/apache/storm/redis/util/JedisTestHelper.java +++ b/external/storm-redis/src/test/java/org/apache/storm/redis/util/JedisTestHelper.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -24,6 +30,7 @@ public class JedisTestHelper { /** * Constructor. + * * @param container Container instance to create a redis client against. */ public JedisTestHelper(final GenericContainer container) { @@ -39,7 +46,8 @@ public void delete(final String key) { jedis.del(key); } - public void geoadd(final String key, final double longitude, final double latitude, final String value) { + public void geoadd(final String key, final double longitude, final double latitude, + final String value) { jedis.geoadd(key, longitude, latitude, value); } diff --git a/external/storm-redis/src/test/java/org/apache/storm/redis/util/StubTuple.java b/external/storm-redis/src/test/java/org/apache/storm/redis/util/StubTuple.java index 2aab8349209..a651e7a6ed5 100644 --- a/external/storm-redis/src/test/java/org/apache/storm/redis/util/StubTuple.java +++ b/external/storm-redis/src/test/java/org/apache/storm/redis/util/StubTuple.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -25,7 +31,6 @@ import org.apache.storm.tuple.MessageId; import org.apache.storm.tuple.Tuple; - /** * Partial Implementation of the Tuple interface for tests. */ diff --git a/external/storm-redis/src/test/java/org/apache/storm/redis/util/TupleTestHelper.java b/external/storm-redis/src/test/java/org/apache/storm/redis/util/TupleTestHelper.java index 0580ec18cf9..ab8a6accb40 100644 --- a/external/storm-redis/src/test/java/org/apache/storm/redis/util/TupleTestHelper.java +++ b/external/storm-redis/src/test/java/org/apache/storm/redis/util/TupleTestHelper.java @@ -1,25 +1,31 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.redis.util; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + import java.util.List; import java.util.Objects; import org.apache.storm.redis.util.outputcollector.EmittedTuple; import org.apache.storm.tuple.Tuple; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; - /** * Utility for common test validations. */ @@ -35,7 +41,8 @@ public static void verifyAnchors(final EmittedTuple emittedTuple, final Tuple ex assertEquals(expectedAnchor, anchor); } - public static void verifyEmittedTuple(final EmittedTuple emittedTuple, final List expectedValues) { + public static void verifyEmittedTuple(final EmittedTuple emittedTuple, + final List expectedValues) { Objects.requireNonNull(emittedTuple); Objects.requireNonNull(expectedValues); diff --git a/external/storm-redis/src/test/java/org/apache/storm/redis/util/outputcollector/EmittedTuple.java b/external/storm-redis/src/test/java/org/apache/storm/redis/util/outputcollector/EmittedTuple.java index 0196db92b72..116bd398bb9 100644 --- a/external/storm-redis/src/test/java/org/apache/storm/redis/util/outputcollector/EmittedTuple.java +++ b/external/storm-redis/src/test/java/org/apache/storm/redis/util/outputcollector/EmittedTuple.java @@ -1,23 +1,28 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.redis.util.outputcollector; -import org.apache.storm.tuple.Tuple; - import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.List; +import org.apache.storm.tuple.Tuple; /** * Used with StubOutputCollector for testing. @@ -27,7 +32,8 @@ public class EmittedTuple { private final List tuple; private final List anchors; - public EmittedTuple(final String streamId, final List tuple, final Collection anchors) { + public EmittedTuple(final String streamId, final List tuple, + final Collection anchors) { this.streamId = streamId; this.tuple = tuple; this.anchors = new ArrayList<>(anchors); diff --git a/external/storm-redis/src/test/java/org/apache/storm/redis/util/outputcollector/StubOutputCollector.java b/external/storm-redis/src/test/java/org/apache/storm/redis/util/outputcollector/StubOutputCollector.java index 3fec79de118..bfd9ecb7f23 100644 --- a/external/storm-redis/src/test/java/org/apache/storm/redis/util/outputcollector/StubOutputCollector.java +++ b/external/storm-redis/src/test/java/org/apache/storm/redis/util/outputcollector/StubOutputCollector.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -30,7 +36,8 @@ public class StubOutputCollector implements IOutputCollector { final List reportedErrors = new ArrayList<>(); @Override - public List emit(final String streamId, final Collection anchors, final List tuple) { + public List emit(final String streamId, final Collection anchors, + final List tuple) { emittedTuples.add( new EmittedTuple(streamId, tuple, anchors) ); @@ -40,7 +47,8 @@ public List emit(final String streamId, final Collection anchors } @Override - public void emitDirect(final int taskId, final String streamId, final Collection anchors, final List tuple) { + public void emitDirect(final int taskId, final String streamId, final Collection anchors, + final List tuple) { throw new RuntimeException("Not implemented yet!"); } diff --git a/flux/flux-core/pom.xml b/flux/flux-core/pom.xml index 3f5b9957a4d..2fcb43fa437 100644 --- a/flux/flux-core/pom.xml +++ b/flux/flux-core/pom.xml @@ -114,6 +114,16 @@ + + org.openrewrite.maven + rewrite-maven-plugin + + + + org.codehaus.gmavenplus + gmavenplus-plugin + + org.apache.maven.plugins maven-checkstyle-plugin diff --git a/flux/flux-core/src/main/java/org/apache/storm/flux/Flux.java b/flux/flux-core/src/main/java/org/apache/storm/flux/Flux.java index 815a7bc1b18..7f574c1a214 100644 --- a/flux/flux-core/src/main/java/org/apache/storm/flux/Flux.java +++ b/flux/flux-core/src/main/java/org/apache/storm/flux/Flux.java @@ -24,7 +24,6 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.util.Properties; - import org.apache.commons.cli.BasicParser; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; @@ -72,6 +71,7 @@ public class Flux { /** * Flux main entry point. + * * @param args command line arguments * @throws Exception if parsing/topology creation fails */ @@ -80,35 +80,45 @@ public static void main(String[] args) throws Exception { options.addOption(option(0, "h", OPTION_HELP, "Print this help message")); - options.addOption(option(0, "l", OPTION_LOCAL, "Ignored: to run in local mode use `storm local`" + options.addOption(option(0, "l", OPTION_LOCAL, + "Ignored: to run in local mode use `storm local`" + " instead of `storm jar`")); options.addOption(option(0, "r", OPTION_REMOTE, "Ignored: to run on a remote cluster launch" + " using `storm jar` to run in a local cluster use `storm local`")); - options.addOption(option(0, "R", OPTION_RESOURCE, "Treat the supplied path as a classpath resource instead of a file.")); + options.addOption(option(0, "R", OPTION_RESOURCE, + "Treat the supplied path as a classpath resource instead of a file.")); options.addOption(option(1, "s", OPTION_SLEEP, "ms", "Ignored: to set cluster run time" + " use `--local-ttl` with `storm local` instead.")); - options.addOption(option(0, "d", OPTION_DRY_RUN, "Do not run or deploy the topology. Just build, validate, " + options.addOption(option(0, "d", OPTION_DRY_RUN, + "Do not run or deploy the topology. Just build, validate, " + "and print information about the topology.")); - options.addOption(option(0, "q", OPTION_NO_DETAIL, "Suppress the printing of topology details.")); + options.addOption(option(0, "q", OPTION_NO_DETAIL, + "Suppress the printing of topology details.")); - options.addOption(option(0, "n", OPTION_NO_SPLASH, "Suppress the printing of the splash screen.")); + options.addOption(option(0, "n", OPTION_NO_SPLASH, + "Suppress the printing of the splash screen.")); - options.addOption(option(0, "i", OPTION_INACTIVE, "Deploy the topology, but do not activate it.")); + options.addOption(option(0, "i", OPTION_INACTIVE, + "Deploy the topology, but do not activate it.")); options.addOption(option(1, "z", OPTION_ZOOKEEPER, "host:port", "Ignored, if you want to" + " set the zookeeper host/port in local mode use `--local-zookeeper` instead")); - options.addOption(option(1, "f", OPTION_FILTER, "file", "Perform property substitution. Use the specified file " - + "as a source of properties, and replace keys identified with {$[property name]} with the value defined " + options.addOption(option(1, "f", OPTION_FILTER, "file", + "Perform property substitution. Use the specified file " + + "as a source of properties, and replace keys identified with {$[property name]} " + + "with the value defined " + "in the properties file.")); - options.addOption(option(0, "e", OPTION_ENV_FILTER, "Perform environment variable substitution. Replace keys" - + " identified with `${ENV-[NAME]}` will be replaced with the corresponding `NAME` environment value")); + options.addOption(option(0, "e", OPTION_ENV_FILTER, + "Perform environment variable substitution. Replace keys" + + " identified with `${ENV-[NAME]}` will be replaced with the corresponding " + + "`NAME` environment value")); CommandLineParser parser = new BasicParser(); CommandLine cmd = parser.parse(options, args); @@ -120,11 +130,13 @@ public static void main(String[] args) throws Exception { runCli(cmd); } - private static Option option(int argCount, String shortName, String longName, String description) { + private static Option option(int argCount, String shortName, String longName, + String description) { return option(argCount, shortName, longName, longName, description); } - private static Option option(int argCount, String shortName, String longName, String argName, String description) { + private static Option option(int argCount, String shortName, String longName, String argName, + String description) { Option option = OptionBuilder.hasArgs(argCount) .withArgName(argName) .withLongOpt(longName) diff --git a/flux/flux-core/src/main/java/org/apache/storm/flux/FluxBuilder.java b/flux/flux-core/src/main/java/org/apache/storm/flux/FluxBuilder.java index b715727bb31..71c136ff63f 100644 --- a/flux/flux-core/src/main/java/org/apache/storm/flux/FluxBuilder.java +++ b/flux/flux-core/src/main/java/org/apache/storm/flux/FluxBuilder.java @@ -29,7 +29,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; - import org.apache.storm.Config; import org.apache.storm.flux.model.BeanDef; import org.apache.storm.flux.model.BeanListReference; @@ -65,9 +64,9 @@ public class FluxBuilder { private static final Logger LOG = LoggerFactory.getLogger(FluxBuilder.class); - /** * Given a topology definition, return a populated `org.apache.storm.Config` instance. + * * @param topologyDef topology definition * @return a Storm Config object */ @@ -82,7 +81,9 @@ public static Config buildConfig(TopologyDef topologyDef) { } /** - * Given a topology definition, return a Storm topology that can be run either locally or remotely. + * Given a topology definition, return a Storm topology that can be run either locally or + * remotely. + * * @param context execution context * @return A runable Storm topology * @throws IllegalAccessException if security policy disallows operation @@ -99,7 +100,8 @@ public static StormTopology buildTopology(ExecutionContext context) throws Illeg TopologyDef topologyDef = context.getTopologyDef(); if (!topologyDef.validate()) { - throw new IllegalArgumentException("Invalid topology config. Spouts, bolts and streams cannot be " + throw new IllegalArgumentException("Invalid topology config. Spouts, bolts and " + + "streams cannot be " + "defined in the same configuration as a topologySource."); } @@ -139,7 +141,8 @@ public static StormTopology buildTopology(ExecutionContext context) throws Illeg } /** - * Given a `java.lang.Object` instance and a method name, attempt to find a method that matches the input + * Given a `java.lang.Object` instance and a method name, attempt to find a method that matches + * the input * parameter: `java.util.Map` or `org.apache.storm.Config`. * * @param topologySource object to inspect for the specified method @@ -147,7 +150,8 @@ public static StormTopology buildTopology(ExecutionContext context) throws Illeg * @return a Method object that returns a storm topology * @throws NoSuchMethodException if no such method exists */ - private static Method findGetTopologyMethod(Object topologySource, String methodName) throws NoSuchMethodException { + private static Method findGetTopologyMethod(Object topologySource, + String methodName) throws NoSuchMethodException { Class clazz = topologySource.getClass(); Method[] methods = clazz.getMethods(); ArrayList candidates = new ArrayList(); @@ -162,15 +166,18 @@ private static Method findGetTopologyMethod(Object topologySource, String method if (paramTypes.length != 1) { continue; } - if (paramTypes[0].isAssignableFrom(Map.class) || paramTypes[0].isAssignableFrom(Config.class)) { + if (paramTypes[0].isAssignableFrom(Map.class) + || paramTypes[0].isAssignableFrom(Config.class)) { candidates.add(method); } } if (candidates.size() == 0) { - throw new IllegalArgumentException("Unable to find method '" + methodName + "' method in class: " + clazz.getName()); + throw new IllegalArgumentException("Unable to find method '" + methodName + + "' method in class: " + clazz.getName()); } else if (candidates.size() > 1) { - LOG.warn("Found multiple candidate methods in class '" + clazz.getName() + "'. Using the first one found"); + LOG.warn("Found multiple candidate methods in class '" + clazz.getName() + + "'. Using the first one found"); } return candidates.get(0); @@ -178,6 +185,7 @@ private static Method findGetTopologyMethod(Object topologySource, String method /** * Builds stream definitions. + * * @param context context * @param builder builder */ @@ -201,15 +209,18 @@ private static void buildStreamDefinitions(ExecutionContext context, TopologyBui topologyDef.parallelismForBolt(stream.getTo())); case IStatefulBolt b -> builder.setBolt(stream.getTo(), b, topologyDef.parallelismForBolt(stream.getTo())); - default -> throw new IllegalArgumentException("Class does not appear to be a bolt: " + default -> throw new IllegalArgumentException("Class does not appear to be a " + + "bolt: " + boltObj.getClass().getName()); }; - // resource and config declarations apply to the bolt as a whole, so only apply them once + // resource and config declarations apply to the bolt as a whole, so only apply them + // once // when the declarer is first created rather than on every incoming stream BoltDef boltDef = topologyDef.getBoltDef(stream.getTo()); if (boltDef.getOnHeapMemoryLoad() > -1) { if (boltDef.getOffHeapMemoryLoad() > -1) { - declarer.setMemoryLoad(boltDef.getOnHeapMemoryLoad(), boltDef.getOffHeapMemoryLoad()); + declarer.setMemoryLoad(boltDef.getOnHeapMemoryLoad(), boltDef + .getOffHeapMemoryLoad()); } else { declarer.setMemoryLoad(boltDef.getOnHeapMemoryLoad()); } @@ -227,8 +238,10 @@ private static void buildStreamDefinitions(ExecutionContext context, TopologyBui } GroupingDef grouping = stream.getGrouping(); - // if the streamId is defined, use it for the grouping, otherwise assume storm's default stream - String streamId = (grouping.getStreamId() == null ? Utils.DEFAULT_STREAM_ID : grouping.getStreamId()); + // if the streamId is defined, use it for the grouping, otherwise assume storm's default + // stream + String streamId = (grouping.getStreamId() == null ? Utils.DEFAULT_STREAM_ID : grouping + .getStreamId()); switch (grouping.getType()) { @@ -236,8 +249,9 @@ private static void buildStreamDefinitions(ExecutionContext context, TopologyBui declarer.shuffleGrouping(stream.getFrom(), streamId); break; case FIELDS: - //TODO: check for null grouping args - declarer.fieldsGrouping(stream.getFrom(), streamId, new Fields(grouping.getArgs())); + // TODO: check for null grouping args + declarer.fieldsGrouping(stream.getFrom(), streamId, new Fields(grouping + .getArgs())); break; case ALL: declarer.allGrouping(stream.getFrom(), streamId); @@ -256,24 +270,29 @@ private static void buildStreamDefinitions(ExecutionContext context, TopologyBui break; case CUSTOM: declarer.customGrouping(stream.getFrom(), streamId, - buildCustomStreamGrouping(stream.getGrouping().getCustomClass(), context)); + buildCustomStreamGrouping(stream.getGrouping().getCustomClass(), + context)); break; default: - throw new UnsupportedOperationException("unsupported grouping type: " + grouping); + throw new UnsupportedOperationException("unsupported grouping type: " + + grouping); } } } - private static void applyProperties(ObjectDef bean, Object instance, ExecutionContext context) throws + private static void applyProperties(ObjectDef bean, Object instance, + ExecutionContext context) throws IllegalAccessException, InvocationTargetException, NoSuchFieldException { List props = bean.getProperties(); Class clazz = instance.getClass(); if (props != null) { for (PropertyDef prop : props) { - Object value = prop.isReference() ? context.getComponent(prop.getRef()) : prop.getValue(); + Object value = prop.isReference() ? context.getComponent(prop.getRef()) : prop + .getValue(); Method setter = findSetter(clazz, prop.getName(), value); if (setter != null) { - Object[] methodArgs = getArgsWithListCoercion(Collections.singletonList(value), setter.getParameterTypes()); + Object[] methodArgs = getArgsWithListCoercion(Collections.singletonList(value), + setter.getParameterTypes()); LOG.debug("found setter, attempting to invoke with {}", methodArgs); // invoke setter setter.invoke(instance, methodArgs); @@ -289,7 +308,8 @@ private static void applyProperties(ObjectDef bean, Object instance, ExecutionCo } } - private static Field findPublicField(Class clazz, String property, Object arg) throws NoSuchFieldException { + private static Field findPublicField(Class clazz, String property, + Object arg) throws NoSuchFieldException { Field field = clazz.getField(property); return field; } @@ -301,8 +321,10 @@ private static Method findSetter(Class clazz, String property, Object arg) { for (Method method : methods) { if (setterName.equals(method.getName())) { Class[] parameterTypes = method.getParameterTypes(); - LOG.debug("Found setter method: {}, parameter types: {}", method.getName(), parameterTypes); - boolean invokable = canInvokeWithArgs(Collections.singletonList(arg), method.getParameterTypes()); + LOG.debug("Found setter method: {}, parameter types: {}", method.getName(), + parameterTypes); + boolean invokable = canInvokeWithArgs(Collections.singletonList(arg), method + .getParameterTypes()); LOG.debug("** invokable --> {}", invokable); if (invokable) { return method; @@ -339,12 +361,14 @@ private static List resolveReferences(List args, ExecutionContex return constructorArgs; } - private static Object buildObject(ObjectDef def, ExecutionContext context) throws ClassNotFoundException, + private static Object buildObject(ObjectDef def, + ExecutionContext context) throws ClassNotFoundException, IllegalAccessException, InstantiationException, NoSuchMethodException, InvocationTargetException, NoSuchFieldException { Class clazz = Class.forName(def.getClassName()); Object obj = null; if (def.hasConstructorArgs()) { - LOG.debug("Found constructor arguments in definition: " + def.getConstructorArgs().getClass().getName()); + LOG.debug("Found constructor arguments in definition: " + def.getConstructorArgs() + .getClass().getName()); List constructorArgs = def.getConstructorArgs(); if (def.hasReferences()) { constructorArgs = resolveReferences(constructorArgs, context); @@ -352,9 +376,12 @@ private static Object buildObject(ObjectDef def, ExecutionContext context) throw Constructor con = findCompatibleConstructor(constructorArgs, clazz); if (con != null) { LOG.debug("Found something seemingly compatible, attempting invocation..."); - obj = con.newInstance(getArgsWithListCoercion(constructorArgs, con.getParameterTypes())); + obj = con.newInstance(getArgsWithListCoercion(constructorArgs, con + .getParameterTypes())); } else { - String msg = String.format("Couldn't find a suitable constructor for class '%s' with arguments '%s'.", + String msg = String + .format("Couldn't find a suitable constructor for class '%s' with " + + "arguments '%s'.", clazz.getName(), constructorArgs); throw new IllegalArgumentException(msg); @@ -370,9 +397,12 @@ private static Object buildObject(ObjectDef def, ExecutionContext context) throw } method = findCompatibleMethod(methodArgs, clazz, def.getFactory()); if (method != null) { - obj = method.invoke(null, getArgsWithListCoercion(methodArgs, method.getParameterTypes())); + obj = method.invoke(null, getArgsWithListCoercion(methodArgs, method + .getParameterTypes())); } else { - String msg = String.format("Couldn't find a suitable static method '%s' for class '%s' with arguments '%s'.", + String msg = String + .format("Couldn't find a suitable static method '%s' for class '%s' with " + + "arguments '%s'.", def.getFactory(), clazz.getName(), methodArgs); @@ -400,11 +430,13 @@ private static StormTopology buildExternalTopology(ObjectDef def, ExecutionConte config.putAll(context.getTopologyDef().getConfig()); return (StormTopology) getTopology.invoke(topologySource, config); } else { - return (StormTopology) getTopology.invoke(topologySource, context.getTopologyDef().getConfig()); + return (StormTopology) getTopology.invoke(topologySource, context.getTopologyDef() + .getConfig()); } } - private static CustomStreamGrouping buildCustomStreamGrouping(ObjectDef def, ExecutionContext context) + private static CustomStreamGrouping buildCustomStreamGrouping(ObjectDef def, + ExecutionContext context) throws ClassNotFoundException, IllegalAccessException, InstantiationException, NoSuchMethodException, InvocationTargetException, NoSuchFieldException { Object grouping = buildObject(def, context); @@ -426,8 +458,8 @@ private static void buildComponents(ExecutionContext context) throws ClassNotFou } } - - private static void buildSpouts(ExecutionContext context, TopologyBuilder builder) throws ClassNotFoundException, + private static void buildSpouts(ExecutionContext context, + TopologyBuilder builder) throws ClassNotFoundException, NoSuchMethodException, InvocationTargetException, InstantiationException, IllegalAccessException, NoSuchFieldException { for (SpoutDef sd : context.getTopologyDef().getSpouts()) { IRichSpout spout = buildSpout(sd, context); @@ -453,15 +485,18 @@ private static void buildSpouts(ExecutionContext context, TopologyBuilder builde } /** - * Given a spout definition, return a Storm spout implementation by attempting to find a matching constructor + * Given a spout definition, return a Storm spout implementation by attempting to find a + * matching constructor * in the given spout class. Perform list to array conversion as necessary. */ - private static IRichSpout buildSpout(SpoutDef def, ExecutionContext context) throws ClassNotFoundException, + private static IRichSpout buildSpout(SpoutDef def, + ExecutionContext context) throws ClassNotFoundException, IllegalAccessException, InstantiationException, NoSuchMethodException, InvocationTargetException, NoSuchFieldException { return (IRichSpout) buildObject(def, context); } - private static void applyComponentConfig(Map config, ComponentConfigurationDeclarer declarer) { + private static void applyComponentConfig(Map config, + ComponentConfigurationDeclarer declarer) { if (config == null || config.isEmpty()) { return; } @@ -470,8 +505,10 @@ private static void applyComponentConfig(Map config, ComponentCo } /** - * Given a list of bolt definitions, build a map of Storm bolts with the bolt definition id as the key. - * Attempt to coerce the given constructor arguments to a matching bolt constructor as much as possible. + * Given a list of bolt definitions, build a map of Storm bolts with the bolt definition id as + * the key. + * Attempt to coerce the given constructor arguments to a matching bolt constructor as much as + * possible. */ private static void buildBolts(ExecutionContext context) throws ClassNotFoundException, IllegalAccessException, InstantiationException, NoSuchMethodException, InvocationTargetException, NoSuchFieldException { @@ -483,10 +520,12 @@ private static void buildBolts(ExecutionContext context) throws ClassNotFoundExc } /** - * Given a list of worker hook definitions, build a Storm worker hook implementation by attempting to find a matching + * Given a list of worker hook definitions, build a Storm worker hook implementation by + * attempting to find a matching * constructor in the given worker hook class and add them to the topology builder. */ - private static void buildWorkerHooks(ExecutionContext context, TopologyBuilder builder) throws ClassNotFoundException, + private static void buildWorkerHooks(ExecutionContext context, + TopologyBuilder builder) throws ClassNotFoundException, NoSuchMethodException, InvocationTargetException, InstantiationException, IllegalAccessException, NoSuchFieldException { for (WorkerHookDef whDef : context.getTopologyDef().getWorkerHooks()) { IWorkerHook workerHook = (IWorkerHook) buildObject(whDef, context); @@ -495,9 +534,11 @@ private static void buildWorkerHooks(ExecutionContext context, TopologyBuilder b } /** - * Given a list of constructor arguments, and a target class, attempt to find a suitable constructor. + * Given a list of constructor arguments, and a target class, attempt to find a suitable + * constructor. */ - private static Constructor findCompatibleConstructor(List args, Class target) throws NoSuchMethodException { + private static Constructor findCompatibleConstructor(List args, + Class target) throws NoSuchMethodException { Constructor retval = null; int eligibleCount = 0; @@ -519,22 +560,24 @@ private static Constructor findCompatibleConstructor(List args, Class ta } } if (eligibleCount > 1) { - LOG.warn("Found multiple invokable constructors for class {}, given arguments {}. Using the last one found.", + LOG.warn("Found multiple invokable constructors for class {}, given arguments {}. " + + "Using the last one found.", target, args); } return retval; } - /** * Invokes configuration methods on an class instance. + * * @param bean the bean/component definition * @param instance the class instance being operated on * @param context execution context * @throws InvocationTargetException if method invocation fails * @throws IllegalAccessException if security policy prefents invocation */ - public static void invokeConfigMethods(ObjectDef bean, Object instance, ExecutionContext context) + public static void invokeConfigMethods(ObjectDef bean, Object instance, + ExecutionContext context) throws InvocationTargetException, IllegalAccessException { List methodDefs = bean.getConfigMethods(); @@ -556,8 +599,10 @@ public static void invokeConfigMethods(ObjectDef bean, Object instance, Executio Object[] methodArgs = getArgsWithListCoercion(args, method.getParameterTypes()); method.invoke(instance, methodArgs); } else { - String msg = String.format("Unable to find configuration method '%s' in class '%s' with arguments %s.", - new Object[]{methodName, clazz.getName(), args}); + String msg = String + .format("Unable to find configuration method '%s' in class '%s' with " + + "arguments %s.", + new Object[]{methodName, clazz.getName(), args}); throw new IllegalArgumentException(msg); } } @@ -591,7 +636,8 @@ private static Method findCompatibleMethod(List args, Class target, Stri } } if (eligibleCount > 1) { - LOG.warn("Found multiple invokable methods for class {}, method {}, given arguments {}. " + LOG.warn("Found multiple invokable methods for class {}, method {}, given arguments " + + "{}. " + "Using the last one found.", new Object[]{target, methodName, args}); } @@ -599,13 +645,16 @@ private static Method findCompatibleMethod(List args, Class target, Stri } /** - * Given a java.util.List of contructor/method arguments, and a list of parameter types, attempt to convert the - * list to an java.lang.Object array that can be used to invoke the constructor. If an argument needs + * Given a java.util.List of contructor/method arguments, and a list of parameter types, attempt + * to convert the + * list to an java.lang.Object array that can be used to invoke the constructor. If an argument + * needs * to be coerced from a List to an Array, do so. */ private static Object[] getArgsWithListCoercion(List args, Class[] parameterTypes) { if (parameterTypes.length != args.size()) { - throw new IllegalArgumentException("Contructor parameter count does not egual argument size."); + throw new IllegalArgumentException("Contructor parameter count does not egual " + + "argument size."); } Object[] constructorParams = new Object[args.size()]; @@ -615,7 +664,8 @@ private static Object[] getArgsWithListCoercion(List args, Class[] param Object obj = args.get(i); Class paramType = parameterTypes[i]; Class objectType = obj.getClass(); - LOG.debug("Comparing parameter class {} to object class {} to see if assignment is possible.", + LOG.debug("Comparing parameter class {} to object class {} to see if assignment is " + + "possible.", paramType, objectType); if (paramType.equals(objectType)) { LOG.debug("They are the same class."); @@ -667,7 +717,8 @@ private static Object[] getArgsWithListCoercion(List args, Class[] param // TODO: more collection content type checking LOG.debug("Conversion appears possible..."); List list = (List) obj; - LOG.debug("Array Type: {}, List type: {}", paramType.getComponentType(), list.get(0).getClass()); + LOG.debug("Array Type: {}, List type: {}", paramType.getComponentType(), list.get(0) + .getClass()); // create an array of the right type Object newArrayObj = Array.newInstance(paramType.getComponentType(), list.size()); @@ -682,9 +733,9 @@ private static Object[] getArgsWithListCoercion(List args, Class[] param return constructorParams; } - /** - * Determine if the given constructor/method parameter types are compatible given arguments List. Consider if + * Determine if the given constructor/method parameter types are compatible given arguments + * List. Consider if * list coercian can make it possible. * * @param args arguments @@ -704,13 +755,15 @@ private static boolean canInvokeWithArgs(List args, Class[] parameterTyp } Class paramType = parameterTypes[i]; Class objectType = obj.getClass(); - LOG.debug("Comparing parameter class {} to object class {} to see if assignment is possible.", + LOG.debug("Comparing parameter class {} to object class {} to see if assignment is " + + "possible.", paramType, objectType); if (paramType.equals(objectType)) { LOG.debug("Yes, they are the same class."); } else if (paramType.isAssignableFrom(objectType)) { LOG.debug("Yes, assignment is possible."); - } else if (isPrimitiveBoolean(paramType) && Boolean.class.isAssignableFrom(objectType)) { + } else if (isPrimitiveBoolean(paramType) && Boolean.class + .isAssignableFrom(objectType)) { LOG.debug("Yes, assignment is possible."); } else if (isPrimitiveNumber(paramType) || Number.class.isAssignableFrom(paramType) && Number.class.isAssignableFrom(objectType)) { @@ -724,7 +777,8 @@ private static boolean canInvokeWithArgs(List args, Class[] parameterTyp } else if (paramType.isArray() && List.class.isAssignableFrom(objectType)) { // TODO: more collection content type checking LOG.debug("Assignment is possible if we convert a List to an array."); - LOG.debug("Array Type: {}, List type: {}", paramType.getComponentType(), ((List) obj).get(0).getClass()); + LOG.debug("Array Type: {}, List type: {}", paramType.getComponentType(), + ((List) obj).get(0).getClass()); } else { return false; } diff --git a/flux/flux-core/src/main/java/org/apache/storm/flux/api/TopologySource.java b/flux/flux-core/src/main/java/org/apache/storm/flux/api/TopologySource.java index a18b65056a5..7ac3b31a9d2 100644 --- a/flux/flux-core/src/main/java/org/apache/storm/flux/api/TopologySource.java +++ b/flux/flux-core/src/main/java/org/apache/storm/flux/api/TopologySource.java @@ -19,7 +19,6 @@ package org.apache.storm.flux.api; import java.util.Map; - import org.apache.storm.generated.StormTopology; /** diff --git a/flux/flux-core/src/main/java/org/apache/storm/flux/model/BeanDef.java b/flux/flux-core/src/main/java/org/apache/storm/flux/model/BeanDef.java index 199f4d23b7d..47c60f79023 100644 --- a/flux/flux-core/src/main/java/org/apache/storm/flux/model/BeanDef.java +++ b/flux/flux-core/src/main/java/org/apache/storm/flux/model/BeanDef.java @@ -19,7 +19,8 @@ package org.apache.storm.flux.model; /** - * A representation of a Java object that is uniquely identifyable, and given a className, constructor arguments, + * A representation of a Java object that is uniquely identifyable, and given a className, + * constructor arguments, * and properties, can be instantiated. */ public class BeanDef extends ObjectDef { diff --git a/flux/flux-core/src/main/java/org/apache/storm/flux/model/ConfigMethodDef.java b/flux/flux-core/src/main/java/org/apache/storm/flux/model/ConfigMethodDef.java index 04fec6386b8..3ecdacdcba6 100644 --- a/flux/flux-core/src/main/java/org/apache/storm/flux/model/ConfigMethodDef.java +++ b/flux/flux-core/src/main/java/org/apache/storm/flux/model/ConfigMethodDef.java @@ -42,6 +42,7 @@ public List getArgs() { /** * Set the method arguments. + * * @param args method parameters */ public void setArgs(List args) { diff --git a/flux/flux-core/src/main/java/org/apache/storm/flux/model/ExecutionContext.java b/flux/flux-core/src/main/java/org/apache/storm/flux/model/ExecutionContext.java index ec0529e832f..741854e077e 100644 --- a/flux/flux-core/src/main/java/org/apache/storm/flux/model/ExecutionContext.java +++ b/flux/flux-core/src/main/java/org/apache/storm/flux/model/ExecutionContext.java @@ -21,7 +21,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; - import org.apache.storm.Config; import org.apache.storm.task.IBolt; import org.apache.storm.topology.IRichSpout; diff --git a/flux/flux-core/src/main/java/org/apache/storm/flux/model/ObjectDef.java b/flux/flux-core/src/main/java/org/apache/storm/flux/model/ObjectDef.java index 1ff4fb454c0..870effe6d90 100644 --- a/flux/flux-core/src/main/java/org/apache/storm/flux/model/ObjectDef.java +++ b/flux/flux-core/src/main/java/org/apache/storm/flux/model/ObjectDef.java @@ -50,6 +50,7 @@ public List getConstructorArgs() { /** * Sets the arguments for the constructor and checks for references. + * * @param constructorArgs Constructor arguments */ public void setConstructorArgs(List constructorArgs) { @@ -120,6 +121,7 @@ public List getFactoryArgs() { /** * Sets factory method arguments and checks for references. + * * @param factoryArgs factory method arguments */ public void setFactoryArgs(List factoryArgs) { diff --git a/flux/flux-core/src/main/java/org/apache/storm/flux/model/PropertyDef.java b/flux/flux-core/src/main/java/org/apache/storm/flux/model/PropertyDef.java index 3debba979a2..083a0c08275 100644 --- a/flux/flux-core/src/main/java/org/apache/storm/flux/model/PropertyDef.java +++ b/flux/flux-core/src/main/java/org/apache/storm/flux/model/PropertyDef.java @@ -38,11 +38,13 @@ public Object getValue() { /** * Sets the value of this property. Throws IllegalArgumentException if a reference has * already been set. + * * @param value property value */ public void setValue(Object value) { if (this.ref != null) { - throw new IllegalStateException("A property can only have a value OR a reference, not both."); + throw new IllegalStateException("A property can only have a value OR a reference, not " + + "both."); } this.value = value; } @@ -52,13 +54,16 @@ public String getRef() { } /** - * Sets the value of this property to a reference. Throws IllegalArgumentException if a value has + * Sets the value of this property to a reference. Throws IllegalArgumentException if a value + * has * already been set. + * * @param ref property reference */ public void setRef(String ref) { if (this.value != null) { - throw new IllegalStateException("A property can only have a value OR a reference, not both."); + throw new IllegalStateException("A property can only have a value OR a reference, not " + + "both."); } this.ref = ref; } diff --git a/flux/flux-core/src/main/java/org/apache/storm/flux/model/StreamDef.java b/flux/flux-core/src/main/java/org/apache/storm/flux/model/StreamDef.java index 2a1b87cf436..311ec61310e 100644 --- a/flux/flux-core/src/main/java/org/apache/storm/flux/model/StreamDef.java +++ b/flux/flux-core/src/main/java/org/apache/storm/flux/model/StreamDef.java @@ -19,9 +19,11 @@ package org.apache.storm.flux.model; /** - * Represents a stream of tuples from one Storm component (Spout or Bolt) to another (an edge in the topology DAG). + * Represents a stream of tuples from one Storm component (Spout or Bolt) to another (an edge in the + * topology DAG). *

    - * Required fields are `from` and `to`, which define the source and destination, and the stream `grouping`. + * Required fields are `from` and `to`, which define the source and destination, and the stream + * `grouping`. * */ public class StreamDef { diff --git a/flux/flux-core/src/main/java/org/apache/storm/flux/model/TopologyDef.java b/flux/flux-core/src/main/java/org/apache/storm/flux/model/TopologyDef.java index fe05275b7e0..ac9c1f08c76 100644 --- a/flux/flux-core/src/main/java/org/apache/storm/flux/model/TopologyDef.java +++ b/flux/flux-core/src/main/java/org/apache/storm/flux/model/TopologyDef.java @@ -23,7 +23,6 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -42,7 +41,8 @@ public class TopologyDef { private static Logger LOG = LoggerFactory.getLogger(TopologyDef.class); private String name; - private Map componentMap = new LinkedHashMap(); // not required + private Map componentMap = + new LinkedHashMap(); // not required private List includes; // not required private Map config = new HashMap(); @@ -55,7 +55,6 @@ public class TopologyDef { private List streams = new ArrayList(); private Map workerHookMap = new LinkedHashMap<>(); - public String getName() { return name; } @@ -66,6 +65,7 @@ public void setName(String name) { /** * Sets the name of the topology. + * * @param name topology name * @param override whether to override if already set */ @@ -79,6 +79,7 @@ public void setName(String name, boolean override) { /** * Returns all spout definitions. + * * @return spout definitions. */ public List getSpouts() { @@ -89,6 +90,7 @@ public List getSpouts() { /** * Set spout definitions. + * * @param spouts spout definitions */ public void setSpouts(List spouts) { @@ -100,6 +102,7 @@ public void setSpouts(List spouts) { /** * Returns bolt definitions. + * * @return bolt definitions */ public List getBolts() { @@ -110,6 +113,7 @@ public List getBolts() { /** * Sets bolt definitions. + * * @param bolts bolt definitions */ public void setBolts(List bolts) { @@ -137,6 +141,7 @@ public void setConfig(Map config) { /** * Returns a list of all component definitions. + * * @return components */ public List getComponents() { @@ -147,6 +152,7 @@ public List getComponents() { /** * Sets the list of component definitions. + * * @param components components definitions */ public void setComponents(List components) { @@ -166,6 +172,7 @@ public void setIncludes(List includes) { /** * Returns worker hook definitions. + * * @return worker hook definitions */ public List getWorkerHooks() { @@ -176,6 +183,7 @@ public List getWorkerHooks() { /** * Sets worker hook definitions. + * * @param workerHooks worker hook definitions */ public void setWorkerHooks(List workerHooks) { @@ -209,6 +217,7 @@ public WorkerHookDef getWorkerHook(String id) { /** * Adds a list of bolt definitions. Optionally overriding existing definitions * if one with the same ID already exists. + * * @param bolts bolt definitions * @param override whether or not to override existing definitions */ @@ -226,6 +235,7 @@ public void addAllBolts(List bolts, boolean override) { /** * Adds a list of spout definitions. Optionally overriding existing definitions * if one with the same ID already exists. + * * @param spouts spout definitions * @param override whether or not to override existing definitions */ @@ -243,6 +253,7 @@ public void addAllSpouts(List spouts, boolean override) { /** * Adds a list of component definitions. Optionally overriding existing definitions * if one with the same ID already exists. + * * @param components component definitions * @param override whether or not to override existing definitions */ @@ -260,18 +271,22 @@ public void addAllComponents(List components, boolean override) { /** * Adds a list of stream definitions. Optionally overriding existing definitions * if one with the same ID already exists. + * * @param streams stream definitions * @param override whether or not to override existing definitions (currently ignored) */ public void addAllStreams(List streams, boolean override) { - //TODO: figure out how we want to deal with overrides. Users may want to add streams even when overriding other - // properties. For now we just add them blindly which could lead to a potentially invalid topology. + // TODO: figure out how we want to deal with overrides. Users may want to add streams even + // when overriding other + // properties. For now we just add them blindly which could lead to a potentially invalid + // topology. this.streams.addAll(streams); } /** * Adds a list of worker hook definitions. Optionally overriding existing definitions * if one with the same ID already exists. + * * @param workerHooks worker hook definitions * @param override whether or not to override existing definitions */ @@ -298,9 +313,9 @@ public boolean isDslTopology() { return this.topologySource == null; } - /** * Determines is this represents a valid Topology. + * * @return true if valid */ public boolean validate() { diff --git a/flux/flux-core/src/main/java/org/apache/storm/flux/parser/FluxParser.java b/flux/flux-core/src/main/java/org/apache/storm/flux/parser/FluxParser.java index c96a5fe10ac..c32b6b5f4ac 100644 --- a/flux/flux-core/src/main/java/org/apache/storm/flux/parser/FluxParser.java +++ b/flux/flux-core/src/main/java/org/apache/storm/flux/parser/FluxParser.java @@ -29,7 +29,6 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; - import org.apache.storm.flux.model.BoltDef; import org.apache.storm.flux.model.IncludeDef; import org.apache.storm.flux.model.SpoutDef; @@ -47,7 +46,8 @@ public class FluxParser { private static final Logger LOG = LoggerFactory.getLogger(FluxParser.class); private static final Pattern propertyPattern = - Pattern.compile(".*\\$\\{(?ENV-(?.+)|(?.+)\\[(?\\d+)]|.+)}.*"); + Pattern.compile(".*\\$\\{(?ENV-(?.+)|(?.+)\\[(?\\d+)]|.+" + + ")}.*"); private FluxParser() { } @@ -83,7 +83,8 @@ public static TopologyDef parseFile(String inputFile, boolean dumpYaml, boolean * @return resulting topologuy definition * @throws IOException if there is a problem reading file(s) */ - public static TopologyDef parseResource(String resource, boolean dumpYaml, boolean processIncludes, + public static TopologyDef parseResource(String resource, boolean dumpYaml, + boolean processIncludes, Properties properties, boolean envSub) throws IOException { InputStream in = FluxParser.class.getResourceAsStream(resource); TopologyDef topology = parseInputStream(in, dumpYaml, processIncludes, properties, envSub); @@ -103,7 +104,8 @@ public static TopologyDef parseResource(String resource, boolean dumpYaml, boole * @return resulting topology definition * @throws IOException if there is a problem reading file(s) */ - public static TopologyDef parseInputStream(InputStream inputStream, boolean dumpYaml, boolean processIncludes, + public static TopologyDef parseInputStream(InputStream inputStream, boolean dumpYaml, + boolean processIncludes, Properties properties, boolean envSub) throws IOException { Yaml yaml = yaml(); @@ -133,7 +135,8 @@ public static TopologyDef parseInputStream(InputStream inputStream, boolean dump * @return resulting filter properties * @throws IOException if there is a problem reading file */ - public static Properties parseProperties(String propertiesFile, boolean resource) throws IOException { + public static Properties parseProperties(String propertiesFile, + boolean resource) throws IOException { Properties properties = null; if (propertiesFile != null) { @@ -151,7 +154,8 @@ public static Properties parseProperties(String propertiesFile, boolean resource return properties; } - private static TopologyDef loadYaml(Yaml yaml, InputStream in, Properties properties, boolean envSubstitution) throws IOException { + private static TopologyDef loadYaml(Yaml yaml, InputStream in, Properties properties, + boolean envSubstitution) throws IOException { LOG.info("loading YAML from input stream..."); try (BufferedReader reader = new BufferedReader(new InputStreamReader(in))) { String conf = reader.lines().map(line -> { @@ -170,7 +174,8 @@ private static TopologyDef loadYaml(Yaml yaml, InputStream in, Properties proper } } - private static Optional getPropertyReplacement(Properties properties, Matcher match, boolean envSubstitution) { + private static Optional getPropertyReplacement(Properties properties, Matcher match, + boolean envSubstitution) { if (match.group("listIndex") != null) { String prop = properties.getProperty(match.group("list")); return Optional.of(parseListAndExtractElem(prop, match.group("listIndex"))); @@ -217,18 +222,21 @@ private static Yaml yaml() { * @param envSub whether or not to perform environment variable substitution * @return The TopologyDef with includes resolved. */ - private static TopologyDef processIncludes(Yaml yaml, TopologyDef topologyDef, Properties properties, boolean envSub) + private static TopologyDef processIncludes(Yaml yaml, TopologyDef topologyDef, + Properties properties, boolean envSub) throws IOException { - //TODO: support multiple levels of includes + // TODO: support multiple levels of includes if (topologyDef.getIncludes() != null) { for (IncludeDef include : topologyDef.getIncludes()) { TopologyDef includeTopologyDef = null; if (include.isResource()) { LOG.info("Loading includes from resource: {}", include.getFile()); - includeTopologyDef = parseResource(include.getFile(), true, false, properties, envSub); + includeTopologyDef = parseResource(include.getFile(), true, false, properties, + envSub); } else { LOG.info("Loading includes from file: {}", include.getFile()); - includeTopologyDef = parseFile(include.getFile(), true, false, properties, envSub); + includeTopologyDef = parseFile(include.getFile(), true, false, properties, + envSub); } // if overrides are disabled, we won't replace anything that already exists @@ -240,7 +248,7 @@ private static TopologyDef processIncludes(Yaml yaml, TopologyDef topologyDef, P // config if (includeTopologyDef.getConfig() != null) { - //TODO: move this logic to the model class + // TODO: move this logic to the model class Map config = topologyDef.getConfig(); Map includeConfig = includeTopologyDef.getConfig(); if (override) { @@ -248,7 +256,8 @@ private static TopologyDef processIncludes(Yaml yaml, TopologyDef topologyDef, P } else { for (String key : includeConfig.keySet()) { if (config.containsKey(key)) { - LOG.warn("Ignoring attempt to set topology config property '{}' with override == false", key); + LOG.warn("Ignoring attempt to set topology config property '{}' " + + "with override == false", key); } else { config.put(key, includeConfig.get(key)); } @@ -256,20 +265,20 @@ private static TopologyDef processIncludes(Yaml yaml, TopologyDef topologyDef, P } } - //component overrides + // component overrides if (includeTopologyDef.getComponents() != null) { topologyDef.addAllComponents(includeTopologyDef.getComponents(), override); } - //bolt overrides + // bolt overrides if (includeTopologyDef.getBolts() != null) { topologyDef.addAllBolts(includeTopologyDef.getBolts(), override); } - //spout overrides + // spout overrides if (includeTopologyDef.getSpouts() != null) { topologyDef.addAllSpouts(includeTopologyDef.getSpouts(), override); } - //stream overrides - //TODO: streams should be uniquely identifiable + // stream overrides + // TODO: streams should be uniquely identifiable if (includeTopologyDef.getStreams() != null) { topologyDef.addAllStreams(includeTopologyDef.getStreams(), override); } diff --git a/flux/flux-core/src/test/java/org/apache/storm/flux/FluxBuilderTest.java b/flux/flux-core/src/test/java/org/apache/storm/flux/FluxBuilderTest.java index f3b82dc2abc..afbcd6334cd 100644 --- a/flux/flux-core/src/test/java/org/apache/storm/flux/FluxBuilderTest.java +++ b/flux/flux-core/src/test/java/org/apache/storm/flux/FluxBuilderTest.java @@ -17,16 +17,16 @@ */ package org.apache.storm.flux; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + import java.util.HashMap; import java.util.Map; - import org.apache.storm.Config; import org.apache.storm.flux.model.TopologyDef; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; public class FluxBuilderTest { diff --git a/flux/flux-core/src/test/java/org/apache/storm/flux/IntegrationTest.java b/flux/flux-core/src/test/java/org/apache/storm/flux/IntegrationTest.java index 4b6133a6fb2..fc50c7d1c60 100644 --- a/flux/flux-core/src/test/java/org/apache/storm/flux/IntegrationTest.java +++ b/flux/flux-core/src/test/java/org/apache/storm/flux/IntegrationTest.java @@ -25,15 +25,16 @@ public class IntegrationTest { static { String skipStr = System.getProperty("skipIntegration"); - if(skipStr != null && skipStr.equalsIgnoreCase("false")){ + if (skipStr != null && skipStr.equalsIgnoreCase("false")) { skipTest = false; } } @Test public void testRunTopologySource() throws Exception { - if(!skipTest) { - Flux.main(new String[]{"-s", "30000", "src/test/resources/configs/existing-topology.yaml"}); + if (!skipTest) { + Flux.main(new String[]{"-s", "30000", + "src/test/resources/configs/existing-topology.yaml"}); } } } diff --git a/flux/flux-core/src/test/java/org/apache/storm/flux/TCKTest.java b/flux/flux-core/src/test/java/org/apache/storm/flux/TCKTest.java index 2135046648b..6fbe3ee4ab3 100644 --- a/flux/flux-core/src/test/java/org/apache/storm/flux/TCKTest.java +++ b/flux/flux-core/src/test/java/org/apache/storm/flux/TCKTest.java @@ -17,27 +17,34 @@ */ package org.apache.storm.flux; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.Collections; +import java.util.Map; +import java.util.Properties; import org.apache.storm.Config; -import org.apache.storm.generated.StormTopology; import org.apache.storm.flux.model.ExecutionContext; import org.apache.storm.flux.model.TopologyDef; import org.apache.storm.flux.parser.FluxParser; import org.apache.storm.flux.test.TestBolt; +import org.apache.storm.generated.StormTopology; import org.apache.storm.shade.net.minidev.json.JSONValue; import org.apache.storm.utils.Utils; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.*; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.MatcherAssert.assertThat; -import java.util.Collections; -import java.util.Map; -import java.util.Properties; public class TCKTest { @Test public void testTCK() throws Exception { - TopologyDef topologyDef = FluxParser.parseResource("/configs/tck.yaml", false, true, null, false); + TopologyDef topologyDef = FluxParser.parseResource("/configs/tck.yaml", false, true, null, + false); Config conf = FluxBuilder.buildConfig(topologyDef); ExecutionContext context = new ExecutionContext(topologyDef, conf); StormTopology topology = FluxBuilder.buildTopology(context); @@ -47,7 +54,8 @@ public void testTCK() throws Exception { @Test public void testShellComponents() throws Exception { - TopologyDef topologyDef = FluxParser.parseResource("/configs/shell_test.yaml", false, true, null, false); + TopologyDef topologyDef = FluxParser.parseResource("/configs/shell_test.yaml", false, true, + null, false); Config conf = FluxBuilder.buildConfig(topologyDef); ExecutionContext context = new ExecutionContext(topologyDef, conf); StormTopology topology = FluxBuilder.buildTopology(context); @@ -57,17 +65,20 @@ public void testShellComponents() throws Exception { @Test public void testBadShellComponents() throws Exception { - TopologyDef topologyDef = FluxParser.parseResource("/configs/bad_shell_test.yaml", false, true, null, false); + TopologyDef topologyDef = FluxParser.parseResource("/configs/bad_shell_test.yaml", false, + true, null, false); Config conf = FluxBuilder.buildConfig(topologyDef); ExecutionContext context = new ExecutionContext(topologyDef, conf); - IllegalArgumentException expectedException = assertThrows(IllegalArgumentException.class, () -> FluxBuilder.buildTopology(context)); + IllegalArgumentException expectedException = assertThrows(IllegalArgumentException.class, + () -> FluxBuilder.buildTopology(context)); assertTrue(expectedException.getMessage().contains("Unable to find configuration method")); } @Test public void testKafkaSpoutConfig() throws Exception { - TopologyDef topologyDef = FluxParser.parseResource("/configs/kafka_test.yaml", false, true, null, false); + TopologyDef topologyDef = FluxParser.parseResource("/configs/kafka_test.yaml", false, true, + null, false); Config conf = FluxBuilder.buildConfig(topologyDef); ExecutionContext context = new ExecutionContext(topologyDef, conf); StormTopology topology = FluxBuilder.buildTopology(context); @@ -77,7 +88,8 @@ public void testKafkaSpoutConfig() throws Exception { @Test public void testLoadFromResource() throws Exception { - TopologyDef topologyDef = FluxParser.parseResource("/configs/kafka_test.yaml", false, true, null, false); + TopologyDef topologyDef = FluxParser.parseResource("/configs/kafka_test.yaml", false, true, + null, false); Config conf = FluxBuilder.buildConfig(topologyDef); ExecutionContext context = new ExecutionContext(topologyDef, conf); StormTopology topology = FluxBuilder.buildTopology(context); @@ -85,10 +97,10 @@ public void testLoadFromResource() throws Exception { topology.validate(); } - @Test public void testHdfs() throws Exception { - TopologyDef topologyDef = FluxParser.parseResource("/configs/hdfs_test.yaml", false, true, null, false); + TopologyDef topologyDef = FluxParser.parseResource("/configs/hdfs_test.yaml", false, true, + null, false); Config conf = FluxBuilder.buildConfig(topologyDef); ExecutionContext context = new ExecutionContext(topologyDef, conf); StormTopology topology = FluxBuilder.buildTopology(context); @@ -98,7 +110,8 @@ public void testHdfs() throws Exception { @Test public void testDiamondTopology() throws Exception { - TopologyDef topologyDef = FluxParser.parseResource("/configs/diamond-topology.yaml", false, true, null, false); + TopologyDef topologyDef = FluxParser.parseResource("/configs/diamond-topology.yaml", false, + true, null, false); Config conf = FluxBuilder.buildConfig(topologyDef); ExecutionContext context = new ExecutionContext(topologyDef, conf); StormTopology topology = FluxBuilder.buildTopology(context); @@ -127,7 +140,8 @@ public void testComponentConfig() throws Exception { @Test public void testComponentConfigOverridesTopologyConfig() throws Exception { - TopologyDef topologyDef = FluxParser.parseResource("/configs/component-config-override-test.yaml", + TopologyDef topologyDef = FluxParser + .parseResource("/configs/component-config-override-test.yaml", false, true, null, false); Config conf = FluxBuilder.buildConfig(topologyDef); ExecutionContext context = new ExecutionContext(topologyDef, conf); @@ -143,17 +157,20 @@ public void testComponentConfigOverridesTopologyConfig() throws Exception { Map boltComponentConf = (Map) JSONValue.parse( topology.get_bolts().get("bolt-1").get_common().get_json_conf()); Map effectiveBoltConf = Utils.merge(conf, boltComponentConf); - assertEquals(Boolean.FALSE, effectiveBoltConf.get(Config.TOPOLOGY_TUPLE_COMPRESSION_ENABLE)); + assertEquals(Boolean.FALSE, effectiveBoltConf + .get(Config.TOPOLOGY_TUPLE_COMPRESSION_ENABLE)); Map spoutComponentConf = (Map) JSONValue.parse( topology.get_spouts().get("spout-1").get_common().get_json_conf()); Map effectiveSpoutConf = Utils.merge(conf, spoutComponentConf); - assertEquals(Boolean.TRUE, effectiveSpoutConf.get(Config.TOPOLOGY_TUPLE_COMPRESSION_ENABLE)); + assertEquals(Boolean.TRUE, effectiveSpoutConf + .get(Config.TOPOLOGY_TUPLE_COMPRESSION_ENABLE)); } @Test public void testComponentConfigWithInvalidValue() throws Exception { - TopologyDef topologyDef = FluxParser.parseResource("/configs/component-config-invalid-test.yaml", false, + TopologyDef topologyDef = FluxParser + .parseResource("/configs/component-config-invalid-test.yaml", false, true, null, false); Config conf = FluxBuilder.buildConfig(topologyDef); ExecutionContext context = new ExecutionContext(topologyDef, conf); @@ -166,7 +183,8 @@ public void testComponentConfigWithInvalidValue() throws Exception { @Test public void testComponentConfigWithNotRegisteredKey() throws Exception { - TopologyDef topologyDef = FluxParser.parseResource("/configs/component-config-invalid-key-test.yaml", false, + TopologyDef topologyDef = FluxParser + .parseResource("/configs/component-config-invalid-key-test.yaml", false, true, null, false); Config conf = FluxBuilder.buildConfig(topologyDef); ExecutionContext context = new ExecutionContext(topologyDef, conf); @@ -180,7 +198,8 @@ public void testComponentConfigWithNotRegisteredKey() throws Exception { @Test public void testComponentConfigMissing() throws Exception { - TopologyDef topologyDef = FluxParser.parseResource("/configs/component-config-missing-test.yaml", false, + TopologyDef topologyDef = FluxParser + .parseResource("/configs/component-config-missing-test.yaml", false, true, null, false); Config conf = FluxBuilder.buildConfig(topologyDef); ExecutionContext context = new ExecutionContext(topologyDef, conf); @@ -190,26 +209,31 @@ public void testComponentConfigMissing() throws Exception { Map spoutConf = (Map) JSONValue.parse( topology.get_spouts().get("spout-1").get_common().get_json_conf()); - assertTrue(spoutConf == null || !spoutConf.containsKey(Config.TOPOLOGY_TUPLE_COMPRESSION_ENABLE)); + assertTrue(spoutConf == null || !spoutConf + .containsKey(Config.TOPOLOGY_TUPLE_COMPRESSION_ENABLE)); Map boltConf = (Map) JSONValue.parse( topology.get_bolts().get("bolt-1").get_common().get_json_conf()); - assertTrue(boltConf == null || !boltConf.containsKey(Config.TOPOLOGY_TUPLE_COMPRESSION_ENABLE)); + assertTrue(boltConf == null || !boltConf + .containsKey(Config.TOPOLOGY_TUPLE_COMPRESSION_ENABLE)); } @Test public void testBadHbase() throws Exception { - TopologyDef topologyDef = FluxParser.parseResource("/configs/bad_hbase.yaml", false, true, null, false); + TopologyDef topologyDef = FluxParser.parseResource("/configs/bad_hbase.yaml", false, true, + null, false); Config conf = FluxBuilder.buildConfig(topologyDef); ExecutionContext context = new ExecutionContext(topologyDef, conf); - IllegalArgumentException expectedException = assertThrows(IllegalArgumentException.class, () -> FluxBuilder.buildTopology(context)); + IllegalArgumentException expectedException = assertThrows(IllegalArgumentException.class, + () -> FluxBuilder.buildTopology(context)); assertTrue(expectedException.getMessage().contains("Couldn't find a suitable constructor")); } @Test public void testIncludes() throws Exception { - TopologyDef topologyDef = FluxParser.parseResource("/configs/include_test.yaml", false, true, null, false); + TopologyDef topologyDef = FluxParser.parseResource("/configs/include_test.yaml", false, + true, null, false); Config conf = FluxBuilder.buildConfig(topologyDef); ExecutionContext context = new ExecutionContext(topologyDef, conf); StormTopology topology = FluxBuilder.buildTopology(context); @@ -222,7 +246,8 @@ public void testIncludes() throws Exception { @Test public void testTopologySource() throws Exception { - TopologyDef topologyDef = FluxParser.parseResource("/configs/existing-topology.yaml", false, true, null, false); + TopologyDef topologyDef = FluxParser.parseResource("/configs/existing-topology.yaml", false, + true, null, false); assertTrue(topologyDef.validate()); Config conf = FluxBuilder.buildConfig(topologyDef); ExecutionContext context = new ExecutionContext(topologyDef, conf); @@ -233,7 +258,9 @@ public void testTopologySource() throws Exception { @Test public void testTopologySourceWithReflection() throws Exception { - TopologyDef topologyDef = FluxParser.parseResource("/configs/existing-topology-reflection.yaml", false, true, null, false); + TopologyDef topologyDef = FluxParser + .parseResource("/configs/existing-topology-reflection.yaml", false, true, null, + false); assertTrue(topologyDef.validate()); Config conf = FluxBuilder.buildConfig(topologyDef); ExecutionContext context = new ExecutionContext(topologyDef, conf); @@ -244,7 +271,9 @@ public void testTopologySourceWithReflection() throws Exception { @Test public void testTopologySourceWithConfigParam() throws Exception { - TopologyDef topologyDef = FluxParser.parseResource("/configs/existing-topology-reflection-config.yaml", false, true, null, false); + TopologyDef topologyDef = FluxParser + .parseResource("/configs/existing-topology-reflection-config.yaml", false, true, + null, false); assertTrue(topologyDef.validate()); Config conf = FluxBuilder.buildConfig(topologyDef); ExecutionContext context = new ExecutionContext(topologyDef, conf); @@ -255,7 +284,9 @@ public void testTopologySourceWithConfigParam() throws Exception { @Test public void testTopologySourceWithMethodName() throws Exception { - TopologyDef topologyDef = FluxParser.parseResource("/configs/existing-topology-method-override.yaml", false, true, null, false); + TopologyDef topologyDef = FluxParser + .parseResource("/configs/existing-topology-method-override.yaml", false, true, null, + false); assertTrue(topologyDef.validate()); Config conf = FluxBuilder.buildConfig(topologyDef); ExecutionContext context = new ExecutionContext(topologyDef, conf); @@ -264,10 +295,10 @@ public void testTopologySourceWithMethodName() throws Exception { topology.validate(); } - @Test public void testTridentTopologySource() throws Exception { - TopologyDef topologyDef = FluxParser.parseResource("/configs/existing-topology-trident.yaml", false, true, null, false); + TopologyDef topologyDef = FluxParser + .parseResource("/configs/existing-topology-trident.yaml", false, true, null, false); assertTrue(topologyDef.validate()); Config conf = FluxBuilder.buildConfig(topologyDef); ExecutionContext context = new ExecutionContext(topologyDef, conf); @@ -277,17 +308,19 @@ public void testTridentTopologySource() throws Exception { } public void testInvalidTopologySource() throws Exception { - TopologyDef topologyDef = FluxParser.parseResource("/configs/invalid-existing-topology.yaml", false, true, null, false); + TopologyDef topologyDef = FluxParser + .parseResource("/configs/invalid-existing-topology.yaml", false, true, null, false); assertFalse(topologyDef.validate(), "Topology config is invalid."); Config conf = FluxBuilder.buildConfig(topologyDef); ExecutionContext context = new ExecutionContext(topologyDef, conf); assertThrows(IllegalArgumentException.class, () -> FluxBuilder.buildTopology(context)); } - @Test public void testTopologySourceWithGetMethodName() throws Exception { - TopologyDef topologyDef = FluxParser.parseResource("/configs/existing-topology-reflection.yaml", false, true, null, false); + TopologyDef topologyDef = FluxParser + .parseResource("/configs/existing-topology-reflection.yaml", false, true, null, + false); assertTrue(topologyDef.validate()); Config conf = FluxBuilder.buildConfig(topologyDef); ExecutionContext context = new ExecutionContext(topologyDef, conf); @@ -298,7 +331,8 @@ public void testTopologySourceWithGetMethodName() throws Exception { @Test public void testTopologySourceWithConfigMethods() throws Exception { - TopologyDef topologyDef = FluxParser.parseResource("/configs/config-methods-test.yaml", false, true, null, false); + TopologyDef topologyDef = FluxParser.parseResource("/configs/config-methods-test.yaml", + false, true, null, false); assertTrue(topologyDef.validate()); Config conf = FluxBuilder.buildConfig(topologyDef); ExecutionContext context = new ExecutionContext(topologyDef, conf); @@ -307,7 +341,7 @@ public void testTopologySourceWithConfigMethods() throws Exception { topology.validate(); // make sure the property was actually set - TestBolt bolt = (TestBolt)context.getBolt("bolt-1"); + TestBolt bolt = (TestBolt) context.getBolt("bolt-1"); assertTrue(bolt.getFoo().equals("foo")); assertTrue(bolt.getBar().equals("bar")); assertTrue(bolt.getFooBar().equals("foobar")); @@ -315,13 +349,15 @@ public void testTopologySourceWithConfigMethods() throws Exception { assertNotNull(context.getBolt("bolt-2")); assertNotNull(context.getBolt("bolt-3")); assertNotNull(context.getBolt("bolt-4")); - assertArrayEquals(new TestBolt.TestClass[] {new TestBolt.TestClass("foo"), new TestBolt.TestClass("bar"), new TestBolt.TestClass("baz")}, bolt.getClasses()); + assertArrayEquals(new TestBolt.TestClass[] {new TestBolt.TestClass("foo"), new TestBolt + .TestClass("bar"), new TestBolt.TestClass("baz")}, bolt.getClasses()); } @Test public void testVariableSubstitution() throws Exception { Properties properties = FluxParser.parseProperties("/configs/test.properties", true); - TopologyDef topologyDef = FluxParser.parseResource("/configs/substitution-test.yaml", false, true, properties, true); + TopologyDef topologyDef = FluxParser.parseResource("/configs/substitution-test.yaml", false, + true, properties, true); assertTrue(topologyDef.validate()); Config conf = FluxBuilder.buildConfig(topologyDef); ExecutionContext context = new ExecutionContext(topologyDef, conf); @@ -337,14 +373,15 @@ public void testVariableSubstitution() throws Exception { // $PATH should be defined on most systems String envPath = System.getenv().get("PATH"); assertEquals(envPath, - context.getTopologyDef().getConfig().get("test.env.value"), "ENV variable not replaced."); + context.getTopologyDef().getConfig() + .get("test.env.value"), "ENV variable not replaced."); - //Test substitution where the target type is List + // Test substitution where the target type is List assertThat("List property is not replaced by the expected value", Collections.singletonList("A string list"), is(context.getTopologyDef().getConfig().get("list.property.target"))); - //Test substitution where the target type is a List element + // Test substitution where the target type is a List element assertThat("List element property is not replaced by the expected value", "A string list", is(context.getTopologyDef().getConfig().get("list.element.property.target"))); @@ -353,18 +390,22 @@ public void testVariableSubstitution() throws Exception { @Test public void testTopologyWithInvalidStaticFactoryArgument() throws Exception { - //STORM-3087. - TopologyDef topologyDef = FluxParser.parseResource("/configs/bad_static_factory_test.yaml", false, true, null, false); + // STORM-3087. + TopologyDef topologyDef = FluxParser.parseResource("/configs/bad_static_factory_test.yaml", + false, true, null, false); Config conf = FluxBuilder.buildConfig(topologyDef); ExecutionContext context = new ExecutionContext(topologyDef, conf); - IllegalArgumentException expectedException = assertThrows(IllegalArgumentException.class, () -> FluxBuilder.buildTopology(context)); - assertTrue(expectedException.getMessage().contains("Couldn't find a suitable static method")); + IllegalArgumentException expectedException = assertThrows(IllegalArgumentException.class, + () -> FluxBuilder.buildTopology(context)); + assertTrue(expectedException.getMessage() + .contains("Couldn't find a suitable static method")); } @Test public void testTopologyWithWorkerHook() throws Exception { - TopologyDef topologyDef = FluxParser.parseResource("/configs/worker_hook.yaml", false, true, null, false); + TopologyDef topologyDef = FluxParser.parseResource("/configs/worker_hook.yaml", false, true, + null, false); Config conf = FluxBuilder.buildConfig(topologyDef); ExecutionContext context = new ExecutionContext(topologyDef, conf); StormTopology topology = FluxBuilder.buildTopology(context); diff --git a/flux/flux-core/src/test/java/org/apache/storm/flux/multilang/MultilangEnvironmentTest.java b/flux/flux-core/src/test/java/org/apache/storm/flux/multilang/MultilangEnvironmentTest.java index b29285b4aee..fc7d61aafc2 100644 --- a/flux/flux-core/src/test/java/org/apache/storm/flux/multilang/MultilangEnvironmentTest.java +++ b/flux/flux-core/src/test/java/org/apache/storm/flux/multilang/MultilangEnvironmentTest.java @@ -17,16 +17,14 @@ */ package org.apache.storm.flux.multilang; - -import org.junit.jupiter.api.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import static org.junit.jupiter.api.Assertions.assertEquals; import java.io.ByteArrayOutputStream; import java.io.InputStream; import java.io.OutputStream; - -import static org.junit.jupiter.api.Assertions.assertEquals; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Sanity checks to make sure we can at least invoke the shells used. diff --git a/flux/flux-core/src/test/java/org/apache/storm/flux/test/SimpleTopology.java b/flux/flux-core/src/test/java/org/apache/storm/flux/test/SimpleTopology.java index ff65a8a1c9b..955af45605c 100644 --- a/flux/flux-core/src/test/java/org/apache/storm/flux/test/SimpleTopology.java +++ b/flux/flux-core/src/test/java/org/apache/storm/flux/test/SimpleTopology.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

    Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -17,29 +17,26 @@ */ package org.apache.storm.flux.test; -import org.apache.storm.generated.StormTopology; -import org.apache.storm.topology.TopologyBuilder; +import java.util.Map; import org.apache.storm.flux.wrappers.bolts.LogInfoBolt; import org.apache.storm.flux.wrappers.spouts.FluxShellSpout; - -import java.util.Map; +import org.apache.storm.generated.StormTopology; +import org.apache.storm.topology.TopologyBuilder; /** * Test topology source that does not implement TopologySource, but has the same * `getTopology()` method. */ -public class SimpleTopology{ +public class SimpleTopology { + public SimpleTopology() {} - public SimpleTopology(){} + public SimpleTopology(String foo, String bar) {} - public SimpleTopology(String foo, String bar){} - - public StormTopology getTopologyWithDifferentMethodName(Map config){ + public StormTopology getTopologyWithDifferentMethodName(Map config) { return getTopology(config); } - public StormTopology getTopology(Map config) { TopologyBuilder builder = new TopologyBuilder(); diff --git a/flux/flux-core/src/test/java/org/apache/storm/flux/test/SimpleTopologySource.java b/flux/flux-core/src/test/java/org/apache/storm/flux/test/SimpleTopologySource.java index 2fadacff7d2..2379aebf37b 100644 --- a/flux/flux-core/src/test/java/org/apache/storm/flux/test/SimpleTopologySource.java +++ b/flux/flux-core/src/test/java/org/apache/storm/flux/test/SimpleTopologySource.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

    Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -17,21 +17,18 @@ */ package org.apache.storm.flux.test; -import org.apache.storm.generated.StormTopology; -import org.apache.storm.topology.TopologyBuilder; +import java.util.Map; import org.apache.storm.flux.api.TopologySource; import org.apache.storm.flux.wrappers.bolts.LogInfoBolt; import org.apache.storm.flux.wrappers.spouts.FluxShellSpout; - -import java.util.Map; +import org.apache.storm.generated.StormTopology; +import org.apache.storm.topology.TopologyBuilder; public class SimpleTopologySource implements TopologySource { + public SimpleTopologySource() {} - public SimpleTopologySource(){} - - public SimpleTopologySource(String foo, String bar){} - + public SimpleTopologySource(String foo, String bar) {} @Override public StormTopology getTopology(Map config) { diff --git a/flux/flux-core/src/test/java/org/apache/storm/flux/test/SimpleTopologyWithConfigParam.java b/flux/flux-core/src/test/java/org/apache/storm/flux/test/SimpleTopologyWithConfigParam.java index 78195b517c9..e7c3a798736 100644 --- a/flux/flux-core/src/test/java/org/apache/storm/flux/test/SimpleTopologyWithConfigParam.java +++ b/flux/flux-core/src/test/java/org/apache/storm/flux/test/SimpleTopologyWithConfigParam.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

    Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -18,10 +18,10 @@ package org.apache.storm.flux.test; import org.apache.storm.Config; -import org.apache.storm.generated.StormTopology; -import org.apache.storm.topology.TopologyBuilder; import org.apache.storm.flux.wrappers.bolts.LogInfoBolt; import org.apache.storm.flux.wrappers.spouts.FluxShellSpout; +import org.apache.storm.generated.StormTopology; +import org.apache.storm.topology.TopologyBuilder; /** * Test topology source that does not implement TopologySource, but has the same @@ -29,11 +29,9 @@ */ public class SimpleTopologyWithConfigParam { + public SimpleTopologyWithConfigParam() {} - public SimpleTopologyWithConfigParam(){} - - public SimpleTopologyWithConfigParam(String foo, String bar){} - + public SimpleTopologyWithConfigParam(String foo, String bar) {} public StormTopology getTopology(Config config) { TopologyBuilder builder = new TopologyBuilder(); diff --git a/flux/flux-core/src/test/java/org/apache/storm/flux/test/TestBolt.java b/flux/flux-core/src/test/java/org/apache/storm/flux/test/TestBolt.java index 45f0a18d9a5..5d65440c345 100644 --- a/flux/flux-core/src/test/java/org/apache/storm/flux/test/TestBolt.java +++ b/flux/flux-core/src/test/java/org/apache/storm/flux/test/TestBolt.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

    Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -17,6 +17,8 @@ */ package org.apache.storm.flux.test; +import java.io.Serializable; +import java.time.Duration; import org.apache.storm.topology.BasicOutputCollector; import org.apache.storm.topology.OutputFieldsDeclarer; import org.apache.storm.topology.base.BaseBasicBolt; @@ -24,10 +26,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.Serializable; -import java.time.Duration; - - public class TestBolt extends BaseBasicBolt { private static final Logger LOG = LoggerFactory.getLogger(TestBolt.class); @@ -50,12 +48,17 @@ public String getField() { @Override public boolean equals(Object o) { - if (this == o) return true; - if (!(o instanceof TestClass)) return false; + if (this == o) { + return true; + } + if (!(o instanceof TestClass)) { + return false; + } TestClass testClass = (TestClass) o; - return getField() != null ? getField().equals(testClass.getField()) : testClass.getField() == null; + return getField() != null ? getField().equals(testClass.getField()) : testClass + .getField() == null; } @Override @@ -64,21 +67,20 @@ public int hashCode() { } } - public static enum TestEnum { FOO, BAR } - public TestBolt(TestEnum te){ + public TestBolt(TestEnum te) { } - public TestBolt(TestEnum te, float f){ + public TestBolt(TestEnum te, float f) { } - public TestBolt(TestEnum te, float f, boolean b){ + public TestBolt(TestEnum te, float f, boolean b) { } @@ -86,7 +88,7 @@ public TestBolt(TestEnum te, float f, boolean b, TestClass... str) { } - public TestBolt(Long l){} + public TestBolt(Long l) {} @Override public void execute(Tuple tuple, BasicOutputCollector basicOutputCollector) { @@ -99,40 +101,51 @@ public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) { } // config methods - public void withFoo(String foo){ + public void withFoo(String foo) { this.foo = foo; } - public void withNone(){ + + public void withNone() { this.none = "hit"; } - public void withBar(String bar){ + + public void withBar(String bar) { this.bar = bar; } - public void withFooBar(String foo, String bar){ + public void withFooBar(String foo, String bar) { this.fooBar = foo + bar; } - public void withClasses(TestClass...classes) { + public void withClasses(TestClass... classes) { this.classes = classes; } - public void setTimeLen(Duration x) { x.toMillis(); } - public void setTimeLenArr(Duration[] x) { x.toString(); } + public void setTimeLen(Duration x) { + x.toMillis(); + } - public void withDuration(Duration x) { x.toMillis(); } + public void setTimeLenArr(Duration[] x) { + x.toString(); + } - public void withDurationArr(Duration[] x) { x.toString(); } + public void withDuration(Duration x) { + x.toMillis(); + } + public void withDurationArr(Duration[] x) { + x.toString(); + } - public String getFoo(){ + public String getFoo() { return this.foo; } - public String getBar(){ + + public String getBar() { return this.bar; } - public String getFooBar(){ + public String getFooBar() { return this.fooBar; } @@ -145,7 +158,7 @@ public static TestBolt newInstance() { return newInstance(TestEnum.FOO); } - public static TestBolt newInstance(TestEnum te){ + public static TestBolt newInstance(TestEnum te) { return new TestBolt(te); } diff --git a/flux/flux-core/src/test/java/org/apache/storm/flux/test/TridentTopologySource.java b/flux/flux-core/src/test/java/org/apache/storm/flux/test/TridentTopologySource.java index b39d771a164..41956e81799 100644 --- a/flux/flux-core/src/test/java/org/apache/storm/flux/test/TridentTopologySource.java +++ b/flux/flux-core/src/test/java/org/apache/storm/flux/test/TridentTopologySource.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

    Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -19,8 +19,6 @@ import org.apache.storm.Config; import org.apache.storm.generated.StormTopology; -import org.apache.storm.tuple.Fields; -import org.apache.storm.tuple.Values; import org.apache.storm.trident.TridentTopology; import org.apache.storm.trident.operation.BaseFunction; import org.apache.storm.trident.operation.TridentCollector; @@ -28,6 +26,8 @@ import org.apache.storm.trident.testing.FixedBatchSpout; import org.apache.storm.trident.testing.MemoryMapState; import org.apache.storm.trident.tuple.TridentTuple; +import org.apache.storm.tuple.Fields; +import org.apache.storm.tuple.Values; /** * Basic Trident example that will return a `StormTopology` from a `getTopology()` method. diff --git a/flux/flux-examples/pom.xml b/flux/flux-examples/pom.xml index 6cfaa7e3f60..bc3746189a7 100644 --- a/flux/flux-examples/pom.xml +++ b/flux/flux-examples/pom.xml @@ -121,6 +121,16 @@ + + org.openrewrite.maven + rewrite-maven-plugin + + + + org.codehaus.gmavenplus + gmavenplus-plugin + + org.apache.maven.plugins maven-checkstyle-plugin diff --git a/flux/flux-examples/src/main/java/org/apache/storm/flux/examples/StatefulWordCounter.java b/flux/flux-examples/src/main/java/org/apache/storm/flux/examples/StatefulWordCounter.java index aa36af9d5f3..564f124bb2a 100644 --- a/flux/flux-examples/src/main/java/org/apache/storm/flux/examples/StatefulWordCounter.java +++ b/flux/flux-examples/src/main/java/org/apache/storm/flux/examples/StatefulWordCounter.java @@ -19,7 +19,6 @@ package org.apache.storm.flux.examples; import java.util.Map; - import org.apache.storm.state.KeyValueState; import org.apache.storm.task.OutputCollector; import org.apache.storm.task.TopologyContext; @@ -35,7 +34,8 @@ public class StatefulWordCounter extends BaseStatefulBolt topoConf, TopologyContext context, OutputCollector collector) { + public void prepare(Map topoConf, TopologyContext context, + OutputCollector collector) { this.collector = collector; } diff --git a/flux/flux-examples/src/main/java/org/apache/storm/flux/examples/TestPrintBolt.java b/flux/flux-examples/src/main/java/org/apache/storm/flux/examples/TestPrintBolt.java index 2d28c7f2ce6..138cd8ecf44 100644 --- a/flux/flux-examples/src/main/java/org/apache/storm/flux/examples/TestPrintBolt.java +++ b/flux/flux-examples/src/main/java/org/apache/storm/flux/examples/TestPrintBolt.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

    Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and diff --git a/flux/flux-examples/src/main/java/org/apache/storm/flux/examples/TestWindowBolt.java b/flux/flux-examples/src/main/java/org/apache/storm/flux/examples/TestWindowBolt.java index 57826b547e5..d9dadeca44d 100644 --- a/flux/flux-examples/src/main/java/org/apache/storm/flux/examples/TestWindowBolt.java +++ b/flux/flux-examples/src/main/java/org/apache/storm/flux/examples/TestWindowBolt.java @@ -19,7 +19,6 @@ package org.apache.storm.flux.examples; import java.util.Map; - import org.apache.storm.task.OutputCollector; import org.apache.storm.task.TopologyContext; import org.apache.storm.topology.OutputFieldsDeclarer; @@ -32,7 +31,8 @@ public class TestWindowBolt extends BaseWindowedBolt { private OutputCollector collector; @Override - public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) { + public void prepare(Map topoConf, TopologyContext context, + OutputCollector collector) { this.collector = collector; } diff --git a/flux/flux-examples/src/main/java/org/apache/storm/flux/examples/WordCountClient.java b/flux/flux-examples/src/main/java/org/apache/storm/flux/examples/WordCountClient.java index b216eb5045f..094c4fa3c4a 100644 --- a/flux/flux-examples/src/main/java/org/apache/storm/flux/examples/WordCountClient.java +++ b/flux/flux-examples/src/main/java/org/apache/storm/flux/examples/WordCountClient.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

    Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -20,7 +20,6 @@ import java.io.FileInputStream; import java.util.Properties; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.TableName; @@ -45,6 +44,7 @@ public class WordCountClient { /** * Entry point for WordCountClient. + * * @param args command line arguments * @throws Exception if an unexpected error occurs */ @@ -63,7 +63,8 @@ public static void main(String[] args) throws Exception { System.exit(1); } - Table table = ConnectionFactory.createConnection(config).getTable(TableName.valueOf("WordCount")); + Table table = ConnectionFactory.createConnection(config).getTable(TableName + .valueOf("WordCount")); String[] words = new String[] {"nathan", "mike", "jackson", "golda", "bertels"}; for (String word : words) { diff --git a/flux/flux-examples/src/main/java/org/apache/storm/flux/examples/WordCounter.java b/flux/flux-examples/src/main/java/org/apache/storm/flux/examples/WordCounter.java index 1604b791950..1b802a820b3 100644 --- a/flux/flux-examples/src/main/java/org/apache/storm/flux/examples/WordCounter.java +++ b/flux/flux-examples/src/main/java/org/apache/storm/flux/examples/WordCounter.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

    Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -21,7 +21,6 @@ import static org.apache.storm.utils.Utils.tuple; import java.util.Map; - import org.apache.storm.task.TopologyContext; import org.apache.storm.topology.BasicOutputCollector; import org.apache.storm.topology.OutputFieldsDeclarer; @@ -41,8 +40,6 @@ public class WordCounter extends BaseBasicBolt { private static final Logger LOG = LoggerFactory.getLogger(WordCounter.class); - - @Override public void prepare(Map topoConf, TopologyContext context) { } diff --git a/flux/flux-wrappers/pom.xml b/flux/flux-wrappers/pom.xml index 0b69c56727d..635daa7884e 100644 --- a/flux/flux-wrappers/pom.xml +++ b/flux/flux-wrappers/pom.xml @@ -52,6 +52,16 @@ + + org.openrewrite.maven + rewrite-maven-plugin + + + + org.codehaus.gmavenplus + gmavenplus-plugin + + org.apache.maven.plugins maven-checkstyle-plugin diff --git a/flux/flux-wrappers/src/main/java/org/apache/storm/flux/wrappers/bolts/FluxShellBolt.java b/flux/flux-wrappers/src/main/java/org/apache/storm/flux/wrappers/bolts/FluxShellBolt.java index 3881e93be78..c9b61719fe9 100644 --- a/flux/flux-wrappers/src/main/java/org/apache/storm/flux/wrappers/bolts/FluxShellBolt.java +++ b/flux/flux-wrappers/src/main/java/org/apache/storm/flux/wrappers/bolts/FluxShellBolt.java @@ -22,7 +22,6 @@ import java.util.Iterator; import java.util.List; import java.util.Map; - import org.apache.storm.task.ShellBolt; import org.apache.storm.topology.IRichBolt; import org.apache.storm.topology.OutputFieldsDeclarer; @@ -39,6 +38,7 @@ public class FluxShellBolt extends ShellBolt implements IRichBolt { /** * Create a ShellBolt with command line arguments. + * * @param command Command line arguments for the bolt */ public FluxShellBolt(String[] command) { @@ -146,6 +146,7 @@ public void setDefaultStream(String[] outputFields) { * - first * - [word, count] * ``` + * * @param name Name of stream the bolt will emit into. * @param outputFields Names of fields the bolt will emit in custom *named* stream. */ diff --git a/flux/flux-wrappers/src/main/java/org/apache/storm/flux/wrappers/spouts/FluxShellSpout.java b/flux/flux-wrappers/src/main/java/org/apache/storm/flux/wrappers/spouts/FluxShellSpout.java index caa920f5b06..1c12de24d8e 100644 --- a/flux/flux-wrappers/src/main/java/org/apache/storm/flux/wrappers/spouts/FluxShellSpout.java +++ b/flux/flux-wrappers/src/main/java/org/apache/storm/flux/wrappers/spouts/FluxShellSpout.java @@ -22,13 +22,11 @@ import java.util.Iterator; import java.util.List; import java.util.Map; - import org.apache.storm.spout.ShellSpout; import org.apache.storm.topology.IRichSpout; import org.apache.storm.topology.OutputFieldsDeclarer; import org.apache.storm.tuple.Fields; - /** * A generic `ShellSpout` implementation that allows you specify output fields * and even streams without having to subclass `ShellSpout` to do so. @@ -40,6 +38,7 @@ public class FluxShellSpout extends ShellSpout implements IRichSpout { /** * Create a ShellSpout with command line arguments. + * * @param command Command line arguments for the bolt */ public FluxShellSpout(String[] command) { @@ -62,6 +61,7 @@ public FluxShellSpout(String[] args, String[] outputFields) { /** * Add configuration for this spout. This method is called from YAML file: + * *

    * ``` * className: "org.apache.storm.flux.wrappers.bolts.FluxShellSpout" @@ -148,6 +148,7 @@ public void setDefaultStream(String[] outputFields) { * - first * - [word, count] * ``` + * * @param name Name of stream the spout will emit into. * @param outputFields Names of fields the spout will emit in custom *named* stream. */ diff --git a/flux/pom.xml b/flux/pom.xml index fc171257313..098290f12b6 100644 --- a/flux/pom.xml +++ b/flux/pom.xml @@ -60,6 +60,16 @@ + + org.openrewrite.maven + rewrite-maven-plugin + + + + org.codehaus.gmavenplus + gmavenplus-plugin + + org.apache.maven.plugins maven-checkstyle-plugin diff --git a/integration-test/pom.xml b/integration-test/pom.xml index 6878b1bae0c..b8e70be8e44 100644 --- a/integration-test/pom.xml +++ b/integration-test/pom.xml @@ -164,6 +164,16 @@ + + org.openrewrite.maven + rewrite-maven-plugin + + + + org.codehaus.gmavenplus + gmavenplus-plugin + + org.apache.maven.plugins maven-checkstyle-plugin diff --git a/integration-test/src/main/java/org/apache/storm/ExclamationTopology.java b/integration-test/src/main/java/org/apache/storm/ExclamationTopology.java index d674e8cfa5a..cbdc5ef569a 100644 --- a/integration-test/src/main/java/org/apache/storm/ExclamationTopology.java +++ b/integration-test/src/main/java/org/apache/storm/ExclamationTopology.java @@ -51,7 +51,8 @@ public static class ExclamationBolt extends BaseRichBolt { OutputCollector collector; @Override - public void prepare(Map conf, TopologyContext context, OutputCollector collector) { + public void prepare(Map conf, TopologyContext context, + OutputCollector collector) { this.collector = collector; } @@ -69,7 +70,8 @@ public void declareOutputFields(OutputFieldsDeclarer declarer) { public static class FixedOrderWordSpout extends BaseRichSpout { - public static final List WORDS = Collections.unmodifiableList(Arrays.asList("nathan", + public static final List WORDS = Collections.unmodifiableList(Arrays + .asList("nathan", "mike", "jackson", "golda", @@ -85,17 +87,18 @@ public void declareOutputFields(OutputFieldsDeclarer declarer) { } @Override - public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { + public void open(Map conf, TopologyContext context, + SpoutOutputCollector collector) { this.collector = collector; } @Override public void nextTuple() { if (numEmitted >= TestableTopology.MAX_SPOUT_EMITS) { - //Stop emitting at a certain point, because log rolling breaks the tests. + // Stop emitting at a certain point, because log rolling breaks the tests. return; } - //Sleep a bit to avoid hogging the CPU. + // Sleep a bit to avoid hogging the CPU. TimeUtil.sleepMilliSec(1); collector.emit(new Values(WORDS.get((currentIndex++) % WORDS.size()))); ++numEmitted; @@ -119,8 +122,10 @@ public static void main(String[] args) throws Exception { public static StormTopology getStormTopology() { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout(WORD, new FixedOrderWordSpout(), SPOUT_EXECUTORS); - builder.setBolt(EXCLAIM_1, new ExclamationTopology.ExclamationBolt(), 3).shuffleGrouping(WORD); - builder.setBolt(EXCLAIM_2, new ExclamationTopology.ExclamationBolt(), EXCLAIM_2_EXECUTORS).shuffleGrouping(EXCLAIM_1); + builder.setBolt(EXCLAIM_1, new ExclamationTopology.ExclamationBolt(), 3) + .shuffleGrouping(WORD); + builder.setBolt(EXCLAIM_2, new ExclamationTopology.ExclamationBolt(), EXCLAIM_2_EXECUTORS) + .shuffleGrouping(EXCLAIM_1); return builder.createTopology(); } } diff --git a/integration-test/src/main/java/org/apache/storm/debug/DebugHelper.java b/integration-test/src/main/java/org/apache/storm/debug/DebugHelper.java index 61ba9d93045..f9efc764f4f 100644 --- a/integration-test/src/main/java/org/apache/storm/debug/DebugHelper.java +++ b/integration-test/src/main/java/org/apache/storm/debug/DebugHelper.java @@ -19,7 +19,6 @@ import java.net.URL; import java.net.URLClassLoader; - import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/integration-test/src/main/java/org/apache/storm/st/topology/TestableTopology.java b/integration-test/src/main/java/org/apache/storm/st/topology/TestableTopology.java index cf10f234f7e..0f19a679e48 100644 --- a/integration-test/src/main/java/org/apache/storm/st/topology/TestableTopology.java +++ b/integration-test/src/main/java/org/apache/storm/st/topology/TestableTopology.java @@ -23,8 +23,9 @@ public interface TestableTopology { String DUMMY_FIELD = "dummy"; int TIMEDATA_SLEEP_BETWEEN_EMITS_MS = 20; - //Some tests rely on reading the worker log. If there are too many emits and too much is logged, the log might roll, breaking the test. - //Ensure the time based windowing tests can emit for 5 minutes + // Some tests rely on reading the worker log. If there are too many emits and too much is + // logged, the log might roll, breaking the test. + // Ensure the time based windowing tests can emit for 5 minutes long MAX_SPOUT_EMITS = TimeUnit.MINUTES.toMillis(5) / TIMEDATA_SLEEP_BETWEEN_EMITS_MS; StormTopology newTopology(); diff --git a/integration-test/src/main/java/org/apache/storm/st/topology/window/IncrementingSpout.java b/integration-test/src/main/java/org/apache/storm/st/topology/window/IncrementingSpout.java index f42f660aadf..1560ee30beb 100644 --- a/integration-test/src/main/java/org/apache/storm/st/topology/window/IncrementingSpout.java +++ b/integration-test/src/main/java/org/apache/storm/st/topology/window/IncrementingSpout.java @@ -44,7 +44,8 @@ public void declareOutputFields(OutputFieldsDeclarer declarer) { } @Override - public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { + public void open(Map conf, TopologyContext context, + SpoutOutputCollector collector) { componentId = context.getThisComponentId(); this.collector = collector; } @@ -52,10 +53,10 @@ public void open(Map conf, TopologyContext context, SpoutOutputC @Override public void nextTuple() { if (currentNum >= TestableTopology.MAX_SPOUT_EMITS) { - //Stop emitting at a certain point, because log rolling breaks the tests. + // Stop emitting at a certain point, because log rolling breaks the tests. return; } - //Sleep a bit to avoid hogging the CPU. + // Sleep a bit to avoid hogging the CPU. TimeUtil.sleepMilliSec(1); currentNum++; final Values tuple = new Values(currentNum); diff --git a/integration-test/src/main/java/org/apache/storm/st/topology/window/SlidingTimeCorrectness.java b/integration-test/src/main/java/org/apache/storm/st/topology/window/SlidingTimeCorrectness.java index 88fcdf4bda3..62df0467997 100644 --- a/integration-test/src/main/java/org/apache/storm/st/topology/window/SlidingTimeCorrectness.java +++ b/integration-test/src/main/java/org/apache/storm/st/topology/window/SlidingTimeCorrectness.java @@ -18,17 +18,14 @@ package org.apache.storm.st.topology.window; import com.google.common.collect.Lists; - import java.util.List; import java.util.concurrent.TimeUnit; - import org.apache.storm.generated.StormTopology; import org.apache.storm.st.topology.TestableTopology; import org.apache.storm.st.topology.window.data.TimeData; import org.apache.storm.st.utils.StringDecorator; import org.apache.storm.topology.TopologyBuilder; import org.apache.storm.topology.base.BaseWindowedBolt; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -47,7 +44,8 @@ public class SlidingTimeCorrectness implements TestableTopology { public SlidingTimeCorrectness(int windowSec, int slideSec) { this.windowSec = windowSec; this.slideSec = slideSec; - final String prefix = this.getClass().getSimpleName() + "-winSec" + windowSec + "slideSec" + slideSec; + final String prefix = this.getClass().getSimpleName() + "-winSec" + windowSec + "slideSec" + + slideSec; spoutName = prefix + "IncrementingSpout"; boltName = prefix + "VerificationBolt"; } diff --git a/integration-test/src/main/java/org/apache/storm/st/topology/window/SlidingWindowCorrectness.java b/integration-test/src/main/java/org/apache/storm/st/topology/window/SlidingWindowCorrectness.java index 5e11abef6b7..b313f804466 100644 --- a/integration-test/src/main/java/org/apache/storm/st/topology/window/SlidingWindowCorrectness.java +++ b/integration-test/src/main/java/org/apache/storm/st/topology/window/SlidingWindowCorrectness.java @@ -18,15 +18,12 @@ package org.apache.storm.st.topology.window; import com.google.common.collect.Lists; - import java.util.List; - import org.apache.storm.generated.StormTopology; import org.apache.storm.st.topology.TestableTopology; import org.apache.storm.st.utils.StringDecorator; import org.apache.storm.topology.TopologyBuilder; import org.apache.storm.topology.base.BaseWindowedBolt; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -47,7 +44,8 @@ public class SlidingWindowCorrectness implements TestableTopology { public SlidingWindowCorrectness(int windowSize, int slideSize) { this.windowSize = windowSize; this.slideSize = slideSize; - final String prefix = this.getClass().getSimpleName() + "-winSize" + windowSize + "slideSize" + slideSize; + final String prefix = this.getClass().getSimpleName() + "-winSize" + windowSize + + "slideSize" + slideSize; spoutName = prefix + "IncrementingSpout"; boltName = prefix + "VerificationBolt"; } @@ -78,7 +76,8 @@ public StormTopology newTopology() { builder.setSpout(getSpoutName(), new IncrementingSpout(), spoutExecutors); builder.setBolt(getBoltName(), new VerificationBolt() - .withWindow(new BaseWindowedBolt.Count(windowSize), new BaseWindowedBolt.Count(slideSize)), + .withWindow(new BaseWindowedBolt.Count(windowSize), new BaseWindowedBolt + .Count(slideSize)), boltExecutors) .shuffleGrouping(getSpoutName()); return builder.createTopology(); diff --git a/integration-test/src/main/java/org/apache/storm/st/topology/window/TimeDataIncrementingSpout.java b/integration-test/src/main/java/org/apache/storm/st/topology/window/TimeDataIncrementingSpout.java index 2c2d4df9bf9..3d5269538b7 100644 --- a/integration-test/src/main/java/org/apache/storm/st/topology/window/TimeDataIncrementingSpout.java +++ b/integration-test/src/main/java/org/apache/storm/st/topology/window/TimeDataIncrementingSpout.java @@ -43,7 +43,8 @@ public void declareOutputFields(OutputFieldsDeclarer declarer) { } @Override - public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { + public void open(Map conf, TopologyContext context, + SpoutOutputCollector collector) { componentId = context.getThisComponentId(); this.collector = collector; } @@ -51,10 +52,11 @@ public void open(Map conf, TopologyContext context, SpoutOutputC @Override public void nextTuple() { if (currentNum >= TestableTopology.MAX_SPOUT_EMITS) { - //Stop emitting at a certain point, because log rolling breaks the tests. + // Stop emitting at a certain point, because log rolling breaks the tests. return; } - //Sleep a bit between emits to ensure that we don't reach the cap too quickly, since this spout is used to test time based windows + // Sleep a bit between emits to ensure that we don't reach the cap too quickly, since this + // spout is used to test time based windows TimeUtil.sleepMilliSec(TestableTopology.TIMEDATA_SLEEP_BETWEEN_EMITS_MS); currentNum++; TimeData data = TimeData.newData(currentNum); diff --git a/integration-test/src/main/java/org/apache/storm/st/topology/window/TimeDataVerificationBolt.java b/integration-test/src/main/java/org/apache/storm/st/topology/window/TimeDataVerificationBolt.java index 019ecbfc23a..e0c6ddead4a 100644 --- a/integration-test/src/main/java/org/apache/storm/st/topology/window/TimeDataVerificationBolt.java +++ b/integration-test/src/main/java/org/apache/storm/st/topology/window/TimeDataVerificationBolt.java @@ -41,7 +41,8 @@ public class TimeDataVerificationBolt extends BaseWindowedBolt { private String componentId; @Override - public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) { + public void prepare(Map topoConf, TopologyContext context, + OutputCollector collector) { componentId = context.getThisComponentId(); this.collector = collector; } diff --git a/integration-test/src/main/java/org/apache/storm/st/topology/window/TumblingTimeCorrectness.java b/integration-test/src/main/java/org/apache/storm/st/topology/window/TumblingTimeCorrectness.java index 934940ffd50..b6570f032c2 100644 --- a/integration-test/src/main/java/org/apache/storm/st/topology/window/TumblingTimeCorrectness.java +++ b/integration-test/src/main/java/org/apache/storm/st/topology/window/TumblingTimeCorrectness.java @@ -18,17 +18,14 @@ package org.apache.storm.st.topology.window; import com.google.common.collect.Lists; - import java.util.List; import java.util.concurrent.TimeUnit; - import org.apache.storm.generated.StormTopology; import org.apache.storm.st.topology.TestableTopology; import org.apache.storm.st.topology.window.data.TimeData; import org.apache.storm.st.utils.StringDecorator; import org.apache.storm.topology.TopologyBuilder; import org.apache.storm.topology.base.BaseWindowedBolt; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -76,7 +73,8 @@ public StormTopology newTopology() { builder.setSpout(getSpoutName(), new TimeDataIncrementingSpout(), spoutExecutors); builder.setBolt(getBoltName(), new TimeDataVerificationBolt() - .withTumblingWindow(new BaseWindowedBolt.Duration(tumbleSec, TimeUnit.SECONDS)) + .withTumblingWindow(new BaseWindowedBolt.Duration(tumbleSec, + TimeUnit.SECONDS)) .withLag(new BaseWindowedBolt.Duration(10, TimeUnit.SECONDS)) .withTimestampField(TimeData.getTimestampFieldName()), boltExecutors) diff --git a/integration-test/src/main/java/org/apache/storm/st/topology/window/TumblingWindowCorrectness.java b/integration-test/src/main/java/org/apache/storm/st/topology/window/TumblingWindowCorrectness.java index 81d46b974d4..857da0f5da2 100644 --- a/integration-test/src/main/java/org/apache/storm/st/topology/window/TumblingWindowCorrectness.java +++ b/integration-test/src/main/java/org/apache/storm/st/topology/window/TumblingWindowCorrectness.java @@ -18,15 +18,12 @@ package org.apache.storm.st.topology.window; import com.google.common.collect.Lists; - import java.util.List; - import org.apache.storm.generated.StormTopology; import org.apache.storm.st.topology.TestableTopology; import org.apache.storm.st.utils.StringDecorator; import org.apache.storm.topology.TopologyBuilder; import org.apache.storm.topology.base.BaseWindowedBolt; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/integration-test/src/main/java/org/apache/storm/st/topology/window/VerificationBolt.java b/integration-test/src/main/java/org/apache/storm/st/topology/window/VerificationBolt.java index d3ed45f22b1..fe3a9861812 100644 --- a/integration-test/src/main/java/org/apache/storm/st/topology/window/VerificationBolt.java +++ b/integration-test/src/main/java/org/apache/storm/st/topology/window/VerificationBolt.java @@ -40,7 +40,8 @@ public class VerificationBolt extends BaseWindowedBolt { private String componentId; @Override - public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) { + public void prepare(Map topoConf, TopologyContext context, + OutputCollector collector) { componentId = context.getThisComponentId(); this.collector = collector; } diff --git a/integration-test/src/main/java/org/apache/storm/st/topology/window/data/TimeData.java b/integration-test/src/main/java/org/apache/storm/st/topology/window/data/TimeData.java index 4a1fc96ff2d..9d84af8fcae 100644 --- a/integration-test/src/main/java/org/apache/storm/st/topology/window/data/TimeData.java +++ b/integration-test/src/main/java/org/apache/storm/st/topology/window/data/TimeData.java @@ -19,10 +19,8 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; - import java.util.Collection; import java.util.Date; - import org.apache.storm.tuple.Fields; import org.apache.storm.tuple.Tuple; import org.apache.storm.tuple.Values; @@ -32,7 +30,8 @@ public class TimeData implements Comparable { private static final String NUMBER_FIELD_NAME = "number"; private static final String STRING_FIELD_NAME = "dateAsStr"; private static final String TIMESTAMP_FIELD_NAME = "date"; - static final Gson GSON = new GsonBuilder().setDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'").create(); + static final Gson GSON = new GsonBuilder().setDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'") + .create(); private final int num; private final Date now; private final long timestamp; @@ -52,7 +51,8 @@ public static TimeData newData(int num) { } public static TimeData fromTuple(Tuple tuple) { - return new TimeData(tuple.getIntegerByField(NUMBER_FIELD_NAME), new Date(tuple.getLongByField(TIMESTAMP_FIELD_NAME))); + return new TimeData(tuple.getIntegerByField(NUMBER_FIELD_NAME), new Date(tuple + .getLongByField(TIMESTAMP_FIELD_NAME))); } public static TimeData fromJson(String jsonStr) { diff --git a/integration-test/src/main/java/org/apache/storm/st/utils/StringDecorator.java b/integration-test/src/main/java/org/apache/storm/st/utils/StringDecorator.java index 8f9feeb64db..99c4371a19f 100644 --- a/integration-test/src/main/java/org/apache/storm/st/utils/StringDecorator.java +++ b/integration-test/src/main/java/org/apache/storm/st/utils/StringDecorator.java @@ -20,8 +20,10 @@ import org.apache.commons.lang3.StringUtils; /** - * This class provides a method to pass data from the test bolts and spouts to the test method, via the worker log. - * Test components can use {@link #decorate(java.lang.String, java.lang.String) } to create a string containing a + * This class provides a method to pass data from the test bolts and spouts to the test method, via + * the worker log. + * Test components can use {@link #decorate(java.lang.String, java.lang.String) } to create a string + * containing a * unique prefix. * Such prefixed log lines can be retrieved from the worker logs, and recognized via * {@link #isDecorated(java.lang.String, java.lang.String) }. diff --git a/integration-test/src/main/java/org/apache/storm/st/utils/TimeUtil.java b/integration-test/src/main/java/org/apache/storm/st/utils/TimeUtil.java index fb99e99fced..800a521cad7 100644 --- a/integration-test/src/main/java/org/apache/storm/st/utils/TimeUtil.java +++ b/integration-test/src/main/java/org/apache/storm/st/utils/TimeUtil.java @@ -20,7 +20,6 @@ import java.time.Duration; import java.time.ZonedDateTime; import java.util.concurrent.TimeUnit; - import org.apache.commons.lang3.exception.ExceptionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/integration-test/src/test/java/org/apache/storm/st/DemoTest.java b/integration-test/src/test/java/org/apache/storm/st/DemoTest.java index 4c148f71f3d..e5ada34c1b8 100644 --- a/integration-test/src/test/java/org/apache/storm/st/DemoTest.java +++ b/integration-test/src/test/java/org/apache/storm/st/DemoTest.java @@ -17,6 +17,11 @@ package org.apache.storm.st; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; import org.apache.storm.ExclamationTopology; import org.apache.storm.st.helper.AbstractTest; import org.apache.storm.st.wrapper.TopoWrap; @@ -25,15 +30,10 @@ import org.testng.annotations.AfterMethod; import org.testng.annotations.Test; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; - -import static org.junit.jupiter.api.Assertions.assertTrue; - public final class DemoTest extends AbstractTest { private static final Logger log = LoggerFactory.getLogger(DemoTest.class); - private static final List exclaim2Output = ExclamationTopology.FixedOrderWordSpout.WORDS.stream() + private static final List exclaim2Output = ExclamationTopology.FixedOrderWordSpout.WORDS + .stream() .map(word -> word + "!!!!!!") .collect(Collectors.toList()); private final String topologyName = this.getClass().getSimpleName(); @@ -45,13 +45,16 @@ public void testExclamationTopology() throws Exception { topo.submitSuccessfully(); final int minExclaim2Emits = 500; final int minSpoutEmits = 10000; - topo.assertProgress(minSpoutEmits, ExclamationTopology.SPOUT_EXECUTORS, ExclamationTopology.WORD, 180); - topo.assertProgress(minExclaim2Emits, ExclamationTopology.EXCLAIM_2_EXECUTORS, ExclamationTopology.EXCLAIM_2, 180); + topo.assertProgress(minSpoutEmits, ExclamationTopology.SPOUT_EXECUTORS, + ExclamationTopology.WORD, 180); + topo.assertProgress(minExclaim2Emits, ExclamationTopology.EXCLAIM_2_EXECUTORS, + ExclamationTopology.EXCLAIM_2, 180); Set boltUrls = topo.getLogUrls(ExclamationTopology.WORD); log.info(boltUrls.toString()); final String actualOutput = topo.getLogs(ExclamationTopology.EXCLAIM_2); for (String oneExpectedOutput : exclaim2Output) { - assertTrue(actualOutput.contains(oneExpectedOutput), "Couldn't find " + oneExpectedOutput + " in urls"); + assertTrue(actualOutput.contains(oneExpectedOutput), "Couldn't find " + + oneExpectedOutput + " in urls"); } } diff --git a/integration-test/src/test/java/org/apache/storm/st/helper/AbstractTest.java b/integration-test/src/test/java/org/apache/storm/st/helper/AbstractTest.java index 57c4930d6ab..12516bfeee9 100644 --- a/integration-test/src/test/java/org/apache/storm/st/helper/AbstractTest.java +++ b/integration-test/src/test/java/org/apache/storm/st/helper/AbstractTest.java @@ -21,6 +21,7 @@ public abstract class AbstractTest { protected final StormCluster cluster = new StormCluster(); + static { System.setProperty("user.timezone", "UTC"); } diff --git a/integration-test/src/test/java/org/apache/storm/st/meta/TestngListener.java b/integration-test/src/test/java/org/apache/storm/st/meta/TestngListener.java index cb9018cd41c..78de8aa6461 100644 --- a/integration-test/src/test/java/org/apache/storm/st/meta/TestngListener.java +++ b/integration-test/src/test/java/org/apache/storm/st/meta/TestngListener.java @@ -17,6 +17,7 @@ package org.apache.storm.st.meta; +import java.util.Arrays; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.log4j.Logger; @@ -26,8 +27,6 @@ import org.testng.ITestListener; import org.testng.ITestResult; -import java.util.Arrays; - /** * Testng listener class. This is useful for things that are applicable to all the tests as well * taking actions that depend on test results. @@ -36,20 +35,22 @@ public class TestngListener implements ITestListener, IExecutionListener { private static final Logger LOGGER = Logger.getLogger(TestngListener.class); private final String hr = StringUtils.repeat("-", 100); - private enum RunResult {SUCCESS, FAILED, SKIPPED, TestFailedButWithinSuccessPercentage } + private enum RunResult { SUCCESS, FAILED, SKIPPED, TestFailedButWithinSuccessPercentage } @Override public void onTestStart(ITestResult result) { LOGGER.info(hr); LOGGER.info( - String.format("Testing going to start for: %s.%s(%s)", result.getTestClass().getName(), + String.format("Testing going to start for: %s.%s(%s)", result.getTestClass() + .getName(), result.getName(), Arrays.toString(result.getParameters()))); NDC.push(result.getName()); } private void endOfTestHook(ITestResult result, RunResult outcome) { LOGGER.info( - String.format("Testing going to end for: %s.%s(%s) ----- Status: %s", result.getTestClass().getName(), + String.format("Testing going to end for: %s.%s(%s) ----- Status: %s", result + .getTestClass().getName(), result.getName(), Arrays.toString(result.getParameters()), outcome)); NDC.pop(); LOGGER.info(hr); diff --git a/integration-test/src/test/java/org/apache/storm/st/tests/window/SlidingWindowTest.java b/integration-test/src/test/java/org/apache/storm/st/tests/window/SlidingWindowTest.java index 11ca2ae3c69..53040b9ab57 100644 --- a/integration-test/src/test/java/org/apache/storm/st/tests/window/SlidingWindowTest.java +++ b/integration-test/src/test/java/org/apache/storm/st/tests/window/SlidingWindowTest.java @@ -17,6 +17,8 @@ package org.apache.storm.st.tests.window; +import static org.junit.jupiter.api.Assertions.assertThrows; + import org.apache.storm.st.helper.AbstractTest; import org.apache.storm.st.topology.window.SlidingTimeCorrectness; import org.apache.storm.st.topology.window.SlidingWindowCorrectness; @@ -24,7 +26,6 @@ import org.testng.annotations.AfterMethod; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; -import static org.junit.jupiter.api.Assertions.assertThrows; public final class SlidingWindowTest extends AbstractTest { private final WindowVerifier windowVerifier = new WindowVerifier(); @@ -51,8 +52,10 @@ public static Object[][] generateCountWindows() { @Test(dataProvider = "generateCountWindows") public void testWindowCount(int windowSize, int slideSize) throws Exception { - final SlidingWindowCorrectness testable = new SlidingWindowCorrectness(windowSize, slideSize); - final String topologyName = this.getClass().getSimpleName() + "-size-window" + windowSize + "-slide" + slideSize; + final SlidingWindowCorrectness testable = new SlidingWindowCorrectness(windowSize, + slideSize); + final String topologyName = this.getClass().getSimpleName() + "-size-window" + windowSize + + "-slide" + slideSize; if (windowSize <= 0 || slideSize <= 0) { assertThrows(IllegalArgumentException.class, () -> testable.newTopology()); } @@ -80,7 +83,8 @@ public static Object[][] generateTimeWindows() { @Test(dataProvider = "generateTimeWindows") public void testTimeWindow(int windowSec, int slideSec) throws Exception { final SlidingTimeCorrectness testable = new SlidingTimeCorrectness(windowSec, slideSec); - final String topologyName = this.getClass().getSimpleName() + "-sec-window" + windowSec + "-slide" + slideSec; + final String topologyName = this.getClass().getSimpleName() + "-sec-window" + windowSec + + "-slide" + slideSec; if (windowSec <= 0 || slideSec <= 0) { assertThrows(IllegalArgumentException.class, () -> testable.newTopology()); } diff --git a/integration-test/src/test/java/org/apache/storm/st/tests/window/TumblingWindowTest.java b/integration-test/src/test/java/org/apache/storm/st/tests/window/TumblingWindowTest.java index 99c8d313cf7..0db986bc752 100644 --- a/integration-test/src/test/java/org/apache/storm/st/tests/window/TumblingWindowTest.java +++ b/integration-test/src/test/java/org/apache/storm/st/tests/window/TumblingWindowTest.java @@ -17,17 +17,17 @@ package org.apache.storm.st.tests.window; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.fail; + import org.apache.storm.st.helper.AbstractTest; -import org.apache.storm.st.wrapper.TopoWrap; import org.apache.storm.st.topology.window.TumblingTimeCorrectness; import org.apache.storm.st.topology.window.TumblingWindowCorrectness; +import org.apache.storm.st.wrapper.TopoWrap; import org.testng.annotations.AfterMethod; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.fail; - public final class TumblingWindowTest extends AbstractTest { private final WindowVerifier windowVerifier = new WindowVerifier(); private TopoWrap topo; diff --git a/integration-test/src/test/java/org/apache/storm/st/tests/window/WindowVerifier.java b/integration-test/src/test/java/org/apache/storm/st/tests/window/WindowVerifier.java index fda6d44c97c..377ecccb76f 100644 --- a/integration-test/src/test/java/org/apache/storm/st/tests/window/WindowVerifier.java +++ b/integration-test/src/test/java/org/apache/storm/st/tests/window/WindowVerifier.java @@ -16,6 +16,8 @@ package org.apache.storm.st.tests.window; +import static org.junit.jupiter.api.Assertions.assertTrue; + import java.io.IOException; import java.time.ZoneOffset; import java.time.ZonedDateTime; @@ -32,48 +34,55 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.junit.jupiter.api.Assertions.assertTrue; - public class WindowVerifier { public static final Logger LOG = LoggerFactory.getLogger(WindowVerifier.class); /** - * Run the topology and verify that the number and contents of count based windows is as expected + * Run the topology and verify that the number and contents of count based windows is as + * expected * once the spout and bolt have emitted sufficient tuples. - * The spout and bolt are required to log exactly one log line per emit/window using {@link StringDecorator} + * The spout and bolt are required to log exactly one log line per emit/window using {@link + * StringDecorator} */ - public void runAndVerifyCount(int windowSize, int slideSize, TestableTopology testable, TopoWrap topo) throws IOException, TException { + public void runAndVerifyCount(int windowSize, int slideSize, TestableTopology testable, + TopoWrap topo) throws IOException, TException { topo.submitSuccessfully(); final int minBoltEmits = 5; - //Sliding windows should produce one window every slideSize tuples - //Wait for the spout to emit at least enough tuples to get minBoltEmit windows and at least one full window + // Sliding windows should produce one window every slideSize tuples + // Wait for the spout to emit at least enough tuples to get minBoltEmit windows and at least + // one full window final int minSpoutEmits = Math.max(windowSize, minBoltEmits * slideSize); String boltName = testable.getBoltName(); String spoutName = testable.getSpoutName(); - //Waiting for spout tuples isn't strictly necessary since we also wait for bolt emits, but do it anyway + // Waiting for spout tuples isn't strictly necessary since we also wait for bolt emits, but + // do it anyway topo.assertProgress(minSpoutEmits, testable.getSpoutExecutors(), spoutName, 180); topo.assertProgress(minBoltEmits, testable.getBoltExecutors(), boltName, 180); final List allDecoratedBoltLogs = topo.getDecoratedLogLines(boltName); final List allDecoratedSpoutLogs = topo.getDecoratedLogLines(spoutName); - //We expect the bolt to log exactly one decorated line per emit + // We expect the bolt to log exactly one decorated line per emit assertTrue(allDecoratedBoltLogs.size() >= minBoltEmits, - "Expecting min " + minBoltEmits + " bolt emits, found: " + allDecoratedBoltLogs.size() + " \n\t" + allDecoratedBoltLogs); + "Expecting min " + minBoltEmits + " bolt emits, found: " + allDecoratedBoltLogs + .size() + " \n\t" + allDecoratedBoltLogs); final int numberOfWindows = allDecoratedBoltLogs.size(); - for(int i = 0; i < numberOfWindows; ++i ) { + for (int i = 0; i < numberOfWindows; ++i) { LOG.info("Comparing window: " + (i + 1) + " of " + numberOfWindows); final int toIndex = (i + 1) * slideSize; final int fromIndex = toIndex - windowSize; final int positiveFromIndex = fromIndex > 0 ? fromIndex : 0; - final List expectedWindowContents = allDecoratedSpoutLogs.subList(positiveFromIndex, toIndex); + final List expectedWindowContents = allDecoratedSpoutLogs + .subList(positiveFromIndex, toIndex); final String actualString = allDecoratedBoltLogs.get(i).toString(); for (DecoratedLogLine windowData : expectedWindowContents) { final String logStr = windowData.getData(); assertTrue(actualString.contains(logStr), - () -> String.format("Missing: '%s' \nActual: '%s' \nCalculated window: '%s'", logStr, actualString, expectedWindowContents)); + () -> String + .format("Missing: '%s' \nActual: '%s' \nCalculated window: '%s'", logStr, + actualString, expectedWindowContents)); } } } @@ -81,9 +90,11 @@ public void runAndVerifyCount(int windowSize, int slideSize, TestableTopology te /** * Run the topology and verify that the number and contents of time based windows is as expected * once the spout and bolt have emitted sufficient tuples. - * The spout and bolt are required to log exactly one log line per emit/window using {@link StringDecorator} + * The spout and bolt are required to log exactly one log line per emit/window using {@link + * StringDecorator} */ - public void runAndVerifyTime(int windowSec, int slideSec, TestableTopology testable, TopoWrap topo) throws IOException, TException { + public void runAndVerifyTime(int windowSec, int slideSec, TestableTopology testable, + TopoWrap topo) throws IOException, TException { topo.submitSuccessfully(); final int minSpoutEmits = 100; final int minBoltEmits = 5; @@ -91,15 +102,22 @@ public void runAndVerifyTime(int windowSec, int slideSec, TestableTopology testa String boltName = testable.getBoltName(); String spoutName = testable.getSpoutName(); - //Waiting for spout tuples isn't strictly necessary since we also wait for bolt emits, but do it anyway - //Allow two minutes for topology startup, then wait for at most the time it should take to produce 10 windows - topo.assertProgress(minSpoutEmits, testable.getSpoutExecutors(), spoutName, 180 + 10 * slideSec); - topo.assertProgress(minBoltEmits, testable.getBoltExecutors(), boltName, 180 + 10 * slideSec); + // Waiting for spout tuples isn't strictly necessary since we also wait for bolt emits, but + // do it anyway + // Allow two minutes for topology startup, then wait for at most the time it should take to + // produce 10 windows + topo.assertProgress(minSpoutEmits, testable.getSpoutExecutors(), spoutName, 180 + + 10 * slideSec); + topo.assertProgress(minBoltEmits, testable.getBoltExecutors(), boltName, 180 + + 10 * slideSec); - final List allSpoutLogLines = topo.getDeserializedDecoratedLogLines(spoutName, TimeData::fromJson); - final List allBoltLogLines = topo.getDeserializedDecoratedLogLines(boltName, TimeDataWindow::fromJson); + final List allSpoutLogLines = topo.getDeserializedDecoratedLogLines(spoutName, + TimeData::fromJson); + final List allBoltLogLines = topo.getDeserializedDecoratedLogLines(boltName, + TimeDataWindow::fromJson); assertTrue(allBoltLogLines.size() >= minBoltEmits, - "Expecting min " + minBoltEmits + " bolt emits, found: " + allBoltLogLines.size() + " \n\t" + allBoltLogLines); + "Expecting min " + minBoltEmits + " bolt emits, found: " + allBoltLogLines.size() + + " \n\t" + allBoltLogLines); ZonedDateTime firstWindowEndTime = TimeUtil.ceil( ZonedDateTime.ofInstant(allSpoutLogLines.get(0).getDate().toInstant(), ZoneOffset.UTC), @@ -108,19 +126,24 @@ public void runAndVerifyTime(int windowSec, int slideSec, TestableTopology testa final int numberOfWindows = allBoltLogLines.size(); /* * Windows should be aligned to the slide size, starting at firstWindowEndTime - windowSec. - * Because all windows are aligned to the slide size, we can partition the spout emitted timestamps by which window they should fall in. - * This checks that the partitioned spout emits fall in the expected windows, based on the logs from the spout and bolt. + * Because all windows are aligned to the slide size, we can partition the spout emitted + * timestamps by which window they should fall in. + * This checks that the partitioned spout emits fall in the expected windows, based on the + * logs from the spout and bolt. */ for (int i = 0; i < numberOfWindows; ++i) { final ZonedDateTime windowEnd = firstWindowEndTime.plusSeconds(i * slideSec); final ZonedDateTime windowStart = windowEnd.minusSeconds(windowSec); - LOG.info("Comparing window: " + windowStart + " to " + windowEnd + " iter " + (i+1) + "/" + numberOfWindows); + LOG.info("Comparing window: " + windowStart + " to " + windowEnd + " iter " + (i + 1) + + "/" + numberOfWindows); final List expectedSpoutEmitsInWindow = allSpoutLogLines.stream() .filter(spoutLog -> { - final ZonedDateTime spoutLogTime = spoutLog.getDate().toInstant().atZone(ZoneOffset.UTC); - //The window boundaries are )windowStart, windowEnd) - return spoutLogTime.isAfter(windowStart) && spoutLogTime.isBefore(windowEnd.plusNanos(1_000_000)); + final ZonedDateTime spoutLogTime = spoutLog.getDate().toInstant() + .atZone(ZoneOffset.UTC); + // The window boundaries are )windowStart, windowEnd) + return spoutLogTime.isAfter(windowStart) && spoutLogTime.isBefore(windowEnd + .plusNanos(1_000_000)); }).collect(Collectors.toList()); TimeDataWindow expectedWindow = new TimeDataWindow(expectedSpoutEmitsInWindow); @@ -129,11 +152,15 @@ public void runAndVerifyTime(int windowSec, int slideSec, TestableTopology testa LOG.info("Expected window: " + expectedWindow.getDescription()); for (TimeData oneLog : expectedWindow.getTimeData()) { assertTrue(actualWindow.getTimeData().contains(oneLog), - () -> String.format("Missing: '%s' \n\tActual: '%s' \n\tComputed window: '%s'", oneLog, actualWindow, expectedWindow)); + () -> String + .format("Missing: '%s' \n\tActual: '%s' \n\tComputed window: '%s'", oneLog, + actualWindow, expectedWindow)); } for (TimeData oneLog : actualWindow.getTimeData()) { assertTrue(expectedWindow.getTimeData().contains(oneLog), - () -> String.format("Extra: '%s' \n\tActual: '%s' \n\tComputed window: '%s'", oneLog, actualWindow, expectedWindow)); + () -> String + .format("Extra: '%s' \n\tActual: '%s' \n\tComputed window: '%s'", oneLog, + actualWindow, expectedWindow)); } } } diff --git a/integration-test/src/test/java/org/apache/storm/st/utils/AssertUtil.java b/integration-test/src/test/java/org/apache/storm/st/utils/AssertUtil.java index 582e1593c27..b7b5fcc58f1 100644 --- a/integration-test/src/test/java/org/apache/storm/st/utils/AssertUtil.java +++ b/integration-test/src/test/java/org/apache/storm/st/utils/AssertUtil.java @@ -17,28 +17,30 @@ package org.apache.storm.st.utils; -import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.File; import java.util.Collection; import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertTrue; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class AssertUtil { private static final Logger log = LoggerFactory.getLogger(AssertUtil.class); public static void empty(Collection collection) { - assertTrue(collection == null || collection.size() == 0, "Expected collection to be non-null, found: " + collection); + assertTrue(collection == null || collection.size() == 0, + "Expected collection to be non-null, found: " + collection); } public static void nonEmpty(Collection collection, String message) { - assertNotNull(collection, message + " Expected collection to be non-null, found: " + collection); - greater(collection.size(), 0, message + " Expected collection to be non-empty, found: " + collection); + assertNotNull(collection, message + " Expected collection to be non-null, found: " + + collection); + greater(collection.size(), 0, message + " Expected collection to be non-empty, found: " + + collection); } public static void greater(int actual, int expected, String message) { @@ -63,12 +65,14 @@ public static void assertTwoElements(Collection collection) { assertNElements(collection, 2); } - public static void assertMatchCount(String actualOutput, List expectedOutput, int requiredMatchCount) { + public static void assertMatchCount(String actualOutput, List expectedOutput, + int requiredMatchCount) { for (String oneExpectedOutput : expectedOutput) { final int matchCount = StringUtils.countMatches(actualOutput, oneExpectedOutput); log.info("In output, found " + matchCount + " occurrences of: " + oneExpectedOutput); assertTrue(matchCount > requiredMatchCount, - "Found " + matchCount + "occurrence of " + oneExpectedOutput + " in urls, expected" + requiredMatchCount); + "Found " + matchCount + "occurrence of " + oneExpectedOutput + + " in urls, expected" + requiredMatchCount); } } } diff --git a/integration-test/src/test/java/org/apache/storm/st/wrapper/DecoratedLogLine.java b/integration-test/src/test/java/org/apache/storm/st/wrapper/DecoratedLogLine.java index c3090c1a993..3e1f33eb090 100644 --- a/integration-test/src/test/java/org/apache/storm/st/wrapper/DecoratedLogLine.java +++ b/integration-test/src/test/java/org/apache/storm/st/wrapper/DecoratedLogLine.java @@ -17,14 +17,13 @@ package org.apache.storm.st.wrapper; -import org.apache.storm.st.utils.AssertUtil; -import org.apache.commons.lang3.StringUtils; -import org.apache.storm.st.utils.StringDecorator; - import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.util.Arrays; import java.util.List; +import org.apache.commons.lang3.StringUtils; +import org.apache.storm.st.utils.AssertUtil; +import org.apache.storm.st.utils.StringDecorator; /** * Convenience class splitting log lines decorated with {@link StringDecorator}. @@ -33,22 +32,26 @@ public class DecoratedLogLine implements Comparable { private final ZonedDateTime logDate; private final String data; - private static final int DATE_LEN = "2016-05-04 23:38:10.702".length(); //format of date in worker logs - private static final DateTimeFormatter DATE_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS"); + private static final int DATE_LEN = + "2016-05-04 23:38:10.702".length(); // format of date in worker logs + private static final DateTimeFormatter DATE_FORMAT = DateTimeFormatter + .ofPattern("yyyy-MM-dd HH:mm:ss.SSS"); public DecoratedLogLine(String logLine) { - final List splitOnDecorator = Arrays.asList(StringDecorator.split2(StringUtils.strip(logLine))); + final List splitOnDecorator = Arrays.asList(StringDecorator.split2(StringUtils + .strip(logLine))); AssertUtil.assertTwoElements(splitOnDecorator); - this.logDate = ZonedDateTime.parse(splitOnDecorator.get(0).substring(0, DATE_LEN), DATE_FORMAT); + this.logDate = ZonedDateTime.parse(splitOnDecorator.get(0).substring(0, DATE_LEN), + DATE_FORMAT); this.data = splitOnDecorator.get(1); } @Override public String toString() { - return "LogData{" + - "logDate=" + DATE_FORMAT.format(logDate) + - ", data='" + getData() + '\'' + - '}'; + return "LogData{" + + "logDate=" + DATE_FORMAT.format(logDate) + + ", data='" + getData() + '\'' + + '}'; } @Override diff --git a/integration-test/src/test/java/org/apache/storm/st/wrapper/StormCluster.java b/integration-test/src/test/java/org/apache/storm/st/wrapper/StormCluster.java index 33f9dbbfc7a..2449c299a17 100644 --- a/integration-test/src/test/java/org/apache/storm/st/wrapper/StormCluster.java +++ b/integration-test/src/test/java/org/apache/storm/st/wrapper/StormCluster.java @@ -17,8 +17,16 @@ package org.apache.storm.st.wrapper; +import static org.junit.jupiter.api.Assertions.assertEquals; + import com.google.common.base.Predicate; import com.google.common.collect.Collections2; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import javax.annotation.Nullable; import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.storm.generated.ClusterSummary; import org.apache.storm.generated.KillOptions; @@ -32,15 +40,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.annotation.Nullable; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; - -import static org.junit.jupiter.api.Assertions.assertEquals; - public class StormCluster { private static final Logger log = LoggerFactory.getLogger(StormCluster.class); private final Nimbus.Iface client; @@ -56,7 +55,8 @@ public static Map getConfig() { public static boolean isSecure() { final String thriftConfig = "" + getConfig().get("storm.thrift.transport"); - final String thriftConfigInSecCluster = "org.apache.storm.security.auth.kerberos.KerberosSaslTransportPlugin"; + final String thriftConfigInSecCluster = + "org.apache.storm.security.auth.kerberos.KerberosSaslTransportPlugin"; return thriftConfigInSecCluster.equals(thriftConfig.trim()); } @@ -76,12 +76,14 @@ public List getKilled() throws TException { private List getTopologiesWithStatus(final String expectedStatus) throws TException { Collection topologySummaries = getSummaries(); - Collection filteredSummary = Collections2.filter(topologySummaries, new Predicate() { - @Override + Collection filteredSummary = Collections2.filter(topologySummaries, + new Predicate() { + @Override public boolean apply(@Nullable TopologySummary input) { - return input != null && input.get_status().toLowerCase().equals(expectedStatus.toLowerCase()); - } - }); + return input != null && input.get_status().toLowerCase().equals(expectedStatus + .toLowerCase()); + } + }); return new ArrayList<>(filteredSummary); } @@ -95,17 +97,20 @@ public void killOrThrow(String topologyName) throws Exception { log.info("Topology killed: " + topologyName); return; } catch (TException e) { - log.warn("Couldn't kill topology: " + topologyName + ", going to retry soon. Exception: " + ExceptionUtils.getStackTrace(e)); + log.warn("Couldn't kill topology: " + topologyName + + ", going to retry soon. Exception: " + ExceptionUtils.getStackTrace(e)); Thread.sleep(TimeUnit.SECONDS.toMillis(2)); } } - throw new RuntimeException("Failed to kill topology " + topologyName + ". Subsequent tests may fail because worker slots are occupied"); + throw new RuntimeException("Failed to kill topology " + topologyName + + ". Subsequent tests may fail because worker slots are occupied"); } public TopologySummary getOneActive() throws TException { List topoSummaries = getActive(); AssertUtil.nonEmpty(topoSummaries, "Expecting one active topology."); - assertEquals(topoSummaries.size(), 1, "Expected one topology to be running, found: " + topoSummaries); + assertEquals(topoSummaries.size(), 1, "Expected one topology to be running, found: " + + topoSummaries); return topoSummaries.get(0); } diff --git a/integration-test/src/test/java/org/apache/storm/st/wrapper/TopoWrap.java b/integration-test/src/test/java/org/apache/storm/st/wrapper/TopoWrap.java index 900509342dd..038e4d1b473 100644 --- a/integration-test/src/test/java/org/apache/storm/st/wrapper/TopoWrap.java +++ b/integration-test/src/test/java/org/apache/storm/st/wrapper/TopoWrap.java @@ -74,6 +74,7 @@ public class TopoWrap { private final String name; private final StormTopology topology; private String id; + static { String jarFile = getJarPath(); LOG.info("setting storm.jar to: " + jarFile); @@ -86,7 +87,8 @@ public TopoWrap(StormCluster cluster, String name, StormTopology topology) { this.topology = topology; } - public void submit(ImmutableMap topoConf) throws AlreadyAliveException, InvalidTopologyException, AuthorizationException { + public void submit(ImmutableMap topoConf) throws AlreadyAliveException, InvalidTopologyException, AuthorizationException { final HashMap newConfig = new HashMap<>(SUBMIT_CONF); newConfig.putAll(topoConf); StormSubmitter.submitTopologyWithProgressBar(name, newConfig, topology); @@ -97,8 +99,10 @@ private static Map getSubmitConf() { submitConf.put("storm.zookeeper.topology.auth.scheme", "digest"); submitConf.put("topology.workers", 3); submitConf.put("topology.debug", true); - //Set the metrics sample rate to 1 to force update the executor stats every time something happens - //This is necessary because getAllTimeEmittedCount relies on the executor emit stats to be accurate + // Set the metrics sample rate to 1 to force update the executor stats every time something + // happens + // This is necessary because getAllTimeEmittedCount relies on the executor emit stats to be + // accurate submitConf.put(Config.TOPOLOGY_STATS_SAMPLE_RATE, 1); return submitConf; } @@ -115,7 +119,9 @@ private static String getJarPath() { .filter(file -> file != null && !file.getName().contains("surefirebooter")) .collect(Collectors.toList()); LOG.info("Found jar files: " + jarsExcludingSurefire); - AssertUtil.nonEmpty(jarsExcludingSurefire, "The jar file is missing - did you run 'mvn clean package -DskipTests' before running tests ?"); + AssertUtil.nonEmpty(jarsExcludingSurefire, + "The jar file is missing - did you run 'mvn clean package -DskipTests' before " + + "running tests ?"); String jarFile = null; for (File jarPath : jarsExcludingSurefire) { @@ -134,7 +140,8 @@ private static String getJarPath() { public void submitSuccessfully(ImmutableMap topoConf) throws TException { submit(topoConf); TopologySummary topologySummary = getSummary(); - assertEquals(topologySummary.get_status().toLowerCase(), "active", "Topology must be active."); + assertEquals(topologySummary.get_status().toLowerCase(), "active", + "Topology must be active."); id = topologySummary.get_id(); } @@ -189,23 +196,26 @@ public long getAllTimeEmittedCount(final String componentId) throws TException { * Get the Logviewer worker log URLs for the specified component. */ public Set getLogUrls(final String componentId) throws TException, MalformedURLException { - ComponentPageInfo componentPageInfo = cluster.getNimbusClient().getComponentPageInfo(id, componentId, null, false); + ComponentPageInfo componentPageInfo = cluster.getNimbusClient().getComponentPageInfo(id, + componentId, null, false); List executorStats = componentPageInfo.get_exec_stats(); Set urls = new HashSet<>(); for (ExecutorAggregateStats execStat : executorStats) { ExecutorSummary execSummary = execStat.get_exec_summary(); String host = execSummary.get_host(); int executorPort = execSummary.get_port(); - //http://supervisor2:8000/download/DemoTest-26-1462229009%2F6703%2Fworker.log - //http://supervisor2:8000/log?file=SlidingWindowCountTest-9-1462388349%2F6703%2Fworker.log + // http://supervisor2:8000/download/DemoTest-26-1462229009%2F6703%2Fworker.log + // http://supervisor2:8000/log?file=SlidingWindowCountTest-9-1462388349%2F6703%2Fworker.log int logViewerPort = 8000; - ExecutorURL executorURL = new ExecutorURL(componentId, host, logViewerPort, executorPort, id); + ExecutorURL executorURL = new ExecutorURL(componentId, host, logViewerPort, + executorPort, id); urls.add(executorURL); } return urls; } - public void waitForProgress(int minEmits, int expectedExecutors, String componentName, int maxWaitSec) throws TException { + public void waitForProgress(int minEmits, int expectedExecutors, String componentName, + int maxWaitSec) throws TException { for (int i = 0; i < (maxWaitSec + 1) / 2; ++i) { LOG.info(getInfo().toString()); long emitCount = getAllTimeEmittedCount(componentName); @@ -219,10 +229,12 @@ public void waitForProgress(int minEmits, int expectedExecutors, String componen } } - public void assertProgress(int minEmits, int expectedExecutors, String componentName, int maxWaitSec) throws TException { + public void assertProgress(int minEmits, int expectedExecutors, String componentName, + int maxWaitSec) throws TException { waitForProgress(minEmits, expectedExecutors, componentName, maxWaitSec); long emitCount = getAllTimeEmittedCount(componentName); - assertTrue(emitCount >= minEmits, "Emit count for component '" + componentName + "' is " + emitCount + ", min is " + minEmits); + assertTrue(emitCount >= minEmits, "Emit count for component '" + componentName + "' is " + + emitCount + ", min is " + minEmits); long executorCount = getComponentExecutorCount(componentName); assertEquals(executorCount, expectedExecutors); } @@ -234,15 +246,25 @@ public static class ExecutorURL { @Override public boolean equals(Object o) { - if (this == o) return true; - if (!(o instanceof ExecutorURL)) return false; + if (this == o) { + return true; + } + if (!(o instanceof ExecutorURL)) { + return false; + } ExecutorURL that = (ExecutorURL) o; - if (componentId != null ? !componentId.equals(that.componentId) : that.componentId != null) return false; - if (getViewUrl() != null ? !getViewUrl().equals(that.getViewUrl()) : that.getViewUrl() != null) + if (componentId != null ? !componentId + .equals(that.componentId) : that.componentId != null) { return false; - return getDownloadUrl() != null ? getDownloadUrl().equals(that.getDownloadUrl()) : that.getDownloadUrl() == null; + } + if (getViewUrl() != null ? !getViewUrl().equals(that.getViewUrl()) : that + .getViewUrl() != null) { + return false; + } + return getDownloadUrl() != null ? getDownloadUrl().equals(that.getDownloadUrl()) : that + .getDownloadUrl() == null; } @@ -254,11 +276,13 @@ public int hashCode() { return result; } - public ExecutorURL(String componentId, String host, int logViewerPort, int executorPort, String topoId) throws MalformedURLException { + public ExecutorURL(String componentId, String host, int logViewerPort, int executorPort, + String topoId) throws MalformedURLException { String viewUrlStr = String.format("http://%s:%s/api/v1/log?file=", host, logViewerPort); String downloadUrlStr = String.format("http://%s:%s/api/v1/download?file=", host, logViewerPort); try { - String workerLogQueryParam = URLEncoder.encode(topoId + "/" + executorPort + "/worker.log", StandardCharsets.UTF_8.name()); + String workerLogQueryParam = URLEncoder.encode(topoId + "/" + executorPort + + "/worker.log", StandardCharsets.UTF_8.name()); viewUrl = new URL(viewUrlStr + workerLogQueryParam); downloadUrl = new URL(downloadUrlStr + workerLogQueryParam); } catch (UnsupportedEncodingException e) { @@ -277,26 +301,31 @@ public URL getViewUrl() { @Override public String toString() { - return "ExecutorURL{" + - "componentId='" + componentId + '\'' + - ", viewUrl=" + viewUrl + - ", downloadUrl=" + downloadUrl + - '}'; + return "ExecutorURL{" + + "componentId='" + componentId + '\'' + + ", viewUrl=" + viewUrl + + ", downloadUrl=" + downloadUrl + + '}'; } } /** - * Get the log lines that contain the unique {@link StringDecorator} string, deserialized from json. - * The intent is that test bolts or spouts can write the unique string, followed by json data to log via {@link StringDecorator}. - * This method will recognize such lines, and deserialize the json data using the provided decoder. + * Get the log lines that contain the unique {@link StringDecorator} string, deserialized from + * json. + * The intent is that test bolts or spouts can write the unique string, followed by json data to + * log via {@link StringDecorator}. + * This method will recognize such lines, and deserialize the json data using the provided + * decoder. */ - public List getDeserializedDecoratedLogLines(final String componentId, final FromJson jsonDeserializer) + public List getDeserializedDecoratedLogLines(final String componentId, + final FromJson jsonDeserializer) throws IOException, TException { final List logData = getDecoratedLogLines(componentId); return deserializeLogData(logData, jsonDeserializer); } - public List deserializeLogData(List logData, final FromJson jsonDeserializer) { + public List deserializeLogData(List logData, + final FromJson jsonDeserializer) { return logData.stream() .peek(Assert::assertNotNull) .map(DecoratedLogLine::getData) @@ -305,8 +334,10 @@ public List deserializeLogData(List logData, final From } /** - * Get the log lines that contain the unique {@link StringDecorator} string for the given component. - * Test spouts and bolts can write logs containing the StringDecorator string, which can be fetched using this method. + * Get the log lines that contain the unique {@link StringDecorator} string for the given + * component. + * Test spouts and bolts can write logs containing the StringDecorator string, which can be + * fetched using this method. */ public List getDecoratedLogLines(final String componentId) throws IOException, TException { final String logs = getLogs(componentId); @@ -330,8 +361,8 @@ public String getLogs(final String componentId) throws IOException, TException, Set componentLogUrls = getLogUrls(componentId); LOG.info("Found " + componentLogUrls.size() + " urls: " + componentLogUrls); List urlContents = new ArrayList<>(); - for(ExecutorURL executorUrl : componentLogUrls) { - if(executorUrl == null || executorUrl.getDownloadUrl() == null) { + for (ExecutorURL executorUrl : componentLogUrls) { + if (executorUrl == null || executorUrl.getDownloadUrl() == null) { continue; } LOG.info("Fetching: " + executorUrl); @@ -341,7 +372,8 @@ public String getLogs(final String componentId) throws IOException, TException, if (urlContent.length() < 500) { LOG.info("Fetched: " + urlContent); } else { - LOG.info("Fetched: " + NumberFormat.getNumberInstance(Locale.US).format(urlContent.length()) + " bytes."); + LOG.info("Fetched: " + NumberFormat.getNumberInstance(Locale.US).format(urlContent + .length()) + " bytes."); } if (System.getProperty("regression.downloadWorkerLogs").equalsIgnoreCase("true")) { downloadLogUrl(downloadUrl, urlContent); @@ -354,7 +386,8 @@ private void downloadLogUrl(URL downloadUrl, String urlContent) { final String userDir = System.getProperty("user.dir"); final File target = new File(userDir, "target"); final File logDir = new File(target, "logs"); - final File logFile = new File(logDir, downloadUrl.getHost() + "-" + downloadUrl.getFile().split("/")[2]); + final File logFile = new File(logDir, downloadUrl.getHost() + "-" + downloadUrl.getFile() + .split("/")[2]); try { FileUtils.forceMkdir(logDir); FileUtils.write(logFile, urlContent, StandardCharsets.UTF_8); diff --git a/pom.xml b/pom.xml index 48347f09a51..aeb7a7424d3 100644 --- a/pom.xml +++ b/pom.xml @@ -69,6 +69,11 @@ 25 + 4.2.0 + 6.46.1 + 2.41.1 + 14.0.0 + 4.0.29 false false @@ -1110,6 +1115,109 @@ + + + org.openrewrite.maven + rewrite-maven-plugin + ${rewrite-maven-plugin.version} + + + org.apache.storm.checkstyle.AutoFix + + + **/generated/** + + + false + false + + + + org.openrewrite.recipe + rewrite-static-analysis + ${rewrite-static-analysis.version} + + + + + checkstyle-autofix-openrewrite + + none + + run + + + + + + + org.codehaus.gmavenplus + gmavenplus-plugin + ${gmavenplus.version} + + + org.apache.groovy + groovy + ${groovy.version} + + + + org.apache.groovy + groovy-xml + ${groovy.version} + + + + com.puppycrawl.tools + checkstyle + ${checkstyle.version} + + + + org.apache.maven.doxia + * + + + + + + commons-logging + commons-logging + ${commons-logging.version} + + + + + checkstyle-autofix + + none + + execute + + + + + + + + + org.apache.maven.plugins @@ -1126,7 +1234,7 @@ checkstyle - 14.0.0 + ${checkstyle.version} diff --git a/rewrite.yml b/rewrite.yml new file mode 100644 index 00000000000..9f29c6f68c1 --- /dev/null +++ b/rewrite.yml @@ -0,0 +1,41 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +# First stage of the checkstyle auto-fix, run on demand by the rewrite-maven-plugin +# execution `checkstyle-autofix-openrewrite` in the root pom (bound to no phase), just before +# checkstyle-autofix.groovy (which then fixes what is left, text-level, driven by +# Checkstyle's own reports). +# +# Only recipes that do exactly one storm_checkstyle.xml rule and nothing else are listed. +# Deliberately NOT here, all tried and rejected: +# - org.openrewrite.java.format.AutoFormat: a whole-file formatter; on a module that +# already passes it rewrote 669 files and introduced star imports (AvoidStarImport). +# - org.openrewrite.staticanalysis.TypecastParenPad: turns `(T) x` into `(T)x`, which +# violates WhitespaceAfter. +# - org.openrewrite.java.OrderImports / RemoveUnusedImports: not needed by any rule, and +# the former collapses imports into star imports. +type: specs.openrewrite.org/v1beta/recipe +name: org.apache.storm.checkstyle.AutoFix +displayName: Fix storm_checkstyle.xml violations that need a parsed AST +description: > + NeedBraces, MultipleVariableDeclarations, ModifierOrder, UpperEll and ArrayTypeStyle. +recipeList: + - org.openrewrite.staticanalysis.NeedBraces + - org.openrewrite.staticanalysis.MultipleVariableDeclarations + - org.openrewrite.staticanalysis.ModifierOrder + - org.openrewrite.staticanalysis.UpperCaseLiteralSuffixes + - org.openrewrite.staticanalysis.UseJavaStyleArrayDeclarations diff --git a/storm-checkstyle/README.md b/storm-checkstyle/README.md new file mode 100644 index 00000000000..18f8da953fc --- /dev/null +++ b/storm-checkstyle/README.md @@ -0,0 +1,100 @@ +# Checkstyle auto-fix + +`storm_checkstyle.xml` is the Checkstyle ruleset enforced on every `mvn` build. +Many rules are `severity=warning` as they do not fail the build, but they should still be +slimmed to the minimal amount + +The auto-fix is **not** part of the normal build. It is run on demand in two stages, +in every module that declares the plugins: + +1. **`rewrite-maven-plugin` (OpenRewrite)** - reduced amount of fixes done on a parsed Java AST. + The recipe list is `rewrite.yml` at the repo root (`org.apache.storm.checkstyle.AutoFix`). +2. **`src/main/resources/storm/checkstyle-autofix.groovy`** (via `gmavenplus-plugin`) - + runs Checkstyle itself, in-process, with the same version and ruleset the build + uses, reads violations (file, line, column, message) and hands each one to a small fixer written for the one + Checkstyle module that reported it. The cycle until no more rules can be fixed. + +The values a fixer needs (`Indentation` `basicOffset`, `LineLength` `max`, +`CustomImportOrder` rules) are read from the ruleset XML at run time avoiding hardcoding. Files that the +module excludes from Checkstyle (`` of `maven-checkstyle-plugin`, e.g. thrift +generated code) are neither audited nor rewritten. + +Safety net, per file: a file that stops parsing after a fix is restored to the state +before that round; a file that ends with more violations than it started with is +restored to its original text. + +## What it fixes + +| Checkstyle module | Stage | +|---|---| +| `NeedBraces`, `MultipleVariableDeclarations`, `ModifierOrder`, `UpperEll`, `ArrayTypeStyle` | OpenRewrite (the Groovy stage has the same fixers as a fallback) | +| `FileTabCharacter`, `RegexpSinglelineJava` (empty-block spacing) | Groovy | +| `WhitespaceAround`, `WhitespaceAfter`, `NoWhitespaceBefore`, `NoWhitespaceBeforeCaseDefaultColon`, `MethodParamPad`, `ParenPad`, `GenericWhitespace` | Groovy | +| `LeftCurly`, `RightCurly`, `OneStatementPerLine`, `AnnotationLocation`, `NoLineWrap` | Groovy | +| `OperatorWrap`, `SeparatorWrap` | Groovy | +| `Indentation`, `CommentsIndentation`, `EmptyLineSeparator` | Groovy | +| `LineLength` | Groovy: re-wraps comments and Javadoc, and breaks code at commas, `&&`/`\|\|`/`+`/`?`, `.method(` chains and `=`, and splits long string literals into `"a" + "b"` | +| `CustomImportOrder` | Groovy: imports regrouped/sorted per the ruleset's rules | +| `AvoidStarImport` | Groovy: expanded to the explicit imports the file uses, resolved against the module's test classpath and the JDK; a star import that can't be resolved is left alone, one that supplies nothing used is dropped. Needs the module's dependencies resolvable (upstream modules built or installed) | +| `IllegalTokenText`, `AvoidEscapedUnicodeCharacters`, `TodoComment`, single-line comment space | Groovy | +| `OverloadMethodsDeclarationOrder`, `ConstructorsDeclarationGrouping` | Groovy: the member is moved next to its overloads/constructors | +| `MissingSwitchDefault` | Groovy: adds a behaviour-neutral `default: break;` | +| `JavadocLeadingAsteriskAlign`, `JavadocMissingLeadingAsterisk`, `JavadocContentLocation`, `JavadocParagraph`, `JavadocTagContinuationIndentation`, `RequireEmptyLineBeforeBlockTagGroup`, `AtclauseOrder`, `InvalidJavadocPosition`, `SummaryJavadoc` (missing period, lowercase first word) | Groovy | + +## Not fixed + +The following violations can't be fixed mechanically in a satisfactory way: + +- `MissingJavadocMethod`, `MissingJavadocType`, `JavadocMethod`, `NonEmptyAtclauseDescription`, + `SingleLineJavadoc`, the rest of `SummaryJavadoc` - human intervention for docs. +- `MethodName`, the other naming rules and `AbbreviationAsWordInName` - a rename needs + whole-repo symbol resolution. +- `FallThrough`, `EmptyCatchBlock` - comments could silence bugs. +- `VariableDeclarationUsageDistance` - moving a declaration can reorder side effects. +- `OneTopLevelClass`, `OuterTypeFilename`, `LeftCurly` on `case X: {`, + `TextBlockGoogleStyleFormatting`. +- `LineLength` on lines with no safe break point (long unbreakable tokens, URLs). + +### Rejected OpenRewrite recipes + +`rewrite.yml` deliberately lists only recipes that do exactly one Checkstyle rule. +`org.openrewrite.java.format.AutoFormat` was tried and rejected (a whole-file +formatter: on a module that already passed it rewrote ~670 files and introduced star +imports); so were `TypecastParenPad` (turns `(T) x` into `(T)x`, violating +`WhitespaceAfter`) and `OrderImports` (collapses to star imports). + +## Running it + +Both stages, on the whole project, then review the result with `git diff`: + +```sh +mvn org.openrewrite.maven:rewrite-maven-plugin:run@checkstyle-autofix-openrewrite \ + org.codehaus.gmavenplus:gmavenplus-plugin:execute@checkstyle-autofix \ + -pl '!storm-shaded-deps' -Dcheckstyle.skip=true +``` + +`-Dcheckstyle.skip=true` is needed: `rewrite:run` forks the lifecycle up to +`process-test-classes`, which would otherwise run the `validate`-bound checkstyle check +and block the auto-fix on the very violations it is meant to fix. The Groovy stage runs +Checkstyle itself and is unaffected by the flag. + +For a single module, replace the `-pl` argument (e.g. `-pl storm-client`); the Groovy +stage may need a second invocation of the same command if you want to be sure nothing +is left, since each run already repeats until nothing more can be fixed. To confirm +the result against the build's own rules afterwards: + +```sh +mvn validate -pl '!storm-shaded-deps' +``` + +### Reviewing what it did + +Both stages edit source files in place, so `git diff` shows exactly what changed. + +### Upgrading Checkstyle + +`checkstyle.version` (root `pom.xml`) is used by both the Checkstyle check and the +Groovy stage, so they can't disagree. If you change it, also update +`storm_checkstyle.xml` to match the new `google_checks.xml`: the Groovy fixers parse +Checkstyle's message text, and a reworded message makes that fixer skip the violation +(nothing breaks - Checkstyle still reports it). diff --git a/storm-checkstyle/src/main/resources/storm/checkstyle-autofix.groovy b/storm-checkstyle/src/main/resources/storm/checkstyle-autofix.groovy new file mode 100644 index 00000000000..02e6c954de0 --- /dev/null +++ b/storm-checkstyle/src/main/resources/storm/checkstyle-autofix.groovy @@ -0,0 +1,2486 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Auto-fixes storm_checkstyle.xml violations. + * + * Script runs Checkstyle itself (in-process, same version and same ruleset the build's checkstyle check uses) and + * reads back the exact violations - file, line, column, message. + * Each violation is handed to a small fixer written for the one + * Checkstyle module that reported it, which edits only the text that violation points at. + * Because the fixers act on Checkstyle's own reports, they never touch code Checkstyle is + * happy with, and there is no formatter with a style of its own to drift from the + * ruleset. The audit/fix cycle repeats (a fix can unmask or move another violation) + * until nothing more can be fixed. Values a fixer needs (Indentation's basicOffset, + * LineLength's max, CustomImportOrder's rule string) are read straight from the ruleset + * XML; a fixer whose parameters aren't declared there is skipped rather than guessing. + * + * Safety net, per file: a file that stops parsing after a fix is restored to its state + * before the last round and left alone; a file that ends up with more violations than it + * started with is restored to its original text. + * + * Fixers (by Checkstyle module): + * Layout FileTabCharacter, EmptyLineSeparator, NoLineWrap, LeftCurly, RightCurly, + * NeedBraces, OneStatementPerLine, MultipleVariableDeclarations, + * AnnotationLocation, Indentation, LineLength (comments, argument/operator/ + * call-chain wraps, long string literals) + * Spacing WhitespaceAround, WhitespaceAfter, NoWhitespaceBefore, + * NoWhitespaceBeforeCaseDefaultColon, MethodParamPad, ParenPad, + * GenericWhitespace, RegexpSinglelineJava (empty-block spacing) + * Wrapping OperatorWrap, SeparatorWrap + * Ordering CustomImportOrder, ModifierOrder, OverloadMethodsDeclarationOrder, + * ConstructorsDeclarationGrouping + * Style ArrayTypeStyle, UpperEll, IllegalTokenText, AvoidEscapedUnicodeCharacters, + * TodoComment, single-line-comment-space (MatchXpath), CommentsIndentation, + * MissingSwitchDefault (adds a behavior-neutral default) + * AvoidStarImport (expands to the explicit imports the file uses, via the module classpath) + * Javadoc JavadocLeadingAsteriskAlign, JavadocMissingLeadingAsterisk, + * JavadocContentLocation, JavadocParagraph, JavadocTagContinuationIndentation, + * RequireEmptyLineBeforeBlockTagGroup, AtclauseOrder, SummaryJavadoc + * (missing period, lowercase first word), InvalidJavadocPosition + * + * Deliberately not fixed - each needs a decision or knowledge a text edit can't have: + * Naming rules and AbbreviationAsWordInName (renames need cross-file symbol resolution), + * AvoidStarImport (needs the classpath), the MissingJavadoc checks, JavadocMethod and + * NonEmptyAtclauseDescription (need real prose), FallThrough and EmptyCatchBlock (a + * comment would only silence a possible bug), VariableDeclarationUsageDistance (moving a + * declaration can reorder side effects), OneTopLevelClass/OuterTypeFilename, LeftCurly on + * `case X: {`, TextBlockGoogleStyleFormatting. Checkstyle keeps checking all of these. + * + * Run on demand via gmavenplus-plugin's `execute` goal (execution id `checkstyle-autofix`, + * bound to no phase, so a normal build never rewrites sources); see storm-checkstyle/README.md + * for the command. Declared per-module (root pom pluginManagement) since + * project.compileSourceRoots / testCompileSourceRoots are module-specific. + */ + +import com.puppycrawl.tools.checkstyle.Checker +import com.puppycrawl.tools.checkstyle.ConfigurationLoader +import com.puppycrawl.tools.checkstyle.PropertiesExpander +import com.puppycrawl.tools.checkstyle.api.AuditEvent +import com.puppycrawl.tools.checkstyle.api.AuditListener +import groovy.io.FileType +import groovy.transform.CompileStatic +import groovy.xml.XmlSlurper + +import java.util.regex.Matcher + +// ================================================================================= +// Src: a mutable source text plus the scanning helpers the fixers share. Everything is +// expressed in character offsets into the *current* text. `mask` classifies each char as +// code / comment / string-or-char-literal, so scans never mistake the contents of a +// string or comment for syntax. +// ================================================================================= + +@CompileStatic +class Src { + static final byte CODE = 0 + static final byte COMMENT = 1 + static final byte LITERAL = 2 + + private final StringBuilder sb + private byte[] maskCache + + Src(String text) { + this.sb = new StringBuilder(text) + } + + @Override + String toString() { + sb.toString() + } + + int length() { + sb.length() + } + + char charAt(int i) { + sb.charAt(i) + } + + String slice(int from, int to) { + sb.substring(from, to) + } + + int indexOf(String s, int from) { + sb.indexOf(s, from) + } + + int lastIndexOf(String s, int from) { + sb.lastIndexOf(s, from) + } + + boolean has(int i, String s) { + if (i < 0 || i + s.length() > sb.length()) { + return false + } + for (int k = 0; k < s.length(); k++) { + if (sb.charAt(i + k) != s.charAt(k)) { + return false + } + } + true + } + + void replace(int from, int to, String s) { + sb.replace(from, to, s) + maskCache = null + } + + void insert(int at, String s) { + sb.insert(at, s) + maskCache = null + } + + void delete(int from, int to) { + sb.delete(from, to) + maskCache = null + } + + // ---- lines ------------------------------------------------------------------ + + int lineStart(int off) { + off <= 0 ? 0 : sb.lastIndexOf('\n', off - 1) + 1 + } + + int lineEnd(int off) { + int i = sb.indexOf('\n', off) + i < 0 ? sb.length() : i + } + + String line(int off) { + slice(lineStart(off), lineEnd(off)) + } + + boolean isHorizontalWs(int i) { + char c = sb.charAt(i) + c == (char) ' ' || c == (char) '\t' + } + + /** Index of the first non-blank char on the line that starts at ls (or the line end). */ + int firstNonWs(int ls) { + int i = ls + while (i < sb.length() && isHorizontalWs(i)) { + i++ + } + i + } + + String indentOf(int off) { + int ls = lineStart(off) + slice(ls, firstNonWs(ls)) + } + + boolean isBlankLine(int off) { + line(off).trim().isEmpty() + } + + /** True if the line holds only comment text (and whitespace). */ + boolean isCommentOnlyLine(int off) { + int ls = lineStart(off) + int le = lineEnd(off) + boolean seen = false + for (int i = ls; i < le; i++) { + if (Character.isWhitespace(sb.charAt(i))) { + continue + } + if (mask()[i] != COMMENT) { + return false + } + seen = true + } + seen + } + + // ---- code/comment/literal classification ------------------------------------ + + byte[] mask() { + if (maskCache == null) { + maskCache = computeMask(sb) + } + maskCache + } + + boolean isCode(int i) { + i >= 0 && i < sb.length() && mask()[i] == CODE + } + + boolean isComment(int i) { + i >= 0 && i < sb.length() && mask()[i] == COMMENT + } + + boolean isLiteral(int i) { + i >= 0 && i < sb.length() && mask()[i] == LITERAL + } + + boolean isCodeChar(int i) { + isCode(i) && !Character.isWhitespace(sb.charAt(i)) + } + + /** Last code (non-whitespace, non-comment, non-literal-interior is NOT excluded) char at or before i. */ + int prevCode(int i) { + byte[] m = mask() + for (int j = Math.min(i, sb.length() - 1); j >= 0; j--) { + if (m[j] != COMMENT && !Character.isWhitespace(sb.charAt(j))) { + return j + } + } + -1 + } + + /** First code char at or after i (literals count: their opening quote is returned). */ + int nextCode(int i) { + byte[] m = mask() + for (int j = Math.max(i, 0); j < sb.length(); j++) { + if (m[j] != COMMENT && !Character.isWhitespace(sb.charAt(j))) { + return j + } + } + -1 + } + + /** Index of the bracket closing the one at open, ignoring comments and literals; -1 if none. */ + int matchClose(int open) { + char o = sb.charAt(open) + char c = o == (char) '(' ? (char) ')' : (o == (char) '[' ? (char) ']' : (char) '}') + byte[] m = mask() + int depth = 0 + for (int i = open; i < sb.length(); i++) { + if (m[i] != CODE) { + continue + } + char ch = sb.charAt(i) + if (ch == o) { + depth++ + } else if (ch == c) { + depth-- + if (depth == 0) { + return i + } + } + } + -1 + } + + /** Index of the bracket opening the one at close; -1 if none. */ + int matchOpen(int close) { + char c = sb.charAt(close) + char o = c == (char) ')' ? (char) '(' : (c == (char) ']' ? (char) '[' : (char) '{') + byte[] m = mask() + int depth = 0 + for (int i = close; i >= 0; i--) { + if (m[i] != CODE) { + continue + } + char ch = sb.charAt(i) + if (ch == c) { + depth++ + } else if (ch == o) { + depth-- + if (depth == 0) { + return i + } + } + } + -1 + } + + /** The Java identifier/keyword starting at i ('' if none). */ + String wordAt(int i) { + int j = i + while (j < sb.length() && Character.isJavaIdentifierPart(sb.charAt(j))) { + j++ + } + slice(i, j) + } + + static byte[] computeMask(CharSequence t) { + int n = t.length() + byte[] m = new byte[n] + int i = 0 + while (i < n) { + char c = t.charAt(i) + char d = i + 1 < n ? t.charAt(i + 1) : (char) 0 + if (c == (char) '/' && d == (char) '/') { + int j = i + while (j < n && t.charAt(j) != (char) '\n') { + m[j++] = COMMENT + } + i = j + } else if (c == (char) '/' && d == (char) '*') { + int j = i + 2 + while (j < n && !(t.charAt(j) == (char) '*' && j + 1 < n && t.charAt(j + 1) == (char) '/')) { + j++ + } + int end = Math.min(n, j + 2) + for (int k = i; k < end; k++) { + m[k] = COMMENT + } + i = end + } else if (c == (char) '"' && d == (char) '"' && i + 2 < n && t.charAt(i + 2) == (char) '"') { + int j = i + 3 + while (j < n) { + char x = t.charAt(j) + if (x == (char) '\\') { + j += 2 + } else if (x == (char) '"' && j + 2 < n + 0 && t.charAt(j + 1) == (char) '"' + && t.charAt(j + 2) == (char) '"') { + j += 3 + break + } else { + j++ + } + } + int end = Math.min(n, j) + for (int k = i; k < end; k++) { + m[k] = LITERAL + } + i = end + } else if (c == (char) '"' || c == (char) '\'') { + int j = i + 1 + while (j < n && t.charAt(j) != c && t.charAt(j) != (char) '\n') { + j += t.charAt(j) == (char) '\\' ? 2 : 1 + } + int end = Math.min(n, j < n && t.charAt(j) == c ? j + 1 : j) + for (int k = i; k < end; k++) { + m[k] = LITERAL + } + i = end + } else { + i++ + } + } + m + } +} + +// ================================================================================= +// Fixers. Every fixer has the signature +// static int fix(Src s, int off, String msg, int[] origLineStarts) +// `off` is the offset Checkstyle reported (already translated to the current text; it is +// only ever handed to a fixer while everything before the previous edit is unchanged), +// `msg` its message. A fixer returns the lowest offset it modified, or NONE if it changed +// nothing. Violations are applied last-to-first, and any violation at or beyond an +// already-modified offset is deferred to the next audit round. +// ================================================================================= + +class Fixers { + static final int NONE = -1 + + /** Set from the ruleset: Indentation.basicOffset, LineLength.max. */ + static int basicOffset = 4 + static int maxLineLength = 100 + + static final List MODIFIER_ORDER = ['public', 'protected', 'private', 'abstract', 'default', + 'static', 'sealed', 'non-sealed', 'final', 'transient', 'volatile', 'synchronized', 'native', + 'strictfp'] + + // ---- small helpers ---------------------------------------------------------- + + static String tokenOf(String msg) { + Matcher m = msg =~ /'([^']+)'/ + m.find() ? m.group(1) : null + } + + static boolean hws(char c) { + c == ' ' as char || c == '\t' as char + } + + static String spaces(int n) { + ' ' * Math.max(0, n) + } + + static String nl(String indent) { + '\n' + indent + } + + static String oneIndent() { + spaces(basicOffset) + } + + /** Start of the run of comment lines that sit directly on top of the declaration at off. */ + static int leadingBlockStart(Src s, int off) { + int ls = s.lineStart(off) + while (ls > 0) { + int pls = s.lineStart(ls - 1) + if (s.isBlankLine(pls) || !s.isCommentOnlyLine(pls)) { + break + } + ls = pls + } + ls + } + + /** End (exclusive) of the member whose header starts at off: after its body's '}' or its ';'. */ + static int memberEnd(Src s, int off) { + int n = s.length() + int i = off + while (i < n) { + if (!s.isCode(i)) { + i++ + continue + } + char c = s.charAt(i) + if (c == '(' as char || c == '[' as char) { + int e = s.matchClose(i) + if (e < 0) { + return -1 + } + i = e + 1 + continue + } + if (c == ';' as char) { + return i + 1 + } + if (c == '{' as char) { + int e = s.matchClose(i) + return e < 0 ? -1 : e + 1 + } + i++ + } + -1 + } + + /** End (exclusive) of the statement whose first token is at i; -1 if it can't be determined. */ + static int stmtEnd(Src s, int from) { + int i = s.nextCode(from) + if (i < 0) { + return -1 + } + char c = s.charAt(i) + if (c == '{' as char) { + int e = s.matchClose(i) + return e < 0 ? -1 : e + 1 + } + if (c == ';' as char) { + return i + 1 + } + String w = s.wordAt(i) + switch (w) { + case 'if': + case 'for': + case 'while': + int p = s.nextCode(i + w.length()) + if (p < 0 || s.charAt(p) != '(' as char) { + return -1 + } + int close = s.matchClose(p) + if (close < 0) { + return -1 + } + int body = stmtEnd(s, close + 1) + if (w == 'if' && body >= 0) { + int nxt = s.nextCode(body) + if (nxt >= 0 && s.wordAt(nxt) == 'else') { + return stmtEnd(s, nxt + 4) + } + } + return body + case 'do': + int b = stmtEnd(s, i + 2) + if (b < 0) { + return -1 + } + int wh = s.nextCode(b) + if (wh < 0 || s.wordAt(wh) != 'while') { + return -1 + } + int wp = s.nextCode(wh + 5) + int wc = wp < 0 ? -1 : s.matchClose(wp) + int semi = wc < 0 ? -1 : s.nextCode(wc + 1) + return semi >= 0 && s.charAt(semi) == ';' as char ? semi + 1 : -1 + case 'try': + int tp = s.nextCode(i + 3) + if (tp >= 0 && s.charAt(tp) == '(' as char) { + int rc = s.matchClose(tp) + tp = rc < 0 ? -1 : s.nextCode(rc + 1) + } + if (tp < 0 || s.charAt(tp) != '{' as char) { + return -1 + } + int e = s.matchClose(tp) + 1 + while (e > 0) { + int nxt = s.nextCode(e) + String nw = nxt < 0 ? '' : s.wordAt(nxt) + if (nw == 'catch') { + int cp = s.nextCode(nxt + 5) + int cc = cp < 0 ? -1 : s.matchClose(cp) + int cb = cc < 0 ? -1 : s.nextCode(cc + 1) + if (cb < 0 || s.charAt(cb) != '{' as char) { + return -1 + } + e = s.matchClose(cb) + 1 + } else if (nw == 'finally') { + int fb = s.nextCode(nxt + 7) + if (fb < 0 || s.charAt(fb) != '{' as char) { + return -1 + } + e = s.matchClose(fb) + 1 + } else { + break + } + } + return e > 0 ? e : -1 + case 'switch': + case 'synchronized': + int sp = s.nextCode(i + w.length()) + int sc = sp < 0 || s.charAt(sp) != '(' as char ? -1 : s.matchClose(sp) + int sb2 = sc < 0 ? -1 : s.nextCode(sc + 1) + if (sb2 < 0 || s.charAt(sb2) != '{' as char) { + return -1 + } + int se = s.matchClose(sb2) + return se < 0 ? -1 : se + 1 + default: + break + } + int j = i + int n = s.length() + while (j < n) { + if (!s.isCode(j)) { + j++ + continue + } + char ch = s.charAt(j) + if (ch == '(' as char || ch == '[' as char || ch == '{' as char) { + int e = s.matchClose(j) + if (e < 0) { + return -1 + } + j = e + 1 + continue + } + if (ch == ';' as char) { + return j + 1 + } + j++ + } + -1 + } + + // ---- spacing ---------------------------------------------------------------- + + static int whitespaceAround(Src s, int off, String msg, int[] ls) { + String tok = tokenOf(msg) + if (tok == null || !s.has(off, tok)) { + return NONE + } + if (msg.contains('is not preceded')) { + if (off > 0 && Character.isWhitespace(s.charAt(off - 1))) { + return NONE + } + s.insert(off, ' ') + return off + } + if (msg.contains('is not followed')) { + int end = off + tok.length() + if (end < s.length() && Character.isWhitespace(s.charAt(end))) { + return NONE + } + s.insert(end, ' ') + return end + } + NONE + } + + static int whitespaceAfter(Src s, int off, String msg, int[] ls) { + String tok = tokenOf(msg) + int len + switch (tok) { + case 'typecast': + len = s.has(off, ')') ? 1 : -1 + break + case ',': + case ';': + len = s.has(off, tok) ? 1 : -1 + break + case '...': + case 'case': + case 'yield': + len = s.has(off, tok) ? tok.length() : -1 + break + default: + len = -1 + } + if (len < 0 || off + len >= s.length() || Character.isWhitespace(s.charAt(off + len))) { + return NONE + } + s.insert(off + len, ' ') + off + len + } + + /** Removes the horizontal whitespace right before off (never a line's indentation). */ + static int noWhitespaceBefore(Src s, int off, String msg, int[] ls) { + int j = off + while (j > 0 && s.isHorizontalWs(j - 1)) { + j-- + } + if (j == off || j == s.lineStart(off)) { + return NONE + } + s.delete(j, off) + j + } + + static int methodParamPad(Src s, int off, String msg, int[] ls) { + if (msg.contains('should be on the previous line')) { + int j = off + while (j > 0 && Character.isWhitespace(s.charAt(j - 1))) { + j-- + } + if (j == 0 || j == off || s.isComment(j - 1)) { + return NONE + } + s.delete(j, off) + return j + } + noWhitespaceBefore(s, off, msg, ls) + } + + static int parenPad(Src s, int off, String msg, int[] ls) { + if (msg.contains('is followed by')) { + int k = off + 1 + while (k < s.length() && s.isHorizontalWs(k)) { + k++ + } + if (k == off + 1 || k >= s.length() || s.charAt(k) == '\n' as char) { + return NONE + } + s.delete(off + 1, k) + return off + 1 + } + noWhitespaceBefore(s, off, msg, ls) + } + + static int genericWhitespace(Src s, int off, String msg, int[] ls) { + String tok = tokenOf(msg) + if (tok == null || !s.has(off, tok)) { + return NONE + } + int end = off + tok.length() + if (msg.contains('is followed by')) { + int k = end + while (k < s.length() && s.isHorizontalWs(k)) { + k++ + } + if (k == end || k >= s.length() || s.charAt(k) == '\n' as char) { + return NONE + } + s.delete(end, k) + return end + } + if (msg.contains('is preceded with')) { + return noWhitespaceBefore(s, off, msg, ls) + } + if (msg.contains('should followed by')) { + s.insert(end, ' ') + return end + } + if (msg.contains('is not preceded')) { + s.insert(off, ' ') + return off + } + NONE + } + + /** RegexpSinglelineJava `\{[ ]+\}`: an empty block written with spaces inside. */ + static int emptyBlockSpacing(Src s, int off, String msg, int[] ls) { + int lineStart = s.lineStart(off) + String line = s.line(off) + Matcher m = line =~ /\{[ ]+\}/ + int touched = NONE + List hits = [] + while (m.find()) { + if (s.isCode(lineStart + m.start())) { + hits << ([m.start(), m.end()] as int[]) + } + } + for (int i = hits.size() - 1; i >= 0; i--) { + s.replace(lineStart + hits[i][0], lineStart + hits[i][1], '{}') + touched = lineStart + hits[i][0] + } + touched + } + + static int commentSpace(Src s, int off, String msg, int[] ls) { + if (msg.contains('must be followed by a whitespace') && s.has(off, '//')) { + s.insert(off + 2, ' ') + return off + 2 + } + NONE + } + + // ---- line structure --------------------------------------------------------- + + static int fileTab(Src s, int off, String msg, int[] ls) { + int lineStart = s.lineStart(off) + int le = s.lineEnd(off) + boolean changed = false + for (int i = le - 1; i >= lineStart; i--) { + if (s.charAt(i) == '\t' as char && !s.isLiteral(i)) { + s.replace(i, i + 1, spaces(basicOffset)) + changed = true + } + } + changed ? lineStart : NONE + } + + static int emptyLineSeparator(Src s, int off, String msg, int[] ls) { + int start = leadingBlockStart(s, off) + if (msg.contains('more than 1 empty line')) { + int touched = NONE + // blank run above the declaration + int b = start + while (b > 0 && s.isBlankLine(s.lineStart(b - 1))) { + b = s.lineStart(b - 1) + } + if (start - b > 1) { + s.delete(b, start) + s.insert(b, '\n') + touched = b + } + return touched + } + if (msg.contains('should be separated from previous')) { + if (start == 0 || s.isBlankLine(s.lineStart(start - 1))) { + return NONE + } + s.insert(start, '\n') + return start + } + NONE + } + + static int noLineWrap(Src s, int off, String msg, int[] ls) { + int semi = s.indexOf(';', off) + if (semi < 0) { + return NONE + } + String stmt = s.slice(off, semi) + if (!stmt.contains('\n') || stmt.contains('/')) { + return NONE + } + s.replace(off, semi, stmt.replaceAll(/\s*\n\s*/, ' ').replaceAll(/\s*\.\s*/, '.')) + off + } + + static int annotationLocation(Src s, int off, String msg, int[] ls) { + if (!msg.contains('should be alone on line') || !s.has(off, '@')) { + return NONE + } + int lineStart = s.lineStart(off) + int nameEnd = off + 1 + while (nameEnd < s.length() && (Character.isJavaIdentifierPart(s.charAt(nameEnd)) || s.charAt(nameEnd) == '.' as char)) { + nameEnd++ + } + int annEnd = nameEnd + int p = nameEnd + while (p < s.length() && s.isHorizontalWs(p)) { + p++ + } + if (p < s.length() && s.charAt(p) == '(' as char) { + int close = s.matchClose(p) + annEnd = close < 0 ? nameEnd : close + 1 + } + if (!s.slice(lineStart, off).trim().isEmpty()) { + int j = off + while (j > lineStart && s.isHorizontalWs(j - 1)) { + j-- + } + s.replace(j, off, nl(s.indentOf(off))) + return j + } + int k = annEnd + while (k < s.length() && s.isHorizontalWs(k)) { + k++ + } + if (k < s.lineEnd(off) && !s.has(k, '//') && !s.has(k, '/*')) { + s.replace(annEnd, k, nl(s.indentOf(off))) + return annEnd + } + NONE + } + + static int leftCurly(Src s, int off, String msg, int[] ls) { + if (!s.has(off, '{')) { + return NONE + } + if (msg.contains('should have line break after')) { + int k = off + 1 + while (k < s.length() && s.isHorizontalWs(k)) { + k++ + } + if (k >= s.length() || s.charAt(k) == '\n' as char) { + return NONE + } + s.replace(off + 1, k, nl(s.indentOf(off) + oneIndent())) + return off + 1 + } + if (msg.contains('should be on the previous line')) { + int j = off + while (j > 0 && Character.isWhitespace(s.charAt(j - 1))) { + j-- + } + if (j == 0 || j == off || s.isComment(j - 1)) { + return NONE + } + s.replace(j, off, ' ') + return j + } + NONE + } + + static int rightCurly(Src s, int off, String msg, int[] ls) { + if (!s.has(off, '}')) { + return NONE + } + if (msg.contains('same line as the next part')) { + int k = off + 1 + while (k < s.length() && Character.isWhitespace(s.charAt(k))) { + k++ + } + String w = s.wordAt(k) + if (!(w in ['else', 'catch', 'finally', 'while']) || !s.isCode(k)) { + return NONE + } + s.replace(off + 1, k, ' ') + return off + 1 + } + if (msg.contains('alone on a line') || msg.contains('line break before')) { + int touched = NONE + int open = s.matchOpen(off) + String indent = s.indentOf(open >= 0 ? open : off) + int k = off + 1 + while (k < s.length() && s.isHorizontalWs(k)) { + k++ + } + if (k < s.lineEnd(off) && Character.isJavaIdentifierStart(s.charAt(k)) && s.isCode(k) + && !(s.wordAt(k) in ['else', 'catch', 'finally', 'while'])) { + s.replace(off + 1, k, nl(indent)) + touched = off + 1 + } + int lineStart = s.lineStart(off) + if (!s.slice(lineStart, off).trim().isEmpty()) { + int j = off + while (j > lineStart && s.isHorizontalWs(j - 1)) { + j-- + } + s.replace(j, off, nl(indent)) + touched = j + } + return touched + } + NONE + } + + /** OneStatementPerLine reports the ';' of every statement after the first on a line. */ + static int oneStatementPerLine(Src s, int off, String msg, int[] ls) { + if (!s.has(off, ';')) { + return NONE + } + int lineStart = s.lineStart(off) + int j = off - 1 + while (j >= lineStart) { + if (!s.isCode(j)) { + j-- + continue + } + char c = s.charAt(j) + if (c == ')' as char || c == ']' as char) { + int open = s.matchOpen(j) + if (open < lineStart) { + return NONE + } + j = open - 1 + continue + } + if (c == ';' as char || c == '{' as char || c == '}' as char) { + break + } + j-- + } + if (j < lineStart) { + return NONE + } + int k = j + 1 + while (k < off && s.isHorizontalWs(k)) { + k++ + } + if (k >= off || !s.isCode(k)) { + return NONE + } + s.replace(j + 1, k, nl(s.indentOf(off))) + j + 1 + } + + static int multipleVariableDeclarations(Src s, int off, String msg, int[] ls) { + int n = s.length() + int semi = -1 + int i = off + List commas = [] + int angle = 0 + while (i < n) { + if (s.isComment(i)) { + return NONE + } + if (!s.isCode(i)) { + i++ + continue + } + char c = s.charAt(i) + if (c == '(' as char || c == '[' as char || c == '{' as char) { + int e = s.matchClose(i) + if (e < 0) { + return NONE + } + i = e + 1 + continue + } + if (c == '<' as char && i > off && Character.isJavaIdentifierPart(s.charAt(i - 1)) + && i + 1 < n && s.charAt(i + 1) != ' ' as char && s.charAt(i + 1) != '=' as char + && s.charAt(i + 1) != '<' as char) { + angle++ + } else if (c == '>' as char && angle > 0 && s.charAt(i - 1) != '-' as char) { + angle-- + } else if (c == ',' as char && angle == 0) { + commas << i + } else if (c == ';' as char && angle == 0) { + semi = i + break + } + i++ + } + if (semi < 0 || commas.isEmpty()) { + return NONE + } + List pieces = [] + int from = off + for (int comma : commas) { + pieces << s.slice(from, comma) + from = comma + 1 + } + pieces << s.slice(from, semi) + // declarator name of the first piece = last identifier before its '=' (or its end) + String first = pieces[0] + int eq = -1 + for (int k = 0; k < first.length(); k++) { + if (first.charAt(k) == '=' as char && !s.isLiteral(off + k) + && (k + 1 >= first.length() || first.charAt(k + 1) != '=' as char)) { + eq = k + break + } + } + String head = (eq >= 0 ? first.substring(0, eq) : first).replaceAll(/\s+$/, '') + if (head.contains('[') || pieces.any { it.replaceAll(/=.*/, '').contains('[') }) { + return NONE + } + int nameStart = head.length() + while (nameStart > 0 && Character.isJavaIdentifierPart(head.charAt(nameStart - 1))) { + nameStart-- + } + String prefix = head.substring(0, nameStart) + if (nameStart == head.length() || prefix.trim().isEmpty() || !(prefix.endsWith(' ') || prefix.endsWith('>'))) { + return NONE + } + String indent = s.indentOf(off) + StringBuilder out = new StringBuilder(first.trim()).append(';') + for (int k = 1; k < pieces.size(); k++) { + out.append('\n').append(indent).append(prefix).append(pieces[k].trim()).append(';') + } + s.replace(off, semi + 1, out.toString()) + off + } + + static int arrayTypeStyle(Src s, int off, String msg, int[] ls) { + if (!s.has(off, '[')) { + return NONE + } + String rest = s.slice(off, s.lineEnd(off)) + Matcher m = rest =~ /^(\s*\[\s*\])+/ + if (!m.find()) { + return NONE + } + String run = m.group() + String brackets = run.replaceAll(/\s/, '') + int ns = off + while (ns > 0 && Character.isJavaIdentifierPart(s.charAt(ns - 1))) { + ns-- + } + if (ns == off) { + return NONE + } + int te = ns + while (te > 0 && s.isHorizontalWs(te - 1)) { + te-- + } + if (te == ns || te == 0) { + return NONE + } + char before = s.charAt(te - 1) + if (!(Character.isJavaIdentifierPart(before) || before == '>' as char || before == ']' as char)) { + return NONE + } + s.delete(off, off + run.length()) + s.insert(te, brackets) + te + } + + static int upperEll(Src s, int off, String msg, int[] ls) { + int e = off + while (e < s.length() && (Character.isLetterOrDigit(s.charAt(e)) || s.charAt(e) == '_' as char + || s.charAt(e) == '.' as char)) { + e++ + } + if (e > off && s.charAt(e - 1) == 'l' as char) { + s.replace(e - 1, e, 'L') + return e - 1 + } + NONE + } + + static int modifierOrder(Src s, int off, String msg, int[] ls) { + String tok = tokenOf(msg) + if (tok == null || !s.has(off, tok)) { + return NONE + } + int lineStart = s.lineStart(off) + int lineEnd = s.lineEnd(off) + int start = off + while (true) { + int k = start + while (k > lineStart && s.isHorizontalWs(k - 1)) { + k-- + } + int w = k + while (w > lineStart && (Character.isJavaIdentifierPart(s.charAt(w - 1)) || s.charAt(w - 1) == '-' as char)) { + w-- + } + if (k == start || w == k || !(s.slice(w, k) in MODIFIER_ORDER)) { + break + } + start = w + } + int end = off + tok.length() + while (true) { + int k = end + while (k < lineEnd && s.isHorizontalWs(k)) { + k++ + } + int w = k + while (w < lineEnd && (Character.isJavaIdentifierPart(s.charAt(w)) || s.charAt(w) == '-' as char)) { + w++ + } + if (k == end || w == k || !(s.slice(k, w) in MODIFIER_ORDER)) { + break + } + end = w + } + List words = s.slice(start, end).split(/\s+/).toList() + List sorted = words.sort(false) { MODIFIER_ORDER.indexOf(it) } + if (sorted == words) { + return NONE + } + s.replace(start, end, sorted.join(' ')) + start + } + + static int missingSwitchDefault(Src s, int off, String msg, int[] ls) { + if (!s.has(off, 'switch')) { + return NONE + } + int p = s.nextCode(off + 6) + int e = p < 0 || s.charAt(p) != '(' as char ? -1 : s.matchClose(p) + int b = e < 0 ? -1 : s.nextCode(e + 1) + int cl = b < 0 || s.charAt(b) != '{' as char ? -1 : s.matchClose(b) + if (cl < 0) { + return NONE + } + int closeLine = s.lineStart(cl) + if (!s.slice(closeLine, cl).trim().isEmpty()) { + return NONE + } + int caseAt = -1 + for (int i = b + 1; i < cl; i++) { + if (s.isCode(i) && s.has(i, 'case') && !Character.isJavaIdentifierPart(s.charAt(i - 1)) + && !Character.isJavaIdentifierPart(s.charAt(i + 4))) { + caseAt = i + break + } + } + if (caseAt < 0) { + return NONE + } + boolean arrow = false + boolean found = false + for (int i = caseAt + 4; i < cl && !found; i++) { + if (!s.isCode(i)) { + continue + } + if (s.has(i, '->')) { + arrow = true + found = true + } else if (s.charAt(i) == ':' as char && !s.has(i, '::') && s.charAt(i - 1) != ':' as char) { + found = true + } + } + if (!found) { + return NONE + } + String indent = s.indentOf(caseAt) + String text = arrow ? indent + 'default -> {}\n' : indent + 'default:\n' + indent + oneIndent() + 'break;\n' + s.insert(closeLine, text) + closeLine + } + + static int needBraces(Src s, int off, String msg, int[] ls) { + String kw = tokenOf(msg) + if (kw == null || !s.has(off, kw)) { + return NONE + } + int hdrEnd + if (kw in ['if', 'for', 'while']) { + int p = s.nextCode(off + kw.length()) + int e = p < 0 || s.charAt(p) != '(' as char ? -1 : s.matchClose(p) + if (e < 0) { + return NONE + } + hdrEnd = e + 1 + } else if (kw == 'else') { + hdrEnd = off + 4 + } else if (kw == 'do') { + hdrEnd = off + 2 + } else { + return NONE + } + int bodyStart = s.nextCode(hdrEnd) + if (bodyStart < 0 || s.charAt(bodyStart) == '{' as char) { + return NONE + } + if (kw == 'else' && s.wordAt(bodyStart) == 'if') { + return NONE + } + if (s.charAt(bodyStart) == ';' as char) { + s.replace(bodyStart, bodyStart + 1, '{}') + return bodyStart + } + int bodyEnd = stmtEnd(s, bodyStart) + if (bodyEnd < 0) { + return NONE + } + String indent = s.indentOf(off) + boolean bodyOnNewLine = s.slice(hdrEnd, bodyStart).contains('\n') + int k = bodyEnd + while (k < s.length() && s.isHorizontalWs(k)) { + k++ + } + int closeAt = s.has(k, '//') ? s.lineEnd(k) : bodyEnd + s.insert(closeAt, nl(indent) + '}') + if (bodyOnNewLine) { + s.insert(hdrEnd, ' {') + } else { + s.replace(hdrEnd, bodyStart, ' {' + nl(indent + oneIndent())) + } + hdrEnd + } + + // ---- wrapping --------------------------------------------------------------- + + /** Operator/'.'/'::' left dangling at a line's end: move it to the start of the next line. */ + static int moveToNextLine(Src s, int off, String tok, boolean spaceAfter) { + if (!s.has(off, tok)) { + return NONE + } + int lineStart = s.lineStart(off) + int le = s.lineEnd(off) + String rest = s.slice(off + tok.length(), le).trim() + String comment = null + if (!rest.isEmpty()) { + if (!rest.startsWith('//')) { + return NONE + } + comment = rest + } + if (le >= s.length() - 1) { + return NONE + } + int nls = le + 1 + int nfn = s.firstNonWs(nls) + if (nfn >= s.lineEnd(nls) || s.isComment(nfn)) { + return NONE + } + int j = off + while (j > lineStart && s.isHorizontalWs(j - 1)) { + j-- + } + if (j == lineStart) { + return NONE + } + s.insert(nfn, tok + (spaceAfter ? ' ' : '')) + s.replace(j, le, comment != null ? ' ' + comment : '') + j + } + + /** ',' or '...' that starts a line: move it to the end of the previous line's code. */ + static int moveToPrevLine(Src s, int off, String tok) { + if (!s.has(off, tok)) { + return NONE + } + int lineStart = s.lineStart(off) + if (lineStart == 0 || !s.slice(lineStart, off).trim().isEmpty()) { + return NONE + } + int pc = s.prevCode(lineStart - 1) + if (pc < 0) { + return NONE + } + int after = off + tok.length() + int k = after + while (k < s.length() && s.isHorizontalWs(k)) { + k++ + } + int le = s.lineEnd(off) + if (k >= le) { + s.delete(lineStart, Math.min(s.length(), le + 1)) + } else { + s.delete(off, k) + } + s.insert(pc + 1, tok) + pc + 1 + } + + static int operatorWrap(Src s, int off, String msg, int[] ls) { + String tok = tokenOf(msg) + if (tok == null || !msg.contains('should be on a new line')) { + return NONE + } + moveToNextLine(s, off, tok, true) + } + + static int separatorWrap(Src s, int off, String msg, int[] ls) { + String tok = tokenOf(msg) + if (tok == null) { + return NONE + } + if (msg.contains('should be on a new line')) { + String line = s.line(off).trim() + if (line.startsWith('import ') || line.startsWith('package ')) { + return NONE + } + return moveToNextLine(s, off, tok, false) + } + if (msg.contains('should be on the previous line')) { + return moveToPrevLine(s, off, tok) + } + NONE + } + + // ---- ordering --------------------------------------------------------------- + + /** Moves the member at off (with its leading comments) to just after the member at targetLine. */ + static int moveMemberAfter(Src s, int off, int targetLine, int[] origLineStarts) { + if (targetLine < 1 || targetLine > origLineStarts.length) { + return NONE + } + int cs = leadingBlockStart(s, off) + int ce = memberEnd(s, off) + int t = s.firstNonWs(origLineStarts[targetLine - 1]) + int te = t >= cs ? -1 : memberEnd(s, t) + if (ce < 0 || te < 0 || te > cs) { + return NONE + } + String member = s.slice(cs, ce) + int del = ce + if (del < s.length() && s.charAt(del) == '\n' as char) { + del++ + } + s.delete(cs, del) + if (cs > 0 && s.isBlankLine(s.lineStart(cs - 1)) && (cs >= s.length() || s.isBlankLine(cs) + || s.line(cs).trim().startsWith('}'))) { + int pb = s.lineStart(cs - 1) + s.delete(pb, Math.min(s.length(), s.lineEnd(pb) + 1)) + } + s.insert(te, '\n\n' + member) + te + } + + static int overloadOrder(Src s, int off, String msg, int[] ls) { + Matcher m = msg =~ /line '(\d+)'/ + m.find() ? moveMemberAfter(s, off, m.group(1) as int, ls) : NONE + } + + static int constructorGrouping(Src s, int off, String msg, int[] ls) { + Matcher m = msg =~ /line '(\d+)'/ + m.find() ? moveMemberAfter(s, off, m.group(1) as int, ls) : NONE + } + + // ---- literals --------------------------------------------------------------- + + private static final Map ESCAPES = [ + 8: '\\b', 9: '\\t', 10: '\\n', 12: '\\f', 13: '\\r', 32: ' ', 34: '\\"', 39: "\\'", 92: '\\\\'] + + static int literalEnd(Src s, int off) { + int e = off + while (e < s.length() && s.isLiteral(e)) { + e++ + } + e + } + + static int illegalTokenText(Src s, int off, String msg, int[] ls) { + if (!s.isLiteral(off)) { + return NONE + } + int end = literalEnd(s, off) + String lit = s.slice(off, end) + StringBuilder out = new StringBuilder() + int i = 0 + while (i < lit.length()) { + char c = lit.charAt(i) + if (c != '\\' as char || i + 1 >= lit.length()) { + out.append(c) + i++ + continue + } + Matcher u = lit.substring(i) =~ /^\\u+([0-9a-fA-F]{4})/ + Matcher o = lit.substring(i) =~ /^\\(0(?:10|11|12|14|15|40|42|47)|134)(?![0-7])/ + if (u.find() && ESCAPES.containsKey(Integer.parseInt(u.group(1), 16))) { + out.append(ESCAPES[Integer.parseInt(u.group(1), 16)]) + i += u.end() + } else if (o.find()) { + out.append(ESCAPES[Integer.parseInt(o.group(1), 8)]) + i += o.end() + } else { + out.append(c).append(lit.charAt(i + 1)) + i += 2 + } + } + if (out.toString() == lit) { + return NONE + } + s.replace(off, end, out.toString()) + off + } + + static int avoidEscapedUnicode(Src s, int off, String msg, int[] ls) { + int start = off + // the reported column may be the string's opening quote or the escape itself + if (!s.isLiteral(off)) { + return NONE + } + while (start > 0 && s.isLiteral(start - 1)) { + start-- + } + int end = literalEnd(s, off) + String lit = s.slice(start, end) + Matcher m = lit =~ /(? s.length() || !s.slice(i, i + 4).equalsIgnoreCase('todo') || !s.isComment(i)) { + return NONE + } + int e = i + 4 + if (e < s.length() && s.charAt(e) == ':' as char) { + e++ + } + s.replace(i, e, 'TODO:') + i + } + + // ---- javadoc ---------------------------------------------------------------- + + // [start, end) of the javadoc comment enclosing off, or null. + static int[] javadocBounds(Src s, int off) { + int start = s.lastIndexOf('/**', off) + if (start < 0) { + return null + } + int end = s.indexOf('*/', start + 3) + if (end < 0 || end + 2 < off) { + return null + } + [start, end + 2] as int[] + } + + static int javadocAsteriskAlign(Src s, int off, String msg, int[] ls) { + Matcher m = msg =~ /expected is (\d+)/ + if (!m.find()) { + return NONE + } + int lineStart = s.lineStart(off) + int fnw = s.firstNonWs(lineStart) + if (fnw >= s.length() || s.charAt(fnw) != '*' as char) { + return NONE + } + s.replace(lineStart, fnw, spaces((m.group(1) as int) - 1)) + lineStart + } + + static int javadocMissingAsterisk(Src s, int off, String msg, int[] ls) { + int[] b = javadocBounds(s, off) + if (b == null) { + return NONE + } + int col = b[0] - s.lineStart(b[0]) + int lineStart = s.lineStart(off) + String t = s.line(off).trim() + String fixed = t.startsWith('*/') ? spaces(col + 1) + t : (t.isEmpty() ? spaces(col + 1) + '*' : spaces(col + 1) + '* ' + t) + s.replace(lineStart, s.lineEnd(off), fixed) + lineStart + } + + static int javadocContentLocation(Src s, int off, String msg, int[] ls) { + if (!s.has(off, '/**')) { + return NONE + } + int k = off + 3 + while (k < s.length() && s.isHorizontalWs(k)) { + k++ + } + if (k >= s.length() || s.charAt(k) == '\n' as char || s.has(k, '*/')) { + return NONE + } + int col = off - s.lineStart(off) + s.replace(off + 3, k, '\n' + spaces(col + 1) + '* ') + off + 3 + } + + static int requireEmptyLineBeforeTags(Src s, int off, String msg, int[] ls) { + int lineStart = s.lineStart(off) + int fnw = s.firstNonWs(lineStart) + if (fnw >= s.length() || s.charAt(fnw) != '*' as char || s.has(fnw, '*/')) { + return NONE + } + s.insert(lineStart, s.slice(lineStart, fnw + 1) + '\n') + lineStart + } + + static int javadocParagraph(Src s, int off, String msg, int[] ls) { + if (msg.contains('should be preceded with an empty line')) { + int lineStart = s.lineStart(off) + int fnw = s.firstNonWs(lineStart) + if (fnw >= s.length() || s.charAt(fnw) != '*' as char || !s.slice(fnw + 1, off).trim().isEmpty()) { + return NONE + } + s.insert(lineStart, s.slice(lineStart, fnw + 1) + '\n') + return lineStart + } + if (msg.contains('immediately before the first word')) { + if (!s.has(off, '

    ')) { + return NONE + } + int k = off + 3 + while (k < s.length() && s.isHorizontalWs(k)) { + k++ + } + if (k < s.length() && s.charAt(k) == '\n' as char) { + int nls = k + 1 + int nfn = s.firstNonWs(nls) + if (nfn >= s.length() || s.charAt(nfn) != '*' as char || s.has(nfn, '*/')) { + return NONE + } + int text = nfn + 1 + while (text < s.length() && s.isHorizontalWs(text)) { + text++ + } + if (text >= s.lineEnd(nls)) { + return NONE + } + s.delete(off + 3, text) + return off + 3 + } + if (k == off + 3) { + return NONE + } + s.delete(off + 3, k) + return off + 3 + } + if (msg.contains('should be followed by

    ')) { + int nls = s.lineEnd(off) + 1 + if (nls >= s.length()) { + return NONE + } + int nfn = s.firstNonWs(nls) + if (s.charAt(nfn) != '*' as char || s.has(nfn, '*/')) { + return NONE + } + int text = nfn + 1 + while (text < s.length() && s.isHorizontalWs(text)) { + text++ + } + if (text >= s.lineEnd(nls) || s.charAt(text) == '@' as char || s.charAt(text) == '<' as char) { + return NONE + } + s.insert(text, '

    ') + return text + } + NONE + } + + static int javadocTagContinuation(Src s, int off, String msg, int[] ls) { + int lineStart = s.lineStart(off) + Matcher m = s.line(off) =~ /^(\s*\*)\s*(\S.*)$/ + if (!m.matches()) { + return NONE + } + s.replace(lineStart, s.lineEnd(off), m.group(1) + spaces(basicOffset + 1) + m.group(2)) + lineStart + } + + static int atclauseOrder(Src s, int off, String msg, int[] ls) { + int[] b = javadocBounds(s, off) + if (b == null) { + return NONE + } + List lines = s.slice(b[0], b[1]).split('\n', -1).toList() + if (lines.size() < 3 || lines.last().trim() != '*/') { + return NONE + } + Map rank = ['@param': 0, '@return': 1, '@throws': 2, '@deprecated': 3] + int first = lines.findIndexOf { it.replaceFirst(/^\s*\*\s?/, '').startsWith('@') } + if (first < 0) { + return NONE + } + List> blocks = [] + List tags = [] + for (int i = first; i < lines.size() - 1; i++) { + String content = lines[i].replaceFirst(/^\s*\*\s?/, '') + if (content.startsWith('@')) { + blocks << [] + tags << content.split(/[\s{]/)[0] + } + blocks.last() << lines[i] + } + List slots = (0.. ordered = slots.sort(false) { rank[tags[it]] } + if (slots == ordered) { + return NONE + } + List> rebuilt = new ArrayList<>(blocks) + for (int k = 0; k < slots.size(); k++) { + rebuilt[slots[k]] = blocks[ordered[k]] + } + List out = lines.subList(0, first) + rebuilt.flatten() + [lines.last()] + s.replace(b[0], b[1], out.join('\n')) + b[0] + } + + static int summaryJavadoc(Src s, int off, String msg, int[] ls) { + // Checkstyle reports these right after the opening slash-star-star: find the first text char + while (off < s.length() && (Character.isWhitespace(s.charAt(off)) + || (s.charAt(off) == '*' as char && !s.has(off, '*/')))) { + off++ + } + if (msg.contains('Forbidden summary fragment')) { + Matcher m = s.slice(off, Math.min(s.length(), off + 40)) =~ /^[a-z]+(?=[\s.,])/ + if (!m.find() || !s.isComment(off)) { + return NONE + } + s.replace(off, off + 1, s.slice(off, off + 1).toUpperCase()) + return off + } + if (msg.contains('missing an ending period')) { + int lineStart = s.lineStart(off) + int lastEnd = -1 + int cur = lineStart + boolean firstLine = true + while (cur <= s.length()) { + int le = s.lineEnd(cur) + int textStart = firstLine ? off : s.firstNonWs(cur) + if (!firstLine && textStart < le && s.charAt(textStart) == '*' as char && !s.has(textStart, '*/')) { + textStart++ + while (textStart < le && s.isHorizontalWs(textStart)) { + textStart++ + } + } + String content = s.slice(Math.min(textStart, le), le) + int close = content.indexOf('*/') + if (close >= 0) { + content = content.substring(0, close) + } + String t = content.trim() + if (t.isEmpty() || (!firstLine && (t.startsWith('@') || t.startsWith('

    ')))) { + break + } + lastEnd = textStart + content.replaceAll(/\s+$/, '').length() + if (close >= 0 || le >= s.length()) { + break + } + cur = le + 1 + firstLine = false + } + if (lastEnd < 0) { + return NONE + } + char last = s.charAt(lastEnd - 1) + if (last == ':' as char || last == '>' as char || last == '.' as char) { + return NONE + } + s.insert(lastEnd, '.') + return lastEnd + } + NONE + } + + static int invalidJavadocPosition(Src s, int off, String msg, int[] ls) { + if (s.has(off, '/**') && !s.has(off, '/**/')) { + s.replace(off, off + 3, '/*') + return off + } + NONE + } + + // ---- indentation ------------------------------------------------------------ + + static int indentation(Src s, int off, String msg, int[] ls) { + Matcher m = msg =~ /expected level should be (?:one of the following: )?(\d+)/ + if (!m.find()) { + return NONE + } + int lineStart = s.lineStart(off) + int fnw = s.firstNonWs(lineStart) + if (fnw != off) { + return NONE + } + int want = m.group(1) as int + if (fnw - lineStart == want) { + return NONE + } + s.replace(lineStart, fnw, spaces(want)) + lineStart + } + + static int commentsIndentation(Src s, int off, String msg, int[] ls) { + Matcher m = msg =~ /expected is (\d+)/ + if (!m.find()) { + return NONE + } + int want = m.group(1) as int + int lineStart = s.lineStart(off) + if (!s.slice(lineStart, off).trim().isEmpty() || off - lineStart == want) { + return NONE + } + if (s.has(off, '//')) { + s.replace(lineStart, off, spaces(want)) + return lineStart + } + int delta = want - (off - lineStart) + int end = s.indexOf('*/', off) + if (end < 0) { + return NONE + } + List starts = [] + int cur = lineStart + while (cur <= end) { + starts << cur + int le = s.lineEnd(cur) + if (le >= s.length()) { + break + } + cur = le + 1 + } + for (int k = starts.size() - 1; k >= 0; k--) { + int st = starts[k] + int fnw = s.firstNonWs(st) + int cur2 = fnw - st + s.replace(st, fnw, spaces(cur2 + delta)) + } + lineStart + } + + // ---- line length ------------------------------------------------------------ + + static int lineLength(Src s, int off, String msg, int[] ls) { + int lineStart = s.lineStart(off) + int le = s.lineEnd(off) + String line = s.slice(lineStart, le) + int max = maxLineLength + if (line.length() <= max) { + return NONE + } + int fnw = s.firstNonWs(lineStart) + if (fnw >= le) { + return NONE + } + if (s.isComment(fnw)) { + return wrapComment(s, lineStart, fnw, le, max) + } + wrapCode(s, lineStart, fnw, le, max) + } + + private static int wrapComment(Src s, int lineStart, int fnw, int le, int max) { + String line = s.slice(lineStart, le) + if (line =~ /(https?|ftp):\/\/|href\s*=/) { + return NONE + } + String prefix + String content + if (s.has(fnw, '//')) { + int t = fnw + 2 + while (t < le && s.isHorizontalWs(t)) { + t++ + } + prefix = s.slice(lineStart, fnw) + '// ' + content = s.slice(t, le).trim() + } else if (s.charAt(fnw) == '*' as char && !s.has(fnw, '*/')) { + int t = fnw + 1 + while (t < le && s.isHorizontalWs(t)) { + t++ + } + content = s.slice(t, le).trim() + if (content.contains('*/') || content.startsWith(' 0) { + int pls = s.lineStart(cur - 1) + String pl = s.line(pls).trim().replaceFirst(/^\/?\*+\s?/, '') + if (pl.contains('

    ') && !pl.contains('
    ')) { + return NONE + } + if (pl.startsWith('@')) { + extra = spaces(basicOffset) + break + } + if (pl.isEmpty() || s.line(pls).trim().startsWith('/**') || s.line(pls).trim().startsWith('/*')) { + break + } + cur = pls + } + if (content.startsWith('@') || !extra.isEmpty()) { + extra = spaces(basicOffset) + } + prefix = s.slice(lineStart, fnw + 1) + ' ' + (content.startsWith('@') ? '' : extra) + if (content.startsWith('@')) { + // wrapped tag lines continue indented + prefix = s.slice(lineStart, fnw + 1) + ' ' + spaces(basicOffset) + return wrapWords(s, lineStart, le, s.slice(lineStart, fnw + 1) + ' ', prefix, content, max) + } + return wrapWords(s, lineStart, le, s.slice(lineStart, fnw + 1) + ' ' + extra, prefix, content, max) + } else { + return NONE + } + wrapWords(s, lineStart, le, prefix, prefix, content, max) + } + + private static int wrapWords(Src s, int lineStart, int le, String firstPrefix, String nextPrefix, String content, int max) { + List words = content.split(/\s+/).toList() + List out = [] + StringBuilder cur = new StringBuilder(firstPrefix) + boolean empty = true + for (String w : words) { + int add = (empty ? 0 : 1) + w.length() + if (!empty && cur.length() + add > max) { + out << cur.toString() + cur = new StringBuilder(nextPrefix) + empty = true + add = w.length() + } + if (!empty) { + cur.append(' ') + } + cur.append(w) + empty = false + } + out << cur.toString() + if (out.size() < 2 || out.any { it.length() > max && it.trim().split(/\s+/).size() > 2 }) { + if (out.size() < 2) { + return NONE + } + } + s.replace(lineStart, le, out.join('\n')) + lineStart + } + + private static int wrapCode(Src s, int lineStart, int fnw, int le, int max) { + String head = s.slice(fnw, Math.min(le, fnw + 8)) + if (head.startsWith('import ') || head.startsWith('package ')) { + return NONE + } + // candidate break offsets, all inside code + int best = -1 + int depth = 0 + for (int i = fnw; i < le; i++) { + if (!s.isCode(i)) { + continue + } + char c = s.charAt(i) + if (c == '(' as char || c == '[' as char) { + depth++ + } else if (c == ')' as char || c == ']' as char) { + depth-- + } + int pos = -1 + if (c == ',' as char && depth > 0) { + pos = i + 1 + } else if ((s.has(i, '&&') || s.has(i, '||')) && i > fnw && s.isHorizontalWs(i - 1)) { + pos = i + } else if (c == '+' as char && i > fnw && s.isHorizontalWs(i - 1) && i + 1 < le && s.isHorizontalWs(i + 1)) { + pos = i + } else if (c == '?' as char && i > fnw && s.isHorizontalWs(i - 1) && i + 1 < le && s.isHorizontalWs(i + 1)) { + pos = i + } else if (c == '.' as char && i > fnw && i + 1 < le && Character.isJavaIdentifierStart(s.charAt(i + 1)) + && (s.charAt(i - 1) == ')' as char || Character.isJavaIdentifierPart(s.charAt(i - 1)))) { + String after = s.slice(i + 1, le) + if (after =~ /^\w+\s*\(/) { + pos = i + } + } + if (pos > fnw && pos - lineStart <= max && (pos - lineStart) > (fnw - lineStart) + 8) { + best = pos + } + } + if (best < 0) { + for (int i = fnw; i < le; i++) { + if (s.isCode(i) && s.charAt(i) == '=' as char && s.isHorizontalWs(i - 1) && i + 1 < le && s.isHorizontalWs(i + 1) + && i + 1 - lineStart <= max && i + 1 > fnw + 8) { + best = i + 1 + } + } + } + if (best < 0) { + return splitString(s, lineStart, fnw, le, max) + } + int trimEnd = best + while (trimEnd > lineStart && s.isHorizontalWs(trimEnd - 1)) { + trimEnd-- + } + int restStart = best + while (restStart < le && s.isHorizontalWs(restStart)) { + restStart++ + } + if (restStart >= le) { + return NONE + } + s.replace(trimEnd, restStart, nl(continuationIndent(s, lineStart, fnw))) + trimEnd + } + + /** Indent for a wrapped continuation: keep an existing continuation's indent, else statement indent + 2 levels. */ + static String continuationIndent(Src s, int lineStart, int fnw) { + String indent = s.slice(lineStart, fnw) + int p = lineStart > 0 ? s.prevCode(lineStart - 1) : -1 + boolean statementStart = p < 0 || s.charAt(p) == ';' as char || s.charAt(p) == '{' as char + || s.charAt(p) == '}' as char + statementStart ? indent + spaces(2 * basicOffset) : indent + } + + private static int splitString(Src s, int lineStart, int fnw, int le, int max) { + for (int i = fnw; i < le; i++) { + if (!s.isLiteral(i) || s.charAt(i) != '"' as char || (i > 0 && s.isLiteral(i - 1))) { + continue + } + int end = literalEnd(s, i) + if (end - i < 6 || s.has(i, '"""') || end <= lineStart + max - 1 || end > le) { + continue + } + int prev = s.prevCode(i - 1) + int next = s.nextCode(end) + if (prev < 0 || next < 0) { + return NONE + } + char pc = s.charAt(prev) + String pw = Character.isJavaIdentifierPart(pc) ? s.slice(Math.max(0, prev - 5), prev + 1) : '' + char nc = s.charAt(next) + boolean okPrev = pc in ['(', ',', '=', '+', '?', ':', '{'] as char[] || pw.endsWith('return') || pw.endsWith('case') + boolean okNext = nc in [',', ')', ';', '+', '?', ':', '}'] as char[] + if (!okPrev || !okNext) { + return NONE + } + // safe split points: boundaries that don't cut an escape sequence + List safe = [] + int k = i + 1 + while (k < end - 1) { + if (s.charAt(k) == '\\' as char) { + Matcher u = s.slice(k, end) =~ /^\\u+[0-9a-fA-F]{4}/ + k += u.find() ? u.end() : 2 + } else { + k++ + } + safe << k + } + int limit = lineStart + max - 2 + int cut = -1 + for (int cand : safe) { + if (cand <= limit && cand > i + 8 && s.charAt(cand - 1) == ' ' as char) { + cut = cand + } + } + if (cut < 0) { + for (int cand : safe) { + if (cand <= limit && cand > i + 8) { + cut = cand + } + } + } + if (cut < 0 || cut >= end - 1) { + return NONE + } + s.replace(cut, cut, '"' + nl(continuationIndent(s, lineStart, fnw)) + '+ "') + return cut + } + NONE + } +} + +// ================================================================================= +// Script body +// ================================================================================= + +// --------------------------------------------------------------------------------- +// Load the ruleset directly - the same file maven-checkstyle-plugin's configLocation uses (root pom). +// --------------------------------------------------------------------------------- + +String multiModuleDir = System.getProperty('maven.multiModuleProjectDirectory') +File rulesetFile = new File("${multiModuleDir}/storm-checkstyle/src/main/resources/storm/storm_checkstyle.xml") +if (!rulesetFile.exists()) { + println "[checkstyle-autofix] ${project.artifactId}: ruleset ${rulesetFile} not found, skipping." + return +} + +// Checkstyle rulesets always carry a pointing at the +// public checkstyle DTD. XmlSlurper's default SAXParser rejects any DOCTYPE outright +// (disallow-doctype-decl=true, a blanket XXE hardening default), so it must be turned +// off here - but external entity/DTD fetching stays disabled so this doesn't reopen +// an XXE hole; the doctype is parsed, its external subset never is. +def xmlSlurper = new XmlSlurper() +xmlSlurper.setFeature('http://apache.org/xml/features/disallow-doctype-decl', false) +xmlSlurper.setFeature('http://xml.org/sax/features/external-general-entities', false) +xmlSlurper.setFeature('http://xml.org/sax/features/external-parameter-entities', false) +xmlSlurper.setFeature('http://apache.org/xml/features/nonvalidating/load-external-dtd', false) +def checker = xmlSlurper.parse(rulesetFile) +def treeWalker = checker.module.find { it.@name == 'TreeWalker' } + +def findModule = { String name -> treeWalker.module.find { it.@name == name } } +def moduleProperty = { module, String propName, String defaultValue = null -> + def prop = module?.property?.find { it.@name == propName } + prop ? prop.@value.text() : defaultValue +} + +Fixers.basicOffset = moduleProperty(findModule('Indentation'), 'basicOffset', '4').toInteger() +Fixers.maxLineLength = moduleProperty(checker.module.find { it.@name == 'LineLength' }, 'max', '100').toInteger() + +def customImportOrderModule = findModule('CustomImportOrder') + +def sourceRoots = ((project.compileSourceRoots ?: []) + (project.testCompileSourceRoots ?: [])).unique() +// Honour the same the module gives maven-checkstyle-plugin (e.g. thrift-generated +// code): those files aren't checked, so they must not be rewritten - or even audited, which +// for the huge generated classes is very slow. +def checkstylePlugin = project.build?.plugins?.find { it.artifactId == 'maven-checkstyle-plugin' } +List excludeMatchers = (checkstylePlugin?.configuration?.getChild('excludes')?.value ?: '') + .split(',').collect { it.trim() }.findAll { it } + .collect { java.nio.file.FileSystems.default.getPathMatcher("glob:/${it}".toString()) } +List javaFiles = [] +sourceRoots.each { rootPath -> + File rootDir = new File(rootPath) + if (rootDir.exists()) { + rootDir.eachFileRecurse(FileType.FILES) { file -> + String relative = '/' + rootDir.toPath().relativize(file.toPath()).toString().replace(File.separator, '/') + if (file.name.endsWith('.java') && !excludeMatchers.any { it.matches(java.nio.file.Paths.get(relative)) }) { + javaFiles << file + } + } + } +} + +/* + * Checkstyle module: CustomImportOrder. + * Reads customImportOrderRules (e.g. "STATIC###THIRD_PARTY_PACKAGE"), + * sortImportsInGroupAlphabetically, and separateLineBetweenGroups directly from the + * module's own properties - an empty/missing rule list disables this fixer entirely + * rather than assuming a group order the ruleset never declared. + * Fix: re-emit the whole leading import block (the contiguous run of "import "/ + * "import static " lines at the top of the file, before the first non-import, + * non-comment, non-package line) grouped and ordered exactly per those properties, + * with a blank line between groups only if separateLineBetweenGroups=true. + * Only STATIC and THIRD_PARTY_PACKAGE groups are recognized (the two this ruleset + * uses); a rule list naming any other group is left unfixed, since guessing that + * group's package-prefix membership isn't something the ruleset spells out. + * Returns the new list of lines, or the original if the block already matches or + * the fixer doesn't apply. + */ +def fixCustomImportOrder = { List lines -> + if (customImportOrderModule == null) { + return lines + } + String rule = moduleProperty(customImportOrderModule, 'customImportOrderRules', '') + List groupOrder = rule.split('###').findAll { it } + if (groupOrder.isEmpty() || !groupOrder.every { it == 'STATIC' || it == 'THIRD_PARTY_PACKAGE' }) { + return lines + } + boolean alphabetical = moduleProperty(customImportOrderModule, 'sortImportsInGroupAlphabetically', 'false') == 'true' + boolean separateGroups = moduleProperty(customImportOrderModule, 'separateLineBetweenGroups', 'false') == 'true' + + int start = -1 + int end = -1 + for (int i = 0; i < lines.size(); i++) { + String trimmed = lines[i].trim() + if (trimmed.startsWith('import ')) { + if (start < 0) { + start = i + } + end = i + } else if (start >= 0 && !trimmed.isEmpty()) { + break + } + } + if (start < 0) { + return lines + } + + List importLines = lines[start..end].findAll { it.trim().startsWith('import ') } + if (importLines.any { !it.trim().endsWith(';') } || lines.size() > end + 1 && !lines[end + 1].trim().isEmpty() + && !lines[end + 1].trim().matches(/(public|final|abstract|class|interface|enum|record|@|\/).*/)) { + // a line-wrapped import (or anything else unexpected): leave it to the NoLineWrap fixer + return lines + } + Map> groups = ['STATIC': [], 'THIRD_PARTY_PACKAGE': []] + importLines.each { String imp -> + String key = imp.trim().startsWith('import static ') ? 'STATIC' : 'THIRD_PARTY_PACKAGE' + groups[key] << imp.trim() + } + groupOrder.each { g -> if (alphabetical) { groups[g] = groups[g].sort() } } + + List rebuilt = [] + groupOrder.eachWithIndex { g, idx -> + if (groups[g].isEmpty()) { + return + } + if (idx > 0 && separateGroups && !rebuilt.isEmpty()) { + rebuilt << '' + } + rebuilt.addAll(groups[g]) + } + + if (rebuilt == lines[start..end]) { + return lines + } + List result = new ArrayList<>(lines.subList(0, start)) + result.addAll(rebuilt) + result.addAll(lines.subList(end + 1, lines.size())) + result +} + +/* + * Checkstyle module: AvoidStarImport. + * Replaces each `import pkg.*;` / `import static pkg.Type.*;` by the explicit imports the file + * actually uses, worked out from the module's classpath (project.testClasspathElements plus the + * JDK's jrt:/ image): a type name for `pkg.*`, a public static method/field/member type name for + * `pkg.Type.*`. A name is imported only if it is used in code (not a comment or string), is not + * qualified (`x.Name`), is not already imported and, for types, is not declared in the file or in + * its own package (those shadow a star import). A star import whose package or type can't be + * resolved is left as is; one that supplies nothing the file uses is dropped. + */ +List classpathElements = ((project.testClasspathElements ?: project.compileClasspathElements ?: []) as List) +ClassLoader classpathLoader = new URLClassLoader( + classpathElements.collect { new File(it).toURI().toURL() } as URL[], ClassLoader.platformClassLoader) +Map> packageClassCache = [:] +def classesInPackage = { String pkg -> + if (packageClassCache.containsKey(pkg)) { + return packageClassCache[pkg] + } + Set names = [] as Set + String dir = pkg.replace('.', '/') + classpathElements.each { String el -> + File ef = new File(el) + if (ef.isDirectory()) { + new File(ef, dir).listFiles()?.each { File c -> + if (c.name.endsWith('.class') && !c.name.contains('$')) { + names << c.name.substring(0, c.name.length() - 6) + } + } + } else if (ef.isFile()) { + new java.util.zip.ZipFile(ef).withCloseable { zf -> + zf.entries().each { e -> + String n = e.name + if (n.startsWith(dir + '/') && n.endsWith('.class') && n.indexOf('/', dir.length() + 1) < 0 + && !n.contains('$')) { + names << n.substring(dir.length() + 1, n.length() - 6) + } + } + } + } + } + def jrt = java.nio.file.FileSystems.getFileSystem(java.net.URI.create('jrt:/')) + def jrtPackage = jrt.getPath('/packages', pkg) + if (java.nio.file.Files.exists(jrtPackage)) { + java.nio.file.Files.list(jrtPackage).withCloseable { mods -> + mods.each { mod -> + def d = jrt.getPath('/modules', mod.fileName.toString(), dir) + if (java.nio.file.Files.isDirectory(d)) { + java.nio.file.Files.list(d).withCloseable { cs -> + cs.each { c -> + String n = c.fileName.toString() + if (n.endsWith('.class') && !n.contains('$')) { + names << n.substring(0, n.length() - 6) + } + } + } + } + } + } + } + names.removeAll { !Character.isJavaIdentifierStart(it.charAt(0)) || it.contains('-') } + packageClassCache[pkg] = names + names +} + +// public static members of a type, by kind; null if the type can't be loaded +def staticMembersOf = { String typeName -> + try { + Class c = Class.forName(typeName, false, classpathLoader) + [methods: c.methods.findAll { java.lang.reflect.Modifier.isStatic(it.modifiers) }*.name as Set, + fields : c.fields.findAll { java.lang.reflect.Modifier.isStatic(it.modifiers) }*.name as Set, + types : c.classes.findAll { java.lang.reflect.Modifier.isStatic(it.modifiers) }*.simpleName as Set] + } catch (Throwable ex) { + null + } +} + +int starImportFilesFixed = 0 +def expandStarImports = { File file -> + List lines = file.readLines('UTF-8') + if (!lines.any { it =~ /^import (static )?[\w.]+\.\*;\s*$/ }) { + return + } + String text = lines.join('\n') + '\n' + Src src = new Src(text) + int bodyStart = 0 + lines.eachWithIndex { String l, int i -> if (l.startsWith('import ')) { bodyStart = lines[0..i].join('\n').length() + 1 } } + String pkgName = (lines.find { it =~ /^package [\w.]+;/ } =~ /^package ([\w.]+);/)[0][1] + Set explicitTypes = lines.findAll { it =~ /^import [\w.]+;/ }.collect { it.substring(it.lastIndexOf('.') + 1, it.length() - 1) } as Set + Set explicitStatics = lines.findAll { it =~ /^import static [\w.]+;/ }.collect { it.substring(it.lastIndexOf('.') + 1, it.length() - 1) } as Set + // identifier -> is directly followed by '(' + Map used = [:] + Set usedAsCall = [] as Set + int n = src.length() + int i = bodyStart + while (i < n) { + if (src.isCode(i) && Character.isJavaIdentifierStart(src.charAt(i)) && (i == 0 || !Character.isJavaIdentifierPart(src.charAt(i - 1)))) { + int e = i + while (e < n && Character.isJavaIdentifierPart(src.charAt(e))) { + e++ + } + int before = i - 1 + while (before >= 0 && Character.isWhitespace(src.charAt(before))) { + before-- + } + boolean qualified = before >= 0 && src.charAt(before) == '.' as char + if (!qualified) { + String name = src.slice(i, e) + used[name] = true + int after = e + while (after < n && Character.isWhitespace(src.charAt(after))) { + after++ + } + if (after < n && src.charAt(after) == '(' as char) { + usedAsCall << name + } + } + i = e + } else { + i++ + } + } + Set declaredHere = [] as Set + (text =~ /\b(?:class|interface|enum|record)\s+(\w+)/).each { declaredHere << it[1] } + Set ownPackage = classesInPackage(pkgName) + Set claimedTypes = [] as Set + Set claimedStatics = [] as Set + boolean changed = false + List out = [] + lines.each { String l -> + Matcher m = l =~ /^import (static )?([\w.]+)\.\*;\s*$/ + if (!m.matches()) { + out << l + return + } + boolean isStatic = m.group(1) != null + String target = m.group(2) + List repl = [] + if (!isStatic) { + Set available = classesInPackage(target) + if (available.isEmpty()) { + out << l + return + } + available.sort().each { String name -> + if (used.containsKey(name) && !explicitTypes.contains(name) && !declaredHere.contains(name) + && !ownPackage.contains(name) && claimedTypes.add(name)) { + repl << "import ${target}.${name};".toString() + } + } + } else { + def members = staticMembersOf(target) + if (members == null) { + out << l + return + } + Set names = [] as Set + members.methods.each { if (usedAsCall.contains(it)) { names << it } } + members.fields.each { if (used.containsKey(it) && !usedAsCall.contains(it)) { names << it } } + members.types.each { if (used.containsKey(it) && !declaredHere.contains(it)) { names << it } } + names.sort().each { String name -> + if (!explicitStatics.contains(name) && claimedStatics.add(name)) { + repl << "import static ${target}.${name};".toString() + } + } + } + changed = true + out.addAll(repl) + } + if (changed) { + file.setText(out.join('\n') + '\n', 'UTF-8') + starImportFilesFixed++ + } +} + +// Import ordering re-emits a whole block, which isn't a per-violation edit, so it runs on its +// own at the top of every round (a wrapped import only becomes orderable once NoLineWrap joined it). +int importFilesFixed = 0 +def orderImports = { File file -> + List lines = file.readLines('UTF-8') + List reordered = fixCustomImportOrder(lines) + if (reordered != lines) { + file.setText(reordered.join('\n') + '\n', 'UTF-8') + importFilesFixed++ + } +} + +// Stage 2: audit with Checkstyle, fix what each reported violation points at, repeat. +Map fixers = [ + FileTabCharacter : Fixers.&fileTab, + EmptyLineSeparator : Fixers.&emptyLineSeparator, + NoLineWrap : Fixers.&noLineWrap, + LeftCurly : Fixers.&leftCurly, + RightCurly : Fixers.&rightCurly, + NeedBraces : Fixers.&needBraces, + OneStatementPerLine : Fixers.&oneStatementPerLine, + MultipleVariableDeclarations : Fixers.&multipleVariableDeclarations, + AnnotationLocation : Fixers.&annotationLocation, + Indentation : Fixers.&indentation, + LineLength : Fixers.&lineLength, + WhitespaceAround : Fixers.&whitespaceAround, + WhitespaceAfter : Fixers.&whitespaceAfter, + NoWhitespaceBefore : Fixers.&noWhitespaceBefore, + NoWhitespaceBeforeCaseDefaultColon: Fixers.&noWhitespaceBefore, + MethodParamPad : Fixers.&methodParamPad, + ParenPad : Fixers.&parenPad, + GenericWhitespace : Fixers.&genericWhitespace, + RegexpSinglelineJava : Fixers.&emptyBlockSpacing, + MatchXpath : Fixers.&commentSpace, + OperatorWrap : Fixers.&operatorWrap, + SeparatorWrap : Fixers.&separatorWrap, + ModifierOrder : Fixers.&modifierOrder, + OverloadMethodsDeclarationOrder : Fixers.&overloadOrder, + ConstructorsDeclarationGrouping : Fixers.&constructorGrouping, + ArrayTypeStyle : Fixers.&arrayTypeStyle, + UpperEll : Fixers.&upperEll, + IllegalTokenText : Fixers.&illegalTokenText, + AvoidEscapedUnicodeCharacters : Fixers.&avoidEscapedUnicode, + TodoComment : Fixers.&todoComment, + CommentsIndentation : Fixers.&commentsIndentation, + MissingSwitchDefault : Fixers.&missingSwitchDefault, + JavadocLeadingAsteriskAlign : Fixers.&javadocAsteriskAlign, + JavadocMissingLeadingAsterisk : Fixers.&javadocMissingAsterisk, + JavadocContentLocation : Fixers.&javadocContentLocation, + JavadocParagraph : Fixers.&javadocParagraph, + JavadocTagContinuationIndentation : Fixers.&javadocTagContinuation, + RequireEmptyLineBeforeBlockTagGroup: Fixers.&requireEmptyLineBeforeTags, + AtclauseOrder : Fixers.&atclauseOrder, + SummaryJavadoc : Fixers.&summaryJavadoc, + InvalidJavadocPosition : Fixers.&invalidJavadocPosition, +] + +final int MAX_ROUNDS = 25 + +Map> violations = [:] +Set unparsable = [] as Set + +Properties checkstyleProps = new Properties() +checkstyleProps.setProperty('org.checkstyle.google.severity', 'error') +def config = ConfigurationLoader.loadConfiguration(rulesetFile.absolutePath, new PropertiesExpander(checkstyleProps)) +def auditor = new Checker() +auditor.setModuleClassLoader(Checker.classLoader) +// columns then equal char offsets + 1 even if a tab slips through +auditor.setTabWidth(1) +auditor.configure(config) +auditor.addListener([ + auditStarted : { AuditEvent e -> }, + auditFinished: { AuditEvent e -> }, + fileStarted : { AuditEvent e -> }, + fileFinished : { AuditEvent e -> }, + addError : { AuditEvent e -> + String key = e.sourceName.substring(e.sourceName.lastIndexOf('.') + 1).replaceFirst(/Check$/, '') + violations.get(e.fileName, []) << [line: e.line, col: e.column, msg: e.message, check: key] + }, + addException : { AuditEvent e, Throwable t -> unparsable << e.fileName }, +] as AuditListener) + +// One file per call: Checkstyle throws out of process() on a file that doesn't parse. +def audit = { List files -> + violations.clear() + unparsable.clear() + files.each { File f -> + try { + auditor.process([f]) + } catch (Exception ex) { + unparsable << f.absolutePath + } + } +} + +Map originalText = [:] +Map beforeRound = [:] +Map initialCount = [:] +Map latestCount = [:] +Map appliedByCheck = [:].withDefault { 0 } +Set frozen = [] as Set +Map fixerErrors = [:] + +// skip CRLF files: the fixers write '\n' +List pending = javaFiles.findAll { File f -> !f.getText('UTF-8').contains('\r') } +int round = 0 +while (!pending.isEmpty() && round < MAX_ROUNDS) { + round++ + pending.each { File f -> originalText.putIfAbsent(f, f.getText('UTF-8')) } + pending.each(expandStarImports) + pending.each(orderImports) + audit(pending) + List next = [] + pending.each { File f -> + List vs = violations[f.absolutePath] ?: [] + latestCount[f] = vs.size() + if (!initialCount.containsKey(f)) { + initialCount[f] = vs.size() + } + if (unparsable.contains(f.absolutePath)) { + if (beforeRound.containsKey(f)) { + f.setText(beforeRound[f], 'UTF-8') + frozen << f + println "[checkstyle-autofix] ${f}: a fix left the file unparsable; reverted the last round." + } + return + } + String text = f.getText('UTF-8') + List fixable = vs.findAll { fixers.containsKey(it.check) } + if (fixable.isEmpty()) { + return + } + int[] lineStarts = lineStartsOf(text) + Src src = new Src(text) + List work = fixable.collect { Map v -> + v + [off: v.line >= 1 && v.line <= lineStarts.length ? lineStarts[v.line - 1] + Math.max(0, v.col - 1) : -1] + }.findAll { it.off >= 0 } + // last-to-first; on ties handle the token's trailing side first + work.sort { a, b -> b.off <=> a.off } + int lowest = Integer.MAX_VALUE + work.each { Map v -> + if (v.off >= lowest) { + return + } + int touched + try { + touched = (fixers[v.check].call(src, v.off, v.msg, lineStarts) as int) + } catch (Exception ex) { + fixerErrors[v.check] = "${ex.class.simpleName}: ${ex.message}".toString() + touched = Fixers.NONE + } + if (touched >= 0) { + lowest = Math.min(lowest, touched) + appliedByCheck[v.check]++ + } + } + String updated = src.toString() + if (updated != text) { + beforeRound[f] = text + f.setText(updated, 'UTF-8') + next << f + } + } + pending = next.findAll { !frozen.contains(it) } +} +if (!pending.isEmpty()) { + audit(pending) + pending.each { File f -> latestCount[f] = (violations[f.absolutePath] ?: []).size() } +} + +// A file that ended with more violations than it started with is put back as it was. +int filesChanged = 0 +originalText.each { File f, String original -> + if (f.getText('UTF-8') == original) { + return + } + if (latestCount[f] != null && initialCount[f] != null && latestCount[f] > initialCount[f]) { + f.setText(original, 'UTF-8') + println "[checkstyle-autofix] ${f}: fixing made it worse (${initialCount[f]} -> ${latestCount[f]} violations); restored." + } else { + filesChanged++ + } +} +auditor.destroy() + +println "[checkstyle-autofix] ${project.artifactId}: ${importFilesFixed} import block(s) reordered, ${starImportFilesFixed} file(s) with star imports expanded; " + + "fixes applied ${appliedByCheck.sort()} over ${round} round(s), ${filesChanged} file(s) changed, " + + "ruleset=storm_checkstyle.xml." + (fixerErrors ? " Fixer errors (bug in the fixer, violation left as is): ${fixerErrors}" : '') + +static int[] lineStartsOf(String text) { + List starts = [0] + int i = text.indexOf('\n') + while (i >= 0) { + starts << i + 1 + i = text.indexOf('\n', i + 1) + } + starts as int[] +} diff --git a/storm-client/pom.xml b/storm-client/pom.xml index cfa56683086..a97f712da1a 100644 --- a/storm-client/pom.xml +++ b/storm-client/pom.xml @@ -199,6 +199,16 @@ ${project.build.directory}/test-reports
    + + org.openrewrite.maven + rewrite-maven-plugin + + + + org.codehaus.gmavenplus + gmavenplus-plugin + + org.apache.maven.plugins maven-checkstyle-plugin diff --git a/storm-client/src/jvm/org/apache/storm/Config.java b/storm-client/src/jvm/org/apache/storm/Config.java index 519b8d943bb..8d6dc65d110 100644 --- a/storm-client/src/jvm/org/apache/storm/Config.java +++ b/storm-client/src/jvm/org/apache/storm/Config.java @@ -31,14 +31,13 @@ import org.apache.storm.serialization.IKryoFactory; import org.apache.storm.utils.ShellLogHandler; import org.apache.storm.utils.Utils; -import org.apache.storm.validation.ConfigValidation; import org.apache.storm.validation.ConfigValidation.EventLoggerRegistryValidator; import org.apache.storm.validation.ConfigValidation.ListOfListOfStringValidator; import org.apache.storm.validation.ConfigValidation.MapOfStringToMapOfStringToObjectValidator; import org.apache.storm.validation.ConfigValidation.MetricRegistryValidator; import org.apache.storm.validation.ConfigValidation.MetricReportersValidator; import org.apache.storm.validation.ConfigValidation.RasConstraintsTypeValidator; -import org.apache.storm.validation.ConfigValidationAnnotations; +import org.apache.storm.validation.ConfigValidation; import org.apache.storm.validation.ConfigValidationAnnotations.CustomComboValidator; import org.apache.storm.validation.ConfigValidationAnnotations.CustomValidator; import org.apache.storm.validation.ConfigValidationAnnotations.IsBoolean; @@ -58,26 +57,34 @@ import org.apache.storm.validation.ConfigValidationAnnotations.IsType; import org.apache.storm.validation.ConfigValidationAnnotations.NotNull; import org.apache.storm.validation.ConfigValidationAnnotations.Password; +import org.apache.storm.validation.ConfigValidationAnnotations; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * Topology configs are specified as a plain old map. This class provides a convenient way to create a topology config map by providing - * setter methods for all the configs that can be set. It also makes it easier to do things like add serializations. + * Topology configs are specified as a plain old map. This class provides a convenient way to create + * a topology config map by providing + * setter methods for all the configs that can be set. It also makes it easier to do things like add + * serializations. * - *

    This class also provides constants for all the configurations possible on a Storm cluster and Storm topology. Each constant is paired - * with an annotation that defines the validity criterion of the corresponding field. Default values for these configs can be found in + *

    This class also provides constants for all the configurations possible on a Storm cluster and + * Storm topology. Each constant is paired + * with an annotation that defines the validity criterion of the corresponding field. Default values + * for these configs can be found in * defaults.yaml. * - *

    Note that you may put other configurations in any of the configs. Storm will ignore anything it doesn't recognize, but your topologies - * are free to make use of them by reading them in the prepare method of Bolts or the open method of Spouts. + *

    Note that you may put other configurations in any of the configs. Storm will ignore anything + * it doesn't recognize, but your topologies + * are free to make use of them by reading them in the prepare method of Bolts or the open method of + * Spouts. */ public class Config extends HashMap { private static final Logger LOG = LoggerFactory.getLogger(Config.class); /** - * The serializer class for ListDelegate (tuple payload). The default serializer will be ListDelegateSerializer + * The serializer class for ListDelegate (tuple payload). The default serializer will be + * ListDelegateSerializer */ @IsString public static final String TOPOLOGY_TUPLE_SERIALIZER = "topology.tuple.serializer"; @@ -86,45 +93,61 @@ public class Config extends HashMap { */ @IsBoolean @NotNull - public static final String TOPOLOGY_DISABLE_LOADAWARE_MESSAGING = "topology.disable.loadaware.messaging"; + public static final String TOPOLOGY_DISABLE_LOADAWARE_MESSAGING = + "topology.disable.loadaware.messaging"; /** - * This signifies the load congestion among target tasks in scope. Currently it's only used in LoadAwareShuffleGrouping. When the - * average load is higher than the higher bound, the executor should choose target tasks in a higher scope, The scopes and their orders + * This signifies the load congestion among target tasks in scope. Currently it's only used in + * LoadAwareShuffleGrouping. When the + * average load is higher than the higher bound, the executor should choose target tasks in a + * higher scope, The scopes and their orders * are: EVERYTHING > RACK_LOCAL > HOST_LOCAL > WORKER_LOCAL */ @IsPositiveNumber @NotNull - public static final String TOPOLOGY_LOCALITYAWARE_HIGHER_BOUND = "topology.localityaware.higher.bound"; + public static final String TOPOLOGY_LOCALITYAWARE_HIGHER_BOUND = + "topology.localityaware.higher.bound"; /** - * This signifies the load congestion among target tasks in scope. Currently it's only used in LoadAwareShuffleGrouping. When the - * average load is lower than the lower bound, the executor should choose target tasks in a lower scope. The scopes and their orders + * This signifies the load congestion among target tasks in scope. Currently it's only used in + * LoadAwareShuffleGrouping. When the + * average load is lower than the lower bound, the executor should choose target tasks in a + * lower scope. The scopes and their orders * are: EVERYTHING > RACK_LOCAL > HOST_LOCAL > WORKER_LOCAL */ @IsPositiveNumber @NotNull - public static final String TOPOLOGY_LOCALITYAWARE_LOWER_BOUND = "topology.localityaware.lower.bound"; + public static final String TOPOLOGY_LOCALITYAWARE_LOWER_BOUND = + "topology.localityaware.lower.bound"; /** - * Try to serialize all tuples, even for local transfers. This should only be used for testing, as a sanity check that all of your + * Try to serialize all tuples, even for local transfers. This should only be used for testing, + * as a sanity check that all of your * tuples are setup properly. */ @IsBoolean - public static final String TOPOLOGY_TESTING_ALWAYS_TRY_SERIALIZE = "topology.testing.always.try.serialize"; - /** - * A map with blobstore keys mapped to each filename the worker will have access to in the launch directory to the blob by local file - * name, uncompress flag, and if the worker should restart when the blob is updated. localname, workerRestart, and uncompress are - * optional. If localname is not specified the name of the key is used instead. Each topologywill have different map of blobs. Example: - * topology.blobstore.map: {"blobstorekey" : {"localname": "myblob", "uncompress": false}, "blobstorearchivekey" : {"localname": + public static final String TOPOLOGY_TESTING_ALWAYS_TRY_SERIALIZE = + "topology.testing.always.try.serialize"; + /** + * A map with blobstore keys mapped to each filename the worker will have access to in the + * launch directory to the blob by local file + * name, uncompress flag, and if the worker should restart when the blob is updated. localname, + * workerRestart, and uncompress are + * optional. If localname is not specified the name of the key is used instead. Each + * topologywill have different map of blobs. Example: + * topology.blobstore.map: {"blobstorekey" : {"localname": "myblob", "uncompress": false}, + * "blobstorearchivekey" : {"localname": * "myarchive", "uncompress": true, "workerRestart": true}} */ @CustomValidator(validatorClass = MapOfStringToMapOfStringToObjectValidator.class) public static final String TOPOLOGY_BLOBSTORE_MAP = "topology.blobstore.map"; /** - * How often a worker should check dynamic log level timeouts for expiration. For expired logger settings, the clean up polling task - * will reset the log levels to the original levels (detected at startup), and will clean up the timeout map + * How often a worker should check dynamic log level timeouts for expiration. For expired logger + * settings, the clean up polling task + * will reset the log levels to the original levels (detected at startup), and will clean up the + * timeout map */ @IsInteger @IsPositiveNumber - public static final String WORKER_LOG_LEVEL_RESET_POLL_SECS = "worker.log.level.reset.poll.secs"; + public static final String WORKER_LOG_LEVEL_RESET_POLL_SECS = + "worker.log.level.reset.poll.secs"; /** * How often a task should sync credentials, worst case. */ @@ -139,35 +162,41 @@ public class Config extends HashMap { @IsBoolean public static final String TOPOLOGY_BACKPRESSURE_ENABLE = "topology.backpressure.enable"; /** - * A list of users that are allowed to interact with the topology. To use this set nimbus.authorizer to + * A list of users that are allowed to interact with the topology. To use this set + * nimbus.authorizer to * org.apache.storm.security.auth.authorizer.SimpleACLAuthorizer */ @IsStringOrStringList public static final String TOPOLOGY_USERS = "topology.users"; /** - * A list of groups that are allowed to interact with the topology. To use this set nimbus.authorizer to + * A list of groups that are allowed to interact with the topology. To use this set + * nimbus.authorizer to * org.apache.storm.security.auth.authorizer.SimpleACLAuthorizer */ @IsStringOrStringList public static final String TOPOLOGY_GROUPS = "topology.groups"; /** - * A list of readonly users that are allowed to interact with the topology. To use this set nimbus.authorizer to + * A list of readonly users that are allowed to interact with the topology. To use this set + * nimbus.authorizer to * org.apache.storm.security.auth.authorizer.SimpleACLAuthorizer */ @IsStringOrStringList public static final String TOPOLOGY_READONLY_USERS = "topology.readonly.users"; /** - * A list of readonly groups that are allowed to interact with the topology. To use this set nimbus.authorizer to + * A list of readonly groups that are allowed to interact with the topology. To use this set + * nimbus.authorizer to * org.apache.storm.security.auth.authorizer.SimpleACLAuthorizer */ @IsStringOrStringList public static final String TOPOLOGY_READONLY_GROUPS = "topology.readonly.groups"; /** - * True if Storm should timeout messages or not. Defaults to true. This is meant to be used in unit tests to prevent tuples from being + * True if Storm should timeout messages or not. Defaults to true. This is meant to be used in + * unit tests to prevent tuples from being * accidentally timed out during the test. */ @IsBoolean - public static final String TOPOLOGY_ENABLE_MESSAGE_TIMEOUTS = "topology.enable.message.timeouts"; + public static final String TOPOLOGY_ENABLE_MESSAGE_TIMEOUTS = + "topology.enable.message.timeouts"; /** * When set to true, Storm will log every message that's emitted. */ @@ -179,8 +208,10 @@ public class Config extends HashMap { @IsString public static final String TOPOLOGY_VERSION = "topology.version"; /** - * The fully qualified name of a {@link ShellLogHandler} to handle output from non-JVM processes e.g. - * "com.mycompany.CustomShellLogHandler". If not provided, org.apache.storm.utils.DefaultLogHandler will be used. + * The fully qualified name of a {@link ShellLogHandler} to handle output from non-JVM processes + * e.g. + * "com.mycompany.CustomShellLogHandler". If not provided, + * org.apache.storm.utils.DefaultLogHandler will be used. */ @IsString public static final String TOPOLOGY_MULTILANG_LOG_HANDLER = "topology.multilang.log.handler"; @@ -190,90 +221,125 @@ public class Config extends HashMap { @IsString public static final String TOPOLOGY_MULTILANG_SERIALIZER = "topology.multilang.serializer"; /** - * How many processes should be spawned around the cluster to execute this topology. Each process will execute some number of tasks as - * threads within them. This parameter should be used in conjunction with the parallelism hints on each component in the topology to - * tune the performance of a topology. The number of workers will be dynamically calculated when the Resource Aware scheduler is used, + * How many processes should be spawned around the cluster to execute this topology. Each + * process will execute some number of tasks as + * threads within them. This parameter should be used in conjunction with the parallelism hints + * on each component in the topology to + * tune the performance of a topology. The number of workers will be dynamically calculated when + * the Resource Aware scheduler is used, * in which case this parameter will not be honored. */ @IsInteger @IsPositiveNumber public static final String TOPOLOGY_WORKERS = "topology.workers"; /** - * How many instances to create for a spout/bolt. A task runs on a thread with zero or more other tasks for the same spout/bolt. The - * number of tasks for a spout/bolt is always the same throughout the lifetime of a topology, but the number of executors (threads) for - * a spout/bolt can change over time. This allows a topology to scale to more or less resources without redeploying the topology or - * violating the constraints of Storm (such as a fields grouping guaranteeing that the same value goes to the same task). + * How many instances to create for a spout/bolt. A task runs on a thread with zero or more + * other tasks for the same spout/bolt. The + * number of tasks for a spout/bolt is always the same throughout the lifetime of a topology, + * but the number of executors (threads) for + * a spout/bolt can change over time. This allows a topology to scale to more or less resources + * without redeploying the topology or + * violating the constraints of Storm (such as a fields grouping guaranteeing that the same + * value goes to the same task). */ @IsInteger @IsPositiveNumber(includeZero = true) public static final String TOPOLOGY_TASKS = "topology.tasks"; /** - * A map of resources used by each component e.g {"cpu.pcore.percent" : 200.0. "onheap.memory.mb": 256.0, "gpu.count" : 2 } + * A map of resources used by each component e.g {"cpu.pcore.percent" : 200.0. + * "onheap.memory.mb": 256.0, "gpu.count" : 2 } */ @IsMapEntryType(keyType = String.class, valueType = Number.class) - public static final String TOPOLOGY_COMPONENT_RESOURCES_MAP = "topology.component.resources.map"; + public static final String TOPOLOGY_COMPONENT_RESOURCES_MAP = + "topology.component.resources.map"; /** - * The maximum amount of memory an instance of a spout/bolt will take on heap. This enables the scheduler to allocate slots on machines - * with enough available memory. A default value will be set for this config if user does not override + * The maximum amount of memory an instance of a spout/bolt will take on heap. This enables the + * scheduler to allocate slots on machines + * with enough available memory. A default value will be set for this config if user does not + * override */ @IsPositiveNumber(includeZero = true) - public static final String TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB = "topology.component.resources.onheap.memory.mb"; + public static final String TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB = + "topology.component.resources.onheap.memory.mb"; /** - * The maximum amount of memory an instance of a spout/bolt will take off heap. This enables the scheduler to allocate slots on machines - * with enough available memory. A default value will be set for this config if user does not override + * The maximum amount of memory an instance of a spout/bolt will take off heap. This enables the + * scheduler to allocate slots on machines + * with enough available memory. A default value will be set for this config if user does not + * override */ @IsPositiveNumber(includeZero = true) - public static final String TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB = "topology.component.resources.offheap.memory.mb"; + public static final String TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB = + "topology.component.resources.offheap.memory.mb"; /** - * The config indicates the percentage of cpu for a core an instance(executor) of a component will use. Assuming the a core value to be - * 100, a value of 10 indicates 10% of the core. The P in PCORE represents the term "physical". A default value will be set for this + * The config indicates the percentage of cpu for a core an instance(executor) of a component + * will use. Assuming the a core value to be + * 100, a value of 10 indicates 10% of the core. The P in PCORE represents the term "physical". + * A default value will be set for this * config if user does not override */ @IsPositiveNumber(includeZero = true) - public static final String TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT = "topology.component.cpu.pcore.percent"; + public static final String TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT = + "topology.component.cpu.pcore.percent"; /** - * The maximum amount of memory an instance of an acker will take on heap. This enables the scheduler to allocate slots on machines with - * enough available memory. A default value will be set for this config if user does not override + * The maximum amount of memory an instance of an acker will take on heap. This enables the + * scheduler to allocate slots on machines with + * enough available memory. A default value will be set for this config if user does not + * override */ @IsPositiveNumber(includeZero = true) - public static final String TOPOLOGY_ACKER_RESOURCES_ONHEAP_MEMORY_MB = "topology.acker.resources.onheap.memory.mb"; + public static final String TOPOLOGY_ACKER_RESOURCES_ONHEAP_MEMORY_MB = + "topology.acker.resources.onheap.memory.mb"; /** - * The maximum amount of memory an instance of an acker will take off heap. This enables the scheduler to allocate slots on machines - * with enough available memory. A default value will be set for this config if user does not override + * The maximum amount of memory an instance of an acker will take off heap. This enables the + * scheduler to allocate slots on machines + * with enough available memory. A default value will be set for this config if user does not + * override */ @IsPositiveNumber(includeZero = true) - public static final String TOPOLOGY_ACKER_RESOURCES_OFFHEAP_MEMORY_MB = "topology.acker.resources.offheap.memory.mb"; + public static final String TOPOLOGY_ACKER_RESOURCES_OFFHEAP_MEMORY_MB = + "topology.acker.resources.offheap.memory.mb"; /** - * The config indicates the percentage of cpu for a core an instance(executor) of an acker will use. Assuming the a core value to be - * 100, a value of 10 indicates 10% of the core. The P in PCORE represents the term "physical". A default value will be set for this + * The config indicates the percentage of cpu for a core an instance(executor) of an acker will + * use. Assuming the a core value to be + * 100, a value of 10 indicates 10% of the core. The P in PCORE represents the term "physical". + * A default value will be set for this * config if user does not override */ @IsPositiveNumber(includeZero = true) - public static final String TOPOLOGY_ACKER_CPU_PCORE_PERCENT = "topology.acker.cpu.pcore.percent"; + public static final String TOPOLOGY_ACKER_CPU_PCORE_PERCENT = + "topology.acker.cpu.pcore.percent"; /** - * The maximum amount of memory an instance of a metrics consumer will take on heap. This enables the scheduler to allocate slots on - * machines with enough available memory. A default value will be set for this config if user does not override + * The maximum amount of memory an instance of a metrics consumer will take on heap. This + * enables the scheduler to allocate slots on + * machines with enough available memory. A default value will be set for this config if user + * does not override */ @IsPositiveNumber(includeZero = true) public static final String TOPOLOGY_METRICS_CONSUMER_RESOURCES_ONHEAP_MEMORY_MB = "topology.metrics.consumer.resources.onheap.memory.mb"; /** - * The maximum amount of memory an instance of a metrics consumer will take off heap. This enables the scheduler to allocate slots on - * machines with enough available memory. A default value will be set for this config if user does not override + * The maximum amount of memory an instance of a metrics consumer will take off heap. This + * enables the scheduler to allocate slots on + * machines with enough available memory. A default value will be set for this config if user + * does not override */ @IsPositiveNumber(includeZero = true) public static final String TOPOLOGY_METRICS_CONSUMER_RESOURCES_OFFHEAP_MEMORY_MB = "topology.metrics.consumer.resources.offheap.memory.mb"; /** - * The config indicates the percentage of cpu for a core an instance(executor) of a metrics consumer will use. Assuming the a core value - * to be 100, a value of 10 indicates 10% of the core. The P in PCORE represents the term "physical". A default value will be set for + * The config indicates the percentage of cpu for a core an instance(executor) of a metrics + * consumer will use. Assuming the a core value + * to be 100, a value of 10 indicates 10% of the core. The P in PCORE represents the term + * "physical". A default value will be set for * this config if user does not override */ @IsPositiveNumber(includeZero = true) - public static final String TOPOLOGY_METRICS_CONSUMER_CPU_PCORE_PERCENT = "topology.metrics.consumer.cpu.pcore.percent"; + public static final String TOPOLOGY_METRICS_CONSUMER_CPU_PCORE_PERCENT = + "topology.metrics.consumer.cpu.pcore.percent"; /** - * This config allows a topology to report metrics data points from the V2 metrics API through the metrics tick. + * This config allows a topology to report metrics data points from the V2 metrics API through + * the metrics tick. */ @IsBoolean public static final String TOPOLOGY_ENABLE_V2_METRICS_TICK = "topology.enable.v2.metrics.tick"; @@ -283,123 +349,160 @@ public class Config extends HashMap { */ @IsInteger @IsPositiveNumber - public static final String TOPOLOGY_V2_METRICS_TICK_INTERVAL_SECONDS = "topology.v2.metrics.tick.interval.seconds"; + public static final String TOPOLOGY_V2_METRICS_TICK_INTERVAL_SECONDS = + "topology.v2.metrics.tick.interval.seconds"; /** * This config allows a topology to enable/disable reporting of __send-iconnection metrics. */ @IsBoolean - public static final String TOPOLOGY_ENABLE_SEND_ICONNECTION_METRICS = "topology.enable.send.iconnection.metrics"; + public static final String TOPOLOGY_ENABLE_SEND_ICONNECTION_METRICS = + "topology.enable.send.iconnection.metrics"; /** - * The class name of the {@link org.apache.storm.state.StateProvider} implementation. If not specified defaults to {@link - * org.apache.storm.state.InMemoryKeyValueStateProvider}. This can be overridden at the component level. + * The class name of the {@link org.apache.storm.state.StateProvider} implementation. If not + * specified defaults to {@link + * org.apache.storm.state.InMemoryKeyValueStateProvider}. This can be overridden at the + * component level. */ @IsString public static final String TOPOLOGY_STATE_PROVIDER = "topology.state.provider"; /** - * The configuration specific to the {@link org.apache.storm.state.StateProvider} implementation. This can be overridden at the - * component level. The value and the interpretation of this config is based on the state provider implementation. For e.g. this could + * The configuration specific to the {@link org.apache.storm.state.StateProvider} + * implementation. This can be overridden at the + * component level. The value and the interpretation of this config is based on the state + * provider implementation. For e.g. this could * be just a config file name which contains the config for the state provider implementation. */ @IsString public static final String TOPOLOGY_STATE_PROVIDER_CONFIG = "topology.state.provider.config"; /** - * Topology configuration to specify the checkpoint interval (in millis) at which the topology state is saved when {@link + * Topology configuration to specify the checkpoint interval (in millis) at which the topology + * state is saved when {@link * org.apache.storm.topology.IStatefulBolt} bolts are involved. */ @IsInteger @IsPositiveNumber - public static final String TOPOLOGY_STATE_CHECKPOINT_INTERVAL = "topology.state.checkpoint.interval.ms"; + public static final String TOPOLOGY_STATE_CHECKPOINT_INTERVAL = + "topology.state.checkpoint.interval.ms"; /** - * A per topology config that specifies the maximum amount of memory a worker can use for that specific topology. + * A per topology config that specifies the maximum amount of memory a worker can use for that + * specific topology. */ @IsPositiveNumber - public static final String TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB = "topology.worker.max.heap.size.mb"; + public static final String TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB = + "topology.worker.max.heap.size.mb"; /** * The strategy to use when scheduling a topology with Resource Aware Scheduler. */ @NotNull @IsString - //NOTE: @IsImplementationOfClass(implementsClass = IStrategy.class) is enforced in DaemonConf, so - // an error will be thrown by nimbus on topology submission and not by the client prior to submitting + // NOTE: @IsImplementationOfClass(implementsClass = IStrategy.class) is enforced in DaemonConf, + // so + // an error will be thrown by nimbus on topology submission and not by the client prior to + // submitting // the topology. public static final String TOPOLOGY_SCHEDULER_STRATEGY = "topology.scheduler.strategy"; /** - * If set to true, unassigned executors will be sorted by topological order with network proximity needs before being scheduled. - * This is a best-effort to split the topology to slices and allocate executors in each slice to closest physical location as possible. + * If set to true, unassigned executors will be sorted by topological order with network + * proximity needs before being scheduled. + * This is a best-effort to split the topology to slices and allocate executors in each slice to + * closest physical location as possible. */ - public static final String TOPOLOGY_RAS_ORDER_EXECUTORS_BY_PROXIMITY_NEEDS = "topology.ras.order.executors.by.proximity.needs"; + public static final String TOPOLOGY_RAS_ORDER_EXECUTORS_BY_PROXIMITY_NEEDS = + "topology.ras.order.executors.by.proximity.needs"; /** - * Declare scheduling constraints for a topology used by the constraint solver strategy. The format can be either - * old style (validated by ListOfListOfStringValidator.class or the newer style, which is a list of specific type of - * Maps (validated by RasConstraintsTypeValidator.class). The value must be in one or the other format. + * Declare scheduling constraints for a topology used by the constraint solver strategy. The + * format can be either + * old style (validated by ListOfListOfStringValidator.class or the newer style, which is a list + * of specific type of + * Maps (validated by RasConstraintsTypeValidator.class). The value must be in one or the other + * format. * - *

    - * Old style Config.TOPOLOGY_RAS_CONSTRAINTS (ListOfListOfString) specified a list of components that cannot + *

    Old style Config.TOPOLOGY_RAS_CONSTRAINTS (ListOfListOfString) specified a list of + * components + * that cannot * co-exist on the same Worker. *

    * - *

    - * New style Config.TOPOLOGY_RAS_CONSTRAINTS is map where each component has a list of other incompatible components - * (which serves the same function as the old style configuration) and optional number that specifies + *

    New style Config.TOPOLOGY_RAS_CONSTRAINTS is map where each component has a list of other + * incompatible components + * (which serves the same function as the old style configuration) and optional number that + * specifies * the maximum co-location count for the component on a node. *

    * - *

    comp-1 cannot exist on same worker as comp-2 or comp-3, and at most "2" comp-1 on same node

    - *

    comp-2 and comp-4 cannot be on same worker (missing comp-1 is implied from comp-1 constraint)

    + *

    comp-1 cannot exist on same worker as comp-2 or comp-3, and at most "2" comp-1 on same + * node

    * - *

    - * { "comp-1": { "maxNodeCoLocationCnt": 2, "incompatibleComponents": ["comp-2", "comp-3" ] }, + *

    comp-2 and comp-4 cannot be on same worker (missing comp-1 is implied from comp-1 + * constraint)

    + * + *

    { "comp-1": { "maxNodeCoLocationCnt": 2, "incompatibleComponents": ["comp-2", "comp-3" ] + * }, * "comp-2": { "incompatibleComponents": [ "comp-4" ] } * } *

    */ - @IsExactlyOneOf(valueValidatorClasses = {ListOfListOfStringValidator.class, RasConstraintsTypeValidator.class}) + @IsExactlyOneOf(valueValidatorClasses = {ListOfListOfStringValidator.class, + RasConstraintsTypeValidator.class}) public static final String TOPOLOGY_RAS_CONSTRAINTS = "topology.ras.constraints"; /** - * Array of components that scheduler should try to place on separate hosts when using the constraint solver strategy or the - * multi-tenant scheduler. Note that this configuration can be specified in TOPOLOGY_RAS_CONSTRAINTS using the + * Array of components that scheduler should try to place on separate hosts when using the + * constraint solver strategy or the + * multi-tenant scheduler. Note that this configuration can be specified in + * TOPOLOGY_RAS_CONSTRAINTS using the * "maxNodeCoLocationCnt" map entry with value of 1. */ @Deprecated @IsStringList public static final String TOPOLOGY_SPREAD_COMPONENTS = "topology.spread.components"; /** - * The maximum number of states that will be searched looking for a solution in resource aware strategies, e.g. + * The maximum number of states that will be searched looking for a solution in resource aware + * strategies, e.g. * in BaseResourceAwareStrategy. */ @IsInteger @IsPositiveNumber - public static final String TOPOLOGY_RAS_CONSTRAINT_MAX_STATE_SEARCH = "topology.ras.constraint.max.state.search"; + public static final String TOPOLOGY_RAS_CONSTRAINT_MAX_STATE_SEARCH = + "topology.ras.constraint.max.state.search"; /* - * Whether to limit each worker to one executor. This is useful for debugging topologies to clearly identify workers that + * Whether to limit each worker to one executor. This is useful for debugging topologies to + * clearly identify workers that * are slow/crashing and for estimating resource requirements and capacity. - * If both {@link #TOPOLOGY_RAS_ONE_EXECUTOR_PER_WORKER} and {@link #TOPOLOGY_RAS_ONE_COMPONENT_PER_WORKER} are enabled, + * If both {@link #TOPOLOGY_RAS_ONE_EXECUTOR_PER_WORKER} and {@link + * #TOPOLOGY_RAS_ONE_COMPONENT_PER_WORKER} are enabled, * {@link #TOPOLOGY_RAS_ONE_COMPONENT_PER_WORKER} is ignored. */ @IsBoolean - public static final String TOPOLOGY_RAS_ONE_EXECUTOR_PER_WORKER = "topology.ras.one.executor.per.worker"; + public static final String TOPOLOGY_RAS_ONE_EXECUTOR_PER_WORKER = + "topology.ras.one.executor.per.worker"; /** - * Whether to limit each worker to one component. This is useful for debugging topologies to clearly identify workers that + * Whether to limit each worker to one component. This is useful for debugging topologies to + * clearly identify workers that * are slow/crashing and for estimating resource requirements and capacity. - * If both TOPOLOGY_RAS_ONE_EXECUTOR_PER_WORKER and TOPOLOGY_RAS_ONE_COMPONENT_PER_WORKER are enabled, + * If both TOPOLOGY_RAS_ONE_EXECUTOR_PER_WORKER and TOPOLOGY_RAS_ONE_COMPONENT_PER_WORKER are + * enabled, * TOPOLOGY_RAS_ONE_COMPONENT_PER_WORKER is ignored. */ @IsBoolean - public static final String TOPOLOGY_RAS_ONE_COMPONENT_PER_WORKER = "topology.ras.one.component.per.worker"; + public static final String TOPOLOGY_RAS_ONE_COMPONENT_PER_WORKER = + "topology.ras.one.component.per.worker"; /** - * The maximum number of seconds to spend scheduling a topology using resource aware strategies, e.g. + * The maximum number of seconds to spend scheduling a topology using resource aware strategies, + * e.g. * in BaseResourceAwareStrategy. Null means no limit. */ @IsInteger @IsPositiveNumber - public static final String TOPOLOGY_RAS_CONSTRAINT_MAX_TIME_SECS = "topology.ras.constraint.max.time.secs"; + public static final String TOPOLOGY_RAS_CONSTRAINT_MAX_TIME_SECS = + "topology.ras.constraint.max.time.secs"; /** - * A list of host names that this topology would prefer to be scheduled on (no guarantee is given though). This is intended for + * A list of host names that this topology would prefer to be scheduled on (no guarantee is + * given though). This is intended for * debugging only. * *

    Favored nodes are moved to the front of the node selection list. @@ -408,9 +511,11 @@ public class Config extends HashMap { *

    */ @IsStringList - public static final String TOPOLOGY_SCHEDULER_FAVORED_NODES = "topology.scheduler.favored.nodes"; + public static final String TOPOLOGY_SCHEDULER_FAVORED_NODES = + "topology.scheduler.favored.nodes"; /** - * A list of host names that this topology would prefer to NOT be scheduled on (no guarantee is given though). This is intended for + * A list of host names that this topology would prefer to NOT be scheduled on (no guarantee is + * given though). This is intended for * debugging only. * *

    Unfavored nodes are moved to the end of the node selection list. @@ -419,16 +524,17 @@ public class Config extends HashMap { *

    */ @IsStringList - public static final String TOPOLOGY_SCHEDULER_UNFAVORED_NODES = "topology.scheduler.unfavored.nodes"; + public static final String TOPOLOGY_SCHEDULER_UNFAVORED_NODES = + "topology.scheduler.unfavored.nodes"; /** * How many executors to spawn for ackers. * - *

    - * 1. If not setting this variable or setting it as null, + *

    1. If not setting this variable or setting it as null, * a. If RAS is not used: * Nimbus will set it to {@link Config#TOPOLOGY_WORKERS}. * b. If RAS is used: - * Nimbus will set it to (the estimate number of workers * {@link Config#TOPOLOGY_RAS_ACKER_EXECUTORS_PER_WORKER}). + * Nimbus will set it to (the estimate number of workers * {@link + * Config#TOPOLOGY_RAS_ACKER_EXECUTORS_PER_WORKER}). * {@link Config#TOPOLOGY_RAS_ACKER_EXECUTORS_PER_WORKER} is default to be 1 if not set. * 2. If this variable is set to 0, * then Storm will immediately ack tuples as soon as they come off the spout, @@ -445,34 +551,41 @@ public class Config extends HashMap { /** * How many ackers to put in when launching a new worker until we run out of ackers. * - *

    - * This setting is RAS specific. + *

    This setting is RAS specific. * If {@link Config#TOPOLOGY_ACKER_EXECUTORS} is not configured, * this setting will be used to calculate {@link Config#TOPOLOGY_ACKER_EXECUTORS}. * * If {@link Config#TOPOLOGY_ACKER_EXECUTORS} is configured, - * nimbus will ignore this and set it as ({@link Config#TOPOLOGY_ACKER_EXECUTORS} / estimate num of workers). + * nimbus will ignore this and set it as ({@link Config#TOPOLOGY_ACKER_EXECUTORS} / estimate num + * of workers). *

    */ @IsInteger @IsPositiveNumber(includeZero = true) - public static final String TOPOLOGY_RAS_ACKER_EXECUTORS_PER_WORKER = "topology.ras.acker.executors.per.worker"; + public static final String TOPOLOGY_RAS_ACKER_EXECUTORS_PER_WORKER = + "topology.ras.acker.executors.per.worker"; /** - * A list of classes implementing IEventLogger (See storm.yaml.example for exact config format). Each listed class will be routed all - * the events sampled from emitting tuples. If there's no class provided to the option, default event logger will be initialized and + * A list of classes implementing IEventLogger (See storm.yaml.example for exact config format). + * Each listed class will be routed all + * the events sampled from emitting tuples. If there's no class provided to the option, default + * event logger will be initialized and * used unless you disable event logger executor. * - *

    Note that EventLoggerBolt takes care of all the implementations of IEventLogger, hence registering many - * implementations (especially they're implemented as 'blocking' manner) would slow down overall topology. + *

    Note that EventLoggerBolt takes care of all the implementations of IEventLogger, hence + * registering many + * implementations (especially they're implemented as 'blocking' manner) would slow down overall + * topology. */ @IsListEntryCustom(entryValidatorClasses = {EventLoggerRegistryValidator.class}) public static final String TOPOLOGY_EVENT_LOGGER_REGISTER = "topology.event.logger.register"; /** * How many executors to spawn for event logger. * - *

    By setting it as null, Storm will set the number of eventlogger executors to be equal to the number of workers - * configured for this topology (or the estimated number of workers if the Resource Aware Scheduler is used). + *

    By setting it as null, Storm will set the number of eventlogger executors to be equal to + * the number of workers + * configured for this topology (or the estimated number of workers if the Resource Aware + * Scheduler is used). * If this variable is set to 0, event logging will be disabled.

    */ @IsInteger @@ -484,17 +597,21 @@ public class Config extends HashMap { */ @IsInteger @IsPositiveNumber - public static final String TOPOLOGY_EVENTLOGGER_ROTATION_SIZE_MB = "topology.eventlogger.rotation.size.mb"; + public static final String TOPOLOGY_EVENTLOGGER_ROTATION_SIZE_MB = + "topology.eventlogger.rotation.size.mb"; /** * The maximum number of retained files for the event logger. * If not specified, a default of 5 is used. */ @IsInteger @IsPositiveNumber - public static final String TOPOLOGY_EVENTLOGGER_MAX_RETAINED_FILES = "topology.eventlogger.max.retained.files"; + public static final String TOPOLOGY_EVENTLOGGER_MAX_RETAINED_FILES = + "topology.eventlogger.max.retained.files"; /** - * The maximum amount of time given to the topology to fully process a message emitted by a spout. If the message is not acked within - * this time frame, Storm will fail the message on the spout. Some spouts implementations will then replay the message at a later time. + * The maximum amount of time given to the topology to fully process a message emitted by a + * spout. If the message is not acked within + * this time frame, Storm will fail the message on the spout. Some spouts implementations will + * then replay the message at a later time. */ @IsInteger @IsPositiveNumber @@ -502,79 +619,102 @@ public class Config extends HashMap { public static final String TOPOLOGY_MESSAGE_TIMEOUT_SECS = "topology.message.timeout.secs"; /** * A list of serialization registrations for Kryo ( https://github.com/EsotericSoftware/kryo ), the underlying serialization framework - * for Storm. A serialization can either be the name of a class (in which case Kryo will automatically create a serializer for the class - * that saves all the object's fields), or an implementation of com.esotericsoftware.kryo.Serializer. + * for Storm. A serialization can either be the name of a class (in which case Kryo will + * automatically create a serializer for the class + * that saves all the object's fields), or an implementation of + * com.esotericsoftware.kryo.Serializer. * *

    See Kryo's documentation for more information about writing custom serializers. */ @IsKryoReg public static final String TOPOLOGY_KRYO_REGISTER = "topology.kryo.register"; /** - * A list of classes that customize storm's kryo instance during start-up. Each listed class name must implement IKryoDecorator. During - * start-up the listed class is instantiated with 0 arguments, then its 'decorate' method is called with storm's kryo instance as the + * A list of classes that customize storm's kryo instance during start-up. Each listed class + * name must implement IKryoDecorator. During + * start-up the listed class is instantiated with 0 arguments, then its 'decorate' method is + * called with storm's kryo instance as the * only argument. */ @IsStringList public static final String TOPOLOGY_KRYO_DECORATORS = "topology.kryo.decorators"; /** - * Class that specifies how to create a Kryo instance for serialization. Storm will then apply topology.kryo.register and - * topology.kryo.decorators on top of this. The default implementation implements topology.fall.back.on.java.serialization and turns + * Class that specifies how to create a Kryo instance for serialization. Storm will then apply + * topology.kryo.register and + * topology.kryo.decorators on top of this. The default implementation implements + * topology.fall.back.on.java.serialization and turns * references off. */ @IsString public static final String TOPOLOGY_KRYO_FACTORY = "topology.kryo.factory"; /** - * Whether or not Storm should skip the loading of kryo registrations for which it does not know the class or have the serializer - * implementation. Otherwise, the task will fail to load and will throw an error at runtime. The use case of this is if you want to - * declare your serializations on the storm.yaml files on the cluster rather than every single time you submit a topology. Different - * applications may use different serializations and so a single application may not have the code for the other serializers used by - * other apps. By setting this config to true, Storm will ignore that it doesn't have those other serializations rather than throw an + * Whether or not Storm should skip the loading of kryo registrations for which it does not know + * the class or have the serializer + * implementation. Otherwise, the task will fail to load and will throw an error at runtime. The + * use case of this is if you want to + * declare your serializations on the storm.yaml files on the cluster rather than every single + * time you submit a topology. Different + * applications may use different serializations and so a single application may not have the + * code for the other serializers used by + * other apps. By setting this config to true, Storm will ignore that it doesn't have those + * other serializations rather than throw an * error. */ @IsBoolean - public static final String TOPOLOGY_SKIP_MISSING_KRYO_REGISTRATIONS = "topology.skip.missing.kryo.registrations"; + public static final String TOPOLOGY_SKIP_MISSING_KRYO_REGISTRATIONS = + "topology.skip.missing.kryo.registrations"; /** * List of classes to register during state serialization. */ @IsStringList public static final String TOPOLOGY_STATE_KRYO_REGISTER = "topology.state.kryo.register"; /** - * A list of classes implementing IMetricsConsumer (See storm.yaml.example for exact config format). Each listed class will be routed - * all the metrics data generated by the storm metrics API. Each listed class maps 1:1 to a system bolt named __metrics_ClassName#N, and + * A list of classes implementing IMetricsConsumer (See storm.yaml.example for exact config + * format). Each listed class will be routed + * all the metrics data generated by the storm metrics API. Each listed class maps 1:1 to a + * system bolt named __metrics_ClassName#N, and * it's parallelism is configurable. */ @IsListEntryCustom(entryValidatorClasses = {MetricRegistryValidator.class}) - public static final String TOPOLOGY_METRICS_CONSUMER_REGISTER = "topology.metrics.consumer.register"; + public static final String TOPOLOGY_METRICS_CONSUMER_REGISTER = + "topology.metrics.consumer.register"; /** - * Enable tracking of network message byte counts per source-destination task. This is off by default as it creates tasks^2 metric + * Enable tracking of network message byte counts per source-destination task. This is off by + * default as it creates tasks^2 metric * values, but is useful for debugging as it exposes data skew when tuple sizes are uneven. */ @IsBoolean - public static final String TOPOLOGY_SERIALIZED_MESSAGE_SIZE_METRICS = "topology.serialized.message.size.metrics"; + public static final String TOPOLOGY_SERIALIZED_MESSAGE_SIZE_METRICS = + "topology.serialized.message.size.metrics"; /** - * A map of metric name to class name implementing IMetric that will be created once per worker JVM. + * A map of metric name to class name implementing IMetric that will be created once per worker + * JVM. */ @IsMapEntryType(keyType = String.class, valueType = String.class) public static final String TOPOLOGY_WORKER_METRICS = "topology.worker.metrics"; /** - * A map of metric name to class name implementing IMetric that will be created once per worker JVM. + * A map of metric name to class name implementing IMetric that will be created once per worker + * JVM. */ @IsMapEntryType(keyType = String.class, valueType = String.class) public static final String WORKER_METRICS = "worker.metrics"; /** - * The maximum parallelism allowed for a component in this topology. This configuration is typically used in testing to limit the number + * The maximum parallelism allowed for a component in this topology. This configuration is + * typically used in testing to limit the number * of threads spawned in local mode. */ @IsInteger @IsPositiveNumber public static final String TOPOLOGY_MAX_TASK_PARALLELISM = "topology.max.task.parallelism"; /** - * The maximum number of tuples that can be pending on a spout task at any given time. This config applies to individual tasks, not to + * The maximum number of tuples that can be pending on a spout task at any given time. This + * config applies to individual tasks, not to * spouts or topologies as a whole. * - *

    A pending tuple is one that has been emitted from a spout but has not been acked or failed yet. Note that this - * config parameter has no effect for unreliable spouts that don't tag their tuples with a message id. + *

    A pending tuple is one that has been emitted from a spout but has not been acked or failed + * yet. Note that this + * config parameter has no effect for unreliable spouts that don't tag their tuples with a + * message id. */ @IsInteger @IsPositiveNumber @@ -584,14 +724,17 @@ public class Config extends HashMap { */ @IsInteger @IsPositiveNumber(includeZero = true) - public static final String TOPOLOGY_SLEEP_SPOUT_WAIT_STRATEGY_TIME_MS = "topology.sleep.spout.wait.strategy.time.ms"; + public static final String TOPOLOGY_SLEEP_SPOUT_WAIT_STRATEGY_TIME_MS = + "topology.sleep.spout.wait.strategy.time.ms"; /** - * The maximum amount of time a component gives a source of state to synchronize before it requests synchronization again. + * The maximum amount of time a component gives a source of state to synchronize before it + * requests synchronization again. */ @IsInteger @IsPositiveNumber @NotNull - public static final String TOPOLOGY_STATE_SYNCHRONIZATION_TIMEOUT_SECS = "topology.state.synchronization.timeout.secs"; + public static final String TOPOLOGY_STATE_SYNCHRONIZATION_TIMEOUT_SECS = + "topology.state.synchronization.timeout.secs"; /** * The percentage of tuples to sample to produce stats for a task. */ @@ -605,12 +748,14 @@ public class Config extends HashMap { @IsBoolean public static final String TOPOLOGY_STATS_EWMA_ENABLE = "topology.stats.ewma.enable"; /** - * The smoothing factor (alpha) used for exponential jitter calculation (RFC 1889 §A.8). The default value is set to 1/16. + * The smoothing factor (alpha) used for exponential jitter calculation (RFC 1889 §A.8). The + * default value is set to 1/16. * * @see RFC 1889 §A.8 */ @CustomValidator(validatorClass = ConfigValidation.ZeroOneOpenIntervalValidator.class) - public static final String TOPOLOGY_STATS_EWMA_SMOOTHING_FACTOR = "topology.stats.ewma.smoothing.factor"; + public static final String TOPOLOGY_STATS_EWMA_SMOOTHING_FACTOR = + "topology.stats.ewma.smoothing.factor"; /** * Flag to enable or disable the feedback channel for upstream communication. * When true, components can send unanchored tuples back to their source tasks. @@ -624,7 +769,8 @@ public class Config extends HashMap { */ @IsBoolean @CustomComboValidator(validatorClass = ConfigValidation.UpstreamFeedbackValidator.class) - public static final String TOPOLOGY_UPSTREAM_FEEDBACK_ENABLE = "topology.upstream.feedback.enable"; + public static final String TOPOLOGY_UPSTREAM_FEEDBACK_ENABLE = + "topology.upstream.feedback.enable"; /** * The period, in seconds, between upstream feedback messages within the topology. * @@ -645,31 +791,39 @@ public class Config extends HashMap { * *

    * - * Defaults to 10 if not explicitly configured. + *

    Defaults to 10 if not explicitly configured. */ @IsInteger @IsPositiveNumber - public static final String TOPOLOGY_UPSTREAM_FEEDBACK_FREQ_SECS = "topology.upstream.feedback.freq.secs"; + public static final String TOPOLOGY_UPSTREAM_FEEDBACK_FREQ_SECS = + "topology.upstream.feedback.freq.secs"; /** * The time period that builtin metrics data in bucketed into. */ @IsInteger - public static final String TOPOLOGY_BUILTIN_METRICS_BUCKET_SIZE_SECS = "topology.builtin.metrics.bucket.size.secs"; + public static final String TOPOLOGY_BUILTIN_METRICS_BUCKET_SIZE_SECS = + "topology.builtin.metrics.bucket.size.secs"; /** - * Whether or not to use Java serialization in a topology. Default is set false for security reasons. + * Whether or not to use Java serialization in a topology. Default is set false for security + * reasons. */ @IsBoolean - public static final String TOPOLOGY_FALL_BACK_ON_JAVA_SERIALIZATION = "topology.fall.back.on.java.serialization"; + public static final String TOPOLOGY_FALL_BACK_ON_JAVA_SERIALIZATION = + "topology.fall.back.on.java.serialization"; /** * Optional JEP-290 serial-filter pattern applied to the - * Java-serialization fallback bridge that {@link #TOPOLOGY_FALL_BACK_ON_JAVA_SERIALIZATION} enables for - * unregistered classes. When set to a non-empty pattern, it is parsed once at kryo construction and installed + * Java-serialization fallback bridge that {@link #TOPOLOGY_FALL_BACK_ON_JAVA_SERIALIZATION} + * enables for + * unregistered classes. When set to a non-empty pattern, it is parsed once at kryo construction + * and installed * on every {@code ObjectInputStream} used to deserialize fallback values, so stream classes rejected by the * filter are neither instantiated nor have their {@code readObject} logic invoked. Unset by default; * {@code docs/SECURITY.md} has a sample deny-list pattern to start from. An empty or unset value leaves the - * bridge unfiltered, as before. An invalid pattern fails topology submission and worker startup with a clear error. + * bridge unfiltered, as before. An invalid pattern fails topology submission and worker startup + * with a clear error. * Note: unlike a JVM-wide {@code jdk.serialFilter}, this is topology-scoped and also applies when the - * deserializer is built programmatically, e.g. local mode; when both are present the two filters are merged, + * deserializer is built programmatically, e.g. local mode; when both are present the two + * filters are merged, * so each takes effect. Filters only {@code DefaultKryoFactory}'s fallback path; a custom * {@code topology.kryo.factory} or the pre-kryo state serializer must arrange its own filtering. * {@code maxbytes} is per value, not per tuple, and approximate: a large primitive array is measured at @@ -679,12 +833,14 @@ public class Config extends HashMap { public static final String TOPOLOGY_FALL_BACK_ON_JAVA_SERIALIZATION_FILTER = "topology.fall.back.on.java.serialization.filter"; /** - * Topology-specific options for the worker child process. This is used in addition to WORKER_CHILDOPTS. + * Topology-specific options for the worker child process. This is used in addition to + * WORKER_CHILDOPTS. */ @IsStringOrStringList public static final String TOPOLOGY_WORKER_CHILDOPTS = "topology.worker.childopts"; /** - * Topology-specific options GC for the worker child process. This overrides WORKER_GC_CHILDOPTS. + * Topology-specific options GC for the worker child process. This overrides + * WORKER_GC_CHILDOPTS. */ @IsStringOrStringList public static final String TOPOLOGY_WORKER_GC_CHILDOPTS = "topology.worker.gc.childopts"; @@ -692,89 +848,115 @@ public class Config extends HashMap { * Topology-specific options for the logwriter process of a worker. */ @IsStringOrStringList - public static final String TOPOLOGY_WORKER_LOGWRITER_CHILDOPTS = "topology.worker.logwriter.childopts"; + public static final String TOPOLOGY_WORKER_LOGWRITER_CHILDOPTS = + "topology.worker.logwriter.childopts"; /** - * Topology-specific classpath for the worker child process. This is combined to the usual classpath. + * Topology-specific classpath for the worker child process. This is combined to the usual + * classpath. */ @IsStringOrStringList public static final String TOPOLOGY_CLASSPATH = "topology.classpath"; /** - * Topology-specific classpath for the worker child process. This will be *prepended* to the usual classpath, meaning it can override - * the Storm classpath. This is for debugging purposes, and is disabled by default. To allow topologies to be submitted with user-first + * Topology-specific classpath for the worker child process. This will be *prepended* to the + * usual classpath, meaning it can override + * the Storm classpath. This is for debugging purposes, and is disabled by default. To allow + * topologies to be submitted with user-first * classpaths, set the storm.topology.classpath.beginning.enabled config to true. */ @IsStringOrStringList public static final String TOPOLOGY_CLASSPATH_BEGINNING = "topology.classpath.beginning"; /** - * Topology-specific environment variables for the worker child process. This is added to the existing environment (that of the + * Topology-specific environment variables for the worker child process. This is added to the + * existing environment (that of the * supervisor) */ @IsMapEntryType(keyType = String.class, valueType = String.class) public static final String TOPOLOGY_ENVIRONMENT = "topology.environment"; /* - * Bolt-specific configuration for windowed bolts to specify the window length as a count of number of tuples + * Bolt-specific configuration for windowed bolts to specify the window length as a count of + * number of tuples * in the window. */ @IsInteger @IsPositiveNumber - public static final String TOPOLOGY_BOLTS_WINDOW_LENGTH_COUNT = "topology.bolts.window.length.count"; + public static final String TOPOLOGY_BOLTS_WINDOW_LENGTH_COUNT = + "topology.bolts.window.length.count"; /* * Bolt-specific configuration for windowed bolts to specify the window length in time duration. */ @IsInteger @IsPositiveNumber - public static final String TOPOLOGY_BOLTS_WINDOW_LENGTH_DURATION_MS = "topology.bolts.window.length.duration.ms"; + public static final String TOPOLOGY_BOLTS_WINDOW_LENGTH_DURATION_MS = + "topology.bolts.window.length.duration.ms"; /* - * Bolt-specific configuration for windowed bolts to specify the sliding interval as a count of number of tuples. + * Bolt-specific configuration for windowed bolts to specify the sliding interval as a count of + * number of tuples. */ @IsInteger @IsPositiveNumber - public static final String TOPOLOGY_BOLTS_SLIDING_INTERVAL_COUNT = "topology.bolts.window.sliding.interval.count"; + public static final String TOPOLOGY_BOLTS_SLIDING_INTERVAL_COUNT = + "topology.bolts.window.sliding.interval.count"; /* - * Bolt-specific configuration for windowed bolts to specify the sliding interval in time duration. + * Bolt-specific configuration for windowed bolts to specify the sliding interval in time + * duration. */ @IsInteger @IsPositiveNumber - public static final String TOPOLOGY_BOLTS_SLIDING_INTERVAL_DURATION_MS = "topology.bolts.window.sliding.interval.duration.ms"; + public static final String TOPOLOGY_BOLTS_SLIDING_INTERVAL_DURATION_MS = + "topology.bolts.window.sliding.interval.duration.ms"; /** - * Bolt-specific configuration for windowed bolts to specify the name of the stream on which late tuples are going to be emitted. This - * configuration should only be used from the BaseWindowedBolt.withLateTupleStream builder method, and not as global parameter, + * Bolt-specific configuration for windowed bolts to specify the name of the stream on which + * late tuples are going to be emitted. This + * configuration should only be used from the BaseWindowedBolt.withLateTupleStream builder + * method, and not as global parameter, * otherwise IllegalArgumentException is going to be thrown. */ @IsString - public static final String TOPOLOGY_BOLTS_LATE_TUPLE_STREAM = "topology.bolts.late.tuple.stream"; + public static final String TOPOLOGY_BOLTS_LATE_TUPLE_STREAM = + "topology.bolts.late.tuple.stream"; /** - * Bolt-specific configuration for windowed bolts to specify the maximum time lag of the tuple timestamp in milliseconds. It means that - * the tuple timestamps cannot be out of order by more than this amount. This config will be effective only if {@link + * Bolt-specific configuration for windowed bolts to specify the maximum time lag of the tuple + * timestamp in milliseconds. It means that + * the tuple timestamps cannot be out of order by more than this amount. This config will be + * effective only if {@link * org.apache.storm.windowing.TimestampExtractor} is specified. */ @IsInteger @IsPositiveNumber - public static final String TOPOLOGY_BOLTS_TUPLE_TIMESTAMP_MAX_LAG_MS = "topology.bolts.tuple.timestamp.max.lag.ms"; + public static final String TOPOLOGY_BOLTS_TUPLE_TIMESTAMP_MAX_LAG_MS = + "topology.bolts.tuple.timestamp.max.lag.ms"; /* * Bolt-specific configuration for windowed bolts to specify the time interval for generating * watermark events. Watermark event tracks the progress of time when tuple timestamp is used. - * This config is effective only if {@link org.apache.storm.windowing.TimestampExtractor} is specified. + * This config is effective only if {@link org.apache.storm.windowing.TimestampExtractor} is + * specified. */ @IsInteger @IsPositiveNumber - public static final String TOPOLOGY_BOLTS_WATERMARK_EVENT_INTERVAL_MS = "topology.bolts.watermark.event.interval.ms"; + public static final String TOPOLOGY_BOLTS_WATERMARK_EVENT_INTERVAL_MS = + "topology.bolts.watermark.event.interval.ms"; /* - * Bolt-specific configuration for windowed bolts to specify the name of the field in the tuple that holds - * the message id. This is used to track the windowing boundaries and avoid re-evaluating the windows + * Bolt-specific configuration for windowed bolts to specify the name of the field in the tuple + * that holds + * the message id. This is used to track the windowing boundaries and avoid re-evaluating the + * windows * during recovery of IStatefulWindowedBolt */ @IsString - public static final String TOPOLOGY_BOLTS_MESSAGE_ID_FIELD_NAME = "topology.bolts.message.id.field.name"; + public static final String TOPOLOGY_BOLTS_MESSAGE_ID_FIELD_NAME = + "topology.bolts.message.id.field.name"; /** - * This config is available for TransactionalSpouts, and contains the id ( a String) for the transactional topology. This id is used to + * This config is available for TransactionalSpouts, and contains the id ( a String) for the + * transactional topology. This id is used to * store the state of the transactional topology in Zookeeper. */ @IsString public static final String TOPOLOGY_TRANSACTIONAL_ID = "topology.transactional.id"; /** - * A list of task hooks that are automatically added to every spout and bolt in the topology. An example of when you'd do this is to add - * a hook that integrates with your internal monitoring system. These hooks are instantiated using the zero-arg constructor. + * A list of task hooks that are automatically added to every spout and bolt in the topology. An + * example of when you'd do this is to add + * a hook that integrates with your internal monitoring system. These hooks are instantiated + * using the zero-arg constructor. */ @IsStringList public static final String TOPOLOGY_AUTO_TASK_HOOKS = "topology.auto.task.hooks"; @@ -783,31 +965,43 @@ public class Config extends HashMap { */ @IsPositiveNumber @IsInteger - public static final String TOPOLOGY_EXECUTOR_RECEIVE_BUFFER_SIZE = "topology.executor.receive.buffer.size"; + public static final String TOPOLOGY_EXECUTOR_RECEIVE_BUFFER_SIZE = + "topology.executor.receive.buffer.size"; /** - * When enabled, each executor's receive queue gets a small, dedicated control lane that is drained before the - * data queue. Low-volume time-driven system tuples (specified here {@link Constants#SYSTEM_CONTROL_STREAM_IDS}) are routed to - * it, insulating them from data-plane buffering, backpressure and overflow so their delivery latency stays - * bounded while the data plane saturates. Control tuples are published to the lane without batching and without - * participating in backpressure; if the lane is full they are dropped (and counted), which is safe because these - * signals are periodic and the next one arrives within the signal's period. Ack and metrics payload streams are + * When enabled, each executor's receive queue gets a small, dedicated control lane that is + * drained before the + * data queue. Low-volume time-driven system tuples (specified here {@link + * Constants#SYSTEM_CONTROL_STREAM_IDS}) are routed to + * it, insulating them from data-plane buffering, backpressure and overflow so their delivery + * latency stays + * bounded while the data plane saturates. Control tuples are published to the lane without + * batching and without + * participating in backpressure; if the lane is full they are dropped (and counted), which is + * safe because these + * signals are periodic and the next one arrives within the signal's period. Ack and metrics + * payload streams are * unaffected: they stay on the data path. * *

    Note that {@code __tick} tuples travel on the control lane. When this is enabled a {@code __tick} tuple may - * occasionally be dropped under sustained saturation rather than blocking until delivered as it does with the lane - * off. Bolts that rely on tick tuples for windowing or expiry logic must therefore tolerate an occasional missed + * occasionally be dropped under sustained saturation rather than blocking until delivered as it + * does with the lane + * off. Bolts that rely on tick tuples for windowing or expiry logic must therefore tolerate an + * occasional missed * tick; it is not only internal signals that may be dropped. Default value: false. */ @IsBoolean - public static final String TOPOLOGY_EXECUTOR_RECEIVE_CONTROL_QUEUE_ENABLE = "topology.executor.receive.control.queue.enable"; + public static final String TOPOLOGY_EXECUTOR_RECEIVE_CONTROL_QUEUE_ENABLE = + "topology.executor.receive.control.queue.enable"; /** * The size of the control lane of the receive queue for each executor. Only used when {@link - * #TOPOLOGY_EXECUTOR_RECEIVE_CONTROL_QUEUE_ENABLE} is set. Control traffic is low-volume, so a small buffer + * #TOPOLOGY_EXECUTOR_RECEIVE_CONTROL_QUEUE_ENABLE} is set. Control traffic is low-volume, so a + * small buffer * suffices; will be internally rounded up to the next power of 2. Default value: 1024. */ @IsPositiveNumber @IsInteger - public static final String TOPOLOGY_EXECUTOR_RECEIVE_CONTROL_BUFFER_SIZE = "topology.executor.receive.control.buffer.size"; + public static final String TOPOLOGY_EXECUTOR_RECEIVE_CONTROL_BUFFER_SIZE = + "topology.executor.receive.control.buffer.size"; /** * The size of the transfer queue for each worker. */ @@ -821,7 +1015,8 @@ public class Config extends HashMap { @IsInteger public static final String TOPOLOGY_TRANSFER_BATCH_SIZE = "topology.transfer.batch.size"; /** - * How often a tick tuple from the "__system" component and "__tick" stream should be sent to tasks. Meant to be used as a + * How often a tick tuple from the "__system" component and "__tick" stream should be sent to + * tasks. Meant to be used as a * component-specific configuration. */ @IsInteger @@ -834,73 +1029,92 @@ public class Config extends HashMap { @NotNull public static final String TOPOLOGY_PRODUCER_BATCH_SIZE = "topology.producer.batch.size"; /** - * When enabled, the producer batch size adapts at runtime between 1 and {@link #TOPOLOGY_PRODUCER_BATCH_SIZE} using AIMD: - * it shrinks toward 1 to reduce latency under light load and grows back toward the configured size to preserve throughput - * under heavy load. Has no effect unless {@link #TOPOLOGY_PRODUCER_BATCH_SIZE} is greater than 1. + * When enabled, the producer batch size adapts at runtime between 1 and {@link + * #TOPOLOGY_PRODUCER_BATCH_SIZE} using AIMD: + * it shrinks toward 1 to reduce latency under light load and grows back toward the configured + * size to preserve throughput + * under heavy load. Has no effect unless {@link #TOPOLOGY_PRODUCER_BATCH_SIZE} is greater than + * 1. */ @IsBoolean public static final String TOPOLOGY_PRODUCER_BATCH_DYNAMIC = "topology.producer.batch.dynamic"; /** - * If number of items in task's overflowQ exceeds this, new messages coming from other workers to this task will be dropped This - * prevents OutOfMemoryException that can occur in rare scenarios in the presence of BackPressure. This affects only inter-worker + * If number of items in task's overflowQ exceeds this, new messages coming from other workers + * to this task will be dropped This + * prevents OutOfMemoryException that can occur in rare scenarios in the presence of + * BackPressure. This affects only inter-worker * messages. Messages originating from within the same worker will not be dropped. */ @IsInteger @IsPositiveNumber(includeZero = true) @NotNull - public static final String TOPOLOGY_EXECUTOR_OVERFLOW_LIMIT = "topology.executor.overflow.limit"; + public static final String TOPOLOGY_EXECUTOR_OVERFLOW_LIMIT = + "topology.executor.overflow.limit"; /** - * How often a worker should check and notify upstream workers about its tasks that are no longer experiencing BP and able to receive + * How often a worker should check and notify upstream workers about its tasks that are no + * longer experiencing BP and able to receive * new messages. */ @IsInteger @IsPositiveNumber @NotNull - public static final String TOPOLOGY_BACKPRESSURE_CHECK_MILLIS = "topology.backpressure.check.millis"; + public static final String TOPOLOGY_BACKPRESSURE_CHECK_MILLIS = + "topology.backpressure.check.millis"; /** * How often to send flush tuple to the executors for flushing out batched events. */ @IsInteger @IsPositiveNumber(includeZero = true) @NotNull - public static final String TOPOLOGY_BATCH_FLUSH_INTERVAL_MILLIS = "topology.batch.flush.interval.millis"; + public static final String TOPOLOGY_BATCH_FLUSH_INTERVAL_MILLIS = + "topology.batch.flush.interval.millis"; /** - * The size of the shared thread pool for worker tasks to make use of. The thread pool can be accessed via the TopologyContext. + * The size of the shared thread pool for worker tasks to make use of. The thread pool can be + * accessed via the TopologyContext. */ @IsInteger - public static final String TOPOLOGY_WORKER_SHARED_THREAD_POOL_SIZE = "topology.worker.shared.thread.pool.size"; + public static final String TOPOLOGY_WORKER_SHARED_THREAD_POOL_SIZE = + "topology.worker.shared.thread.pool.size"; /** - * The interval in seconds to use for determining whether to throttle error reported to Zookeeper. For example, an interval of 10 - * seconds with topology.max.error.report.per.interval set to 5 will only allow 5 errors to be reported to Zookeeper per task for every + * The interval in seconds to use for determining whether to throttle error reported to + * Zookeeper. For example, an interval of 10 + * seconds with topology.max.error.report.per.interval set to 5 will only allow 5 errors to be + * reported to Zookeeper per task for every * 10 second interval of time. */ @IsInteger - public static final String TOPOLOGY_ERROR_THROTTLE_INTERVAL_SECS = "topology.error.throttle.interval.secs"; + public static final String TOPOLOGY_ERROR_THROTTLE_INTERVAL_SECS = + "topology.error.throttle.interval.secs"; /** * See doc for {@link #TOPOLOGY_ERROR_THROTTLE_INTERVAL_SECS}. */ @IsInteger @IsPositiveNumber - public static final String TOPOLOGY_MAX_ERROR_REPORT_PER_INTERVAL = "topology.max.error.report.per.interval"; + public static final String TOPOLOGY_MAX_ERROR_REPORT_PER_INTERVAL = + "topology.max.error.report.per.interval"; /** * How often a batch can be emitted in a Trident topology. */ @IsInteger @IsPositiveNumber - public static final String TOPOLOGY_TRIDENT_BATCH_EMIT_INTERVAL_MILLIS = "topology.trident.batch.emit.interval.millis"; + public static final String TOPOLOGY_TRIDENT_BATCH_EMIT_INTERVAL_MILLIS = + "topology.trident.batch.emit.interval.millis"; /** - * Maximum number of tuples that can be stored inmemory cache in windowing operators for fast access without fetching them from store. + * Maximum number of tuples that can be stored inmemory cache in windowing operators for fast + * access without fetching them from store. */ @IsInteger @IsPositiveNumber - public static final String TOPOLOGY_TRIDENT_WINDOWING_INMEMORY_CACHE_LIMIT = "topology.trident.windowing.cache.tuple.limit"; + public static final String TOPOLOGY_TRIDENT_WINDOWING_INMEMORY_CACHE_LIMIT = + "topology.trident.windowing.cache.tuple.limit"; /** * The id assigned to a running topology. The id is the storm name with a unique nonce appended. */ @IsString public static final String STORM_ID = "storm.id"; /** - * Name of the topology. This config is automatically set by Storm when the topology is submitted. + * Name of the topology. This config is automatically set by Storm when the topology is + * submitted. */ @IsString public static final String TOPOLOGY_NAME = "topology.name"; @@ -927,13 +1141,16 @@ public class Config extends HashMap { @IsPositiveNumber public static final String TOPOLOGY_SHELLBOLT_MAX_PENDING = "topology.shellbolt.max.pending"; /** - * How long a subprocess can go without heartbeating before the ShellSpout/ShellBolt tries to suicide itself. + * How long a subprocess can go without heartbeating before the ShellSpout/ShellBolt tries to + * suicide itself. */ @IsInteger @IsPositiveNumber - public static final String TOPOLOGY_SUBPROCESS_TIMEOUT_SECS = "topology.subprocess.timeout.secs"; + public static final String TOPOLOGY_SUBPROCESS_TIMEOUT_SECS = + "topology.subprocess.timeout.secs"; /** - * Topology central logging sensitivity to determine who has access to logs in central logging system. The possible values are: S0 - + * Topology central logging sensitivity to determine who has access to logs in central logging + * system. The possible values are: S0 - * Public (open to all users on grid) S1 - Restricted S2 - Confidential S3 - Secret (default.) */ @IsString(acceptedValues = {"S0", "S1", "S2", "S3"}) @@ -960,12 +1177,14 @@ public class Config extends HashMap { @IsString public static final String TRANSACTIONAL_ZOOKEEPER_ROOT = "transactional.zookeeper.root"; /** - * The list of zookeeper servers in which to keep the transactional state. If null (which is default), will use storm.zookeeper.servers + * The list of zookeeper servers in which to keep the transactional state. If null (which is + * default), will use storm.zookeeper.servers */ @IsStringList public static final String TRANSACTIONAL_ZOOKEEPER_SERVERS = "transactional.zookeeper.servers"; /** - * The port to use to connect to the transactional zookeeper servers. If null (which is default), will use storm.zookeeper.port + * The port to use to connect to the transactional zookeeper servers. If null (which is + * default), will use storm.zookeeper.port */ @IsInteger @IsPositiveNumber @@ -979,9 +1198,12 @@ public class Config extends HashMap { * The maximum number of machines that should be used by this topology. This configuration can * be used to isolate topologies from each other. See {@code org.apache.storm.scheduler.multitenant.MultitenantScheduler}. * Round Robin Strategy uses this value to avoid spreading a topology too - * thinly over a large number of machines - avoiding the extreme case where the topology would be spread over - * all workers and thus deny scheduling of other topologies. Round Robin scheduling will occupy all the workers on - * this limited number of machines, forcing other topologies to be scheduled on other machines; thus isolating the + * thinly over a large number of machines - avoiding the extreme case where the topology would + * be spread over + * all workers and thus deny scheduling of other topologies. Round Robin scheduling will occupy + * all the workers on + * this limited number of machines, forcing other topologies to be scheduled on other machines; + * thus isolating the * topology from other topologies. * Set {@code storm.scheduler} to {@code org.apache.storm.scheduler.multitenant.MultitenantScheduler} * Alternatively set {@code storm.scheduler} to {@code org.apache.storm.scheduler.resource.ResourceAwareScheduler} @@ -993,42 +1215,52 @@ public class Config extends HashMap { @IsPositiveNumber public static final String TOPOLOGY_ISOLATED_MACHINES = "topology.isolate.machines"; /** - * A class that implements a wait strategy for spout. Waiting is triggered in one of two conditions: + * A class that implements a wait strategy for spout. Waiting is triggered in one of two + * conditions: * - *

    1. nextTuple emits no tuples 2. The spout has hit maxSpoutPending and can't emit any more tuples + *

    1. nextTuple emits no tuples 2. The spout has hit maxSpoutPending and can't emit any more + * tuples * *

    This class must implement {@link IWaitStrategy}. */ @IsString public static final String TOPOLOGY_SPOUT_WAIT_STRATEGY = "topology.spout.wait.strategy"; /** - * Configures park time for WaitStrategyPark for spout. If set to 0, returns immediately (i.e busy wait). + * Configures park time for WaitStrategyPark for spout. If set to 0, returns immediately (i.e + * busy wait). */ @NotNull @IsPositiveNumber(includeZero = true) - public static final String TOPOLOGY_SPOUT_WAIT_PARK_MICROSEC = "topology.spout.wait.park.microsec"; + public static final String TOPOLOGY_SPOUT_WAIT_PARK_MICROSEC = + "topology.spout.wait.park.microsec"; /** - * Configures number of iterations to spend in level 1 of WaitStrategyProgressive, before progressing to level 2. + * Configures number of iterations to spend in level 1 of WaitStrategyProgressive, before + * progressing to level 2. */ @NotNull @IsInteger @IsPositiveNumber(includeZero = true) - public static final String TOPOLOGY_SPOUT_WAIT_PROGRESSIVE_LEVEL1_COUNT = "topology.spout.wait.progressive.level1.count"; + public static final String TOPOLOGY_SPOUT_WAIT_PROGRESSIVE_LEVEL1_COUNT = + "topology.spout.wait.progressive.level1.count"; /** - * Configures number of iterations to spend in level 2 of WaitStrategyProgressive, before progressing to level 3. + * Configures number of iterations to spend in level 2 of WaitStrategyProgressive, before + * progressing to level 3. */ @NotNull @IsInteger @IsPositiveNumber(includeZero = true) - public static final String TOPOLOGY_SPOUT_WAIT_PROGRESSIVE_LEVEL2_COUNT = "topology.spout.wait.progressive.level2.count"; + public static final String TOPOLOGY_SPOUT_WAIT_PROGRESSIVE_LEVEL2_COUNT = + "topology.spout.wait.progressive.level2.count"; /** * Configures sleep time for WaitStrategyProgressive. */ @NotNull @IsPositiveNumber(includeZero = true) - public static final String TOPOLOGY_SPOUT_WAIT_PROGRESSIVE_LEVEL3_SLEEP_MILLIS = "topology.spout.wait.progressive.level3.sleep.millis"; + public static final String TOPOLOGY_SPOUT_WAIT_PROGRESSIVE_LEVEL3_SLEEP_MILLIS = + "topology.spout.wait.progressive.level3.sleep.millis"; /** - * Selects the Bolt's Wait Strategy to use when there are no incoming msgs. Used to trade off latency vs CPU usage. This class must + * Selects the Bolt's Wait Strategy to use when there are no incoming msgs. Used to trade off + * latency vs CPU usage. This class must * implement {@link IWaitStrategy}. */ @IsString @@ -1038,43 +1270,54 @@ public class Config extends HashMap { */ @NotNull @IsPositiveNumber(includeZero = true) - public static final String TOPOLOGY_BOLT_WAIT_PARK_MICROSEC = "topology.bolt.wait.park.microsec"; + public static final String TOPOLOGY_BOLT_WAIT_PARK_MICROSEC = + "topology.bolt.wait.park.microsec"; /** - * Configures number of iterations to spend in level 1 of WaitStrategyProgressive, before progressing to level 2. + * Configures number of iterations to spend in level 1 of WaitStrategyProgressive, before + * progressing to level 2. */ @NotNull @IsInteger @IsPositiveNumber(includeZero = true) - public static final String TOPOLOGY_BOLT_WAIT_PROGRESSIVE_LEVEL1_COUNT = "topology.bolt.wait.progressive.level1.count"; + public static final String TOPOLOGY_BOLT_WAIT_PROGRESSIVE_LEVEL1_COUNT = + "topology.bolt.wait.progressive.level1.count"; /** - * Configures number of iterations to spend in level 2 of WaitStrategyProgressive, before progressing to level 3. + * Configures number of iterations to spend in level 2 of WaitStrategyProgressive, before + * progressing to level 3. */ @NotNull @IsInteger @IsPositiveNumber(includeZero = true) - public static final String TOPOLOGY_BOLT_WAIT_PROGRESSIVE_LEVEL2_COUNT = "topology.bolt.wait.progressive.level2.count"; + public static final String TOPOLOGY_BOLT_WAIT_PROGRESSIVE_LEVEL2_COUNT = + "topology.bolt.wait.progressive.level2.count"; /** * Configures sleep time for WaitStrategyProgressive. */ @NotNull @IsPositiveNumber(includeZero = true) - public static final String TOPOLOGY_BOLT_WAIT_PROGRESSIVE_LEVEL3_SLEEP_MILLIS = "topology.bolt.wait.progressive.level3.sleep.millis"; + public static final String TOPOLOGY_BOLT_WAIT_PROGRESSIVE_LEVEL3_SLEEP_MILLIS = + "topology.bolt.wait.progressive.level3.sleep.millis"; /** - * A class that implements a wait strategy for an upstream component (spout/bolt) trying to write to a downstream component whose recv - * queue is full + * A class that implements a wait strategy for an upstream component (spout/bolt) trying to + * write to a downstream component whose recv + * queue is full. * - *

    1. nextTuple emits no tuples 2. The spout has hit maxSpoutPending and can't emit any more tuples + *

    1. nextTuple emits no tuples 2. The spout has hit maxSpoutPending and can't emit any more + * tuples * *

    This class must implement {@link IWaitStrategy}. */ @IsString - public static final String TOPOLOGY_BACKPRESSURE_WAIT_STRATEGY = "topology.backpressure.wait.strategy"; + public static final String TOPOLOGY_BACKPRESSURE_WAIT_STRATEGY = + "topology.backpressure.wait.strategy"; /** - * Configures park time if using WaitStrategyPark for BackPressure. If set to 0, returns immediately (i.e busy wait). + * Configures park time if using WaitStrategyPark for BackPressure. If set to 0, returns + * immediately (i.e busy wait). */ @NotNull @IsPositiveNumber(includeZero = true) - public static final String TOPOLOGY_BACKPRESSURE_WAIT_PARK_MICROSEC = "topology.backpressure.wait.park.microsec"; + public static final String TOPOLOGY_BACKPRESSURE_WAIT_PARK_MICROSEC = + "topology.backpressure.wait.park.microsec"; /** * Configures sleep time if using WaitStrategyProgressive for BackPressure. */ @@ -1083,21 +1326,26 @@ public class Config extends HashMap { public static final String TOPOLOGY_BACKPRESSURE_WAIT_PROGRESSIVE_LEVEL3_SLEEP_MILLIS = "topology.backpressure.wait.progressive.level3.sleep.millis"; /** - * Configures steps used to determine progression to the next level of wait .. if using WaitStrategyProgressive for BackPressure. + * Configures steps used to determine progression to the next level of wait .. if using + * WaitStrategyProgressive for BackPressure. */ @NotNull @IsInteger @IsPositiveNumber(includeZero = true) - public static final String TOPOLOGY_BACKPRESSURE_WAIT_PROGRESSIVE_LEVEL1_COUNT = "topology.backpressure.wait.progressive.level1.count"; + public static final String TOPOLOGY_BACKPRESSURE_WAIT_PROGRESSIVE_LEVEL1_COUNT = + "topology.backpressure.wait.progressive.level1.count"; /** - * Configures steps used to determine progression to the next level of wait .. if using WaitStrategyProgressive for BackPressure. + * Configures steps used to determine progression to the next level of wait .. if using + * WaitStrategyProgressive for BackPressure. */ @NotNull @IsInteger @IsPositiveNumber(includeZero = true) - public static final String TOPOLOGY_BACKPRESSURE_WAIT_PROGRESSIVE_LEVEL2_COUNT = "topology.backpressure.wait.progressive.level2.count"; + public static final String TOPOLOGY_BACKPRESSURE_WAIT_PROGRESSIVE_LEVEL2_COUNT = + "topology.backpressure.wait.progressive.level2.count"; /** - * Check recvQ after every N invocations of Spout's nextTuple() [when ACKing is disabled]. Spouts receive very few msgs if ACK is + * Check recvQ after every N invocations of Spout's nextTuple() [when ACKing is disabled]. + * Spouts receive very few msgs if ACK is * disabled. This avoids checking the recvQ after each nextTuple(). */ @IsInteger @@ -1105,32 +1353,41 @@ public class Config extends HashMap { @NotNull public static final String TOPOLOGY_SPOUT_RECVQ_SKIPS = "topology.spout.recvq.skips"; /** - * Minimum number of nimbus hosts where the code must be replicated before leader nimbus is allowed to perform topology activation tasks + * Minimum number of nimbus hosts where the code must be replicated before leader nimbus is + * allowed to perform topology activation tasks * like setting up heartbeats/assignments and marking the topology as active. default is 0. */ @IsNumber public static final String TOPOLOGY_MIN_REPLICATION_COUNT = "topology.min.replication.count"; /** - * Maximum wait time for the nimbus host replication to achieve the nimbus.min.replication.count. Once this time is elapsed nimbus will - * go ahead and perform topology activation tasks even if required nimbus.min.replication.count is not achieved. The default is 0 + * Maximum wait time for the nimbus host replication to achieve the + * nimbus.min.replication.count. Once this time is elapsed nimbus will + * go ahead and perform topology activation tasks even if required nimbus.min.replication.count + * is not achieved. The default is 0 * seconds, a value of -1 indicates to wait for ever. */ @IsNumber - public static final String TOPOLOGY_MAX_REPLICATION_WAIT_TIME_SEC = "topology.max.replication.wait.time.sec"; + public static final String TOPOLOGY_MAX_REPLICATION_WAIT_TIME_SEC = + "topology.max.replication.wait.time.sec"; /** * The list of servers that Pacemaker is running on. * - * @deprecated Pacemaker is deprecated and only kept for backward compatibility; it will be removed in a future release. - * Use the default heartbeat path (workers heartbeat to their supervisor, which reports them to Nimbus) instead. + * @deprecated Pacemaker is deprecated and only kept for backward compatibility; it will be + * removed in a future release. + * Use the default heartbeat path (workers heartbeat to their supervisor, which reports them + * to Nimbus) instead. */ @IsStringList @Deprecated public static final String PACEMAKER_SERVERS = "pacemaker.servers"; /** - * The port Pacemaker should run on. Clients should connect to this port to submit or read heartbeats. + * The port Pacemaker should run on. Clients should connect to this port to submit or read + * heartbeats. * - * @deprecated Pacemaker is deprecated and only kept for backward compatibility; it will be removed in a future release. - * Use the default heartbeat path (workers heartbeat to their supervisor, which reports them to Nimbus) instead. + * @deprecated Pacemaker is deprecated and only kept for backward compatibility; it will be + * removed in a future release. + * Use the default heartbeat path (workers heartbeat to their supervisor, which reports them + * to Nimbus) instead. */ @IsNumber @IsPositiveNumber @@ -1141,21 +1398,28 @@ public class Config extends HashMap { * When Pacemaker gets loaded it will spawn new threads, up to * this many total, to handle the load. * - * @deprecated Pacemaker is deprecated and only kept for backward compatibility; it will be removed in a future release. - * Use the default heartbeat path (workers heartbeat to their supervisor, which reports them to Nimbus) instead. + * @deprecated Pacemaker is deprecated and only kept for backward compatibility; it will be + * removed in a future release. + * Use the default heartbeat path (workers heartbeat to their supervisor, which reports them + * to Nimbus) instead. */ @IsNumber @IsPositiveNumber @Deprecated public static final String PACEMAKER_CLIENT_MAX_THREADS = "pacemaker.client.max.threads"; /** - * This should be one of "DIGEST", "KERBEROS", or "NONE" Determines the mode of authentication the pacemaker server and client use. The - * client must either match the server, or be NONE. In the case of NONE, no authentication is performed for the client, and if the - * server is running with DIGEST or KERBEROS, the client can only write to the server (no reads). This is intended to provide a + * This should be one of "DIGEST", "KERBEROS", or "NONE" Determines the mode of authentication + * the pacemaker server and client use. The + * client must either match the server, or be NONE. In the case of NONE, no authentication is + * performed for the client, and if the + * server is running with DIGEST or KERBEROS, the client can only write to the server (no + * reads). This is intended to provide a * primitive form of access-control. * - * @deprecated Pacemaker is deprecated and only kept for backward compatibility; it will be removed in a future release. - * Use the default heartbeat path (workers heartbeat to their supervisor, which reports them to Nimbus) instead. + * @deprecated Pacemaker is deprecated and only kept for backward compatibility; it will be + * removed in a future release. + * Use the default heartbeat path (workers heartbeat to their supervisor, which reports them + * to Nimbus) instead. */ @CustomValidator(validatorClass = ConfigValidation.PacemakerAuthTypeValidator.class) @Deprecated @@ -1163,25 +1427,30 @@ public class Config extends HashMap { /** * Pacemaker Thrift Max Message Size (bytes). * - * @deprecated Pacemaker is deprecated and only kept for backward compatibility; it will be removed in a future release. - * Use the default heartbeat path (workers heartbeat to their supervisor, which reports them to Nimbus) instead. + * @deprecated Pacemaker is deprecated and only kept for backward compatibility; it will be + * removed in a future release. + * Use the default heartbeat path (workers heartbeat to their supervisor, which reports them + * to Nimbus) instead. */ @IsInteger @IsPositiveNumber @Deprecated - public static final String PACEMAKER_THRIFT_MESSAGE_SIZE_MAX = "pacemaker.thrift.message.size.max"; + public static final String PACEMAKER_THRIFT_MESSAGE_SIZE_MAX = + "pacemaker.thrift.message.size.max"; /** - * Max no.of seconds group mapping service will cache user groups + * Max no.of seconds group mapping service will cache user groups. */ @IsInteger - public static final String STORM_GROUP_MAPPING_SERVICE_CACHE_DURATION_SECS = "storm.group.mapping.service.cache.duration.secs"; + public static final String STORM_GROUP_MAPPING_SERVICE_CACHE_DURATION_SECS = + "storm.group.mapping.service.cache.duration.secs"; /** * List of DRPC servers so that the DRPCSpout knows who to talk to. */ @IsStringList public static final String DRPC_SERVERS = "drpc.servers"; /** - * This port on Storm DRPC is used by DRPC topologies to receive function invocations and send results back. + * This port on Storm DRPC is used by DRPC topologies to receive function invocations and send + * results back. */ @IsInteger @IsPositiveNumber @@ -1200,32 +1469,40 @@ public class Config extends HashMap { * The ceiling of the interval between retries of a client connect to Nimbus operation. */ @IsNumber - public static final String STORM_NIMBUS_RETRY_INTERVAL_CEILING = "storm.nimbus.retry.intervalceiling.millis"; + public static final String STORM_NIMBUS_RETRY_INTERVAL_CEILING = + "storm.nimbus.retry.intervalceiling.millis"; /** * The Nimbus transport plug-in for Thrift client/server communication. */ @IsString public static final String NIMBUS_THRIFT_TRANSPORT_PLUGIN = "nimbus.thrift.transport"; /** - * Which port the Thrift interface of Nimbus should run on. Clients should connect to this port to upload jars and submit topologies. + * Which port the Thrift interface of Nimbus should run on. Clients should connect to this port + * to upload jars and submit topologies. */ @IsInteger @IsPositiveNumber public static final String NIMBUS_THRIFT_PORT = "nimbus.thrift.port"; /** - * Nimbus thrift server queue size, default is 100000. This is the request queue size , when there are more requests than number of - * threads to serve the requests, those requests will be queued to this queue. If the request queue size > this config, then the + * Nimbus thrift server queue size, default is 100000. This is the request queue size , when + * there are more requests than number of + * threads to serve the requests, those requests will be queued to this queue. If the request + * queue size > this config, then the * incoming requests will be rejected. */ @IsInteger @IsPositiveNumber public static final String NIMBUS_QUEUE_SIZE = "nimbus.queue.size"; /** - * Nimbus assignments backend for storing local assignments. We will use it to store physical plan and runtime storm ids. + * Nimbus assignments backend for storing local assignments. We will use it to store physical + * plan and runtime storm ids. */ @IsString - @ConfigValidationAnnotations.IsImplementationOfClass(implementsClass = org.apache.storm.assignments.ILocalAssignmentsBackend.class) - public static final String NIMBUS_LOCAL_ASSIGNMENTS_BACKEND_CLASS = "nimbus.local.assignments.backend.class"; + @ConfigValidationAnnotations + .IsImplementationOfClass(implementsClass = + org.apache.storm.assignments.ILocalAssignmentsBackend.class) + public static final String NIMBUS_LOCAL_ASSIGNMENTS_BACKEND_CLASS = + "nimbus.local.assignments.backend.class"; /** * The number of threads that should be used by the nimbus thrift server. */ @@ -1266,20 +1543,23 @@ public class Config extends HashMap { * The default value is set false. */ @IsBoolean - public static final String NIMBUS_THRIFT_TLS_CLIENT_AUTH_REQUIRED = "nimbus.thrift.tls.client.auth.required"; + public static final String NIMBUS_THRIFT_TLS_CLIENT_AUTH_REQUIRED = + "nimbus.thrift.tls.client.auth.required"; /** * The path to the keystore that the nimbus TLS server uses. */ @IsString - public static final String NIMBUS_THRIFT_TLS_SERVER_KEYSTORE_PATH = "nimbus.thrift.tls.server.keystore.path"; + public static final String NIMBUS_THRIFT_TLS_SERVER_KEYSTORE_PATH = + "nimbus.thrift.tls.server.keystore.path"; /** * The password of the keystore that the nimbus TLS server uses. */ @IsString @Password - public static final String NIMBUS_THRIFT_TLS_SERVER_KEYSTORE_PASSWORD = "nimbus.thrift.tls.server.keystore.password"; + public static final String NIMBUS_THRIFT_TLS_SERVER_KEYSTORE_PASSWORD = + "nimbus.thrift.tls.server.keystore.password"; /** * Launch only the tls server. @@ -1291,14 +1571,16 @@ public class Config extends HashMap { * The path to the truststore that the nimbus TLS server uses. */ @IsString - public static final String NIMBUS_THRIFT_TLS_SERVER_TRUSTSTORE_PATH = "nimbus.thrift.tls.server.truststore.path"; + public static final String NIMBUS_THRIFT_TLS_SERVER_TRUSTSTORE_PATH = + "nimbus.thrift.tls.server.truststore.path"; /** * The password of the truststore that the nimbus TLS server uses. */ @IsString @Password - public static final String NIMBUS_THRIFT_TLS_SERVER_TRUSTSTORE_PASSWORD = "nimbus.thrift.tls.server.truststore.password"; + public static final String NIMBUS_THRIFT_TLS_SERVER_TRUSTSTORE_PASSWORD = + "nimbus.thrift.tls.server.truststore.password"; /** * The path to the keystore that the nimbus TLS client uses. @@ -1306,39 +1588,45 @@ public class Config extends HashMap { * and nimbus.thrift.tls.client.cert.path in order to use client keystore */ @IsString - public static final String NIMBUS_THRIFT_TLS_CLIENT_KEYSTORE_PATH = "nimbus.thrift.tls.client.keystore.path"; + public static final String NIMBUS_THRIFT_TLS_CLIENT_KEYSTORE_PATH = + "nimbus.thrift.tls.client.keystore.path"; /** * The password of the keystore that the nimbus TLS client uses. */ @IsString @Password - public static final String NIMBUS_THRIFT_TLS_CLIENT_KEYSTORE_PASSWORD = "nimbus.thrift.tls.client.keystore.password"; + public static final String NIMBUS_THRIFT_TLS_CLIENT_KEYSTORE_PASSWORD = + "nimbus.thrift.tls.client.keystore.password"; /** * The path to the key that the nimbus TLS client uses. */ @IsString - public static final String NIMBUS_THRIFT_TLS_CLIENT_KEY_PATH = "nimbus.thrift.tls.client.key.path"; + public static final String NIMBUS_THRIFT_TLS_CLIENT_KEY_PATH = + "nimbus.thrift.tls.client.key.path"; /** * The path of the certificate that the nimbus TLS client uses. */ @IsString - public static final String NIMBUS_THRIFT_TLS_CLIENT_CERT_PATH = "nimbus.thrift.tls.client.cert.path"; + public static final String NIMBUS_THRIFT_TLS_CLIENT_CERT_PATH = + "nimbus.thrift.tls.client.cert.path"; /** * The path to the truststore that the nimbus TLS client uses. */ @IsString - public static final String NIMBUS_THRIFT_TLS_CLIENT_TRUSTSTORE_PATH = "nimbus.thrift.tls.client.truststore.path"; + public static final String NIMBUS_THRIFT_TLS_CLIENT_TRUSTSTORE_PATH = + "nimbus.thrift.tls.client.truststore.path"; /** * The password of the truststore that the nimbus TLS client uses. */ @IsString @Password - public static final String NIMBUS_THRIFT_TLS_CLIENT_TRUSTSTORE_PASSWORD = "nimbus.thrift.tls.client.truststore.password"; + public static final String NIMBUS_THRIFT_TLS_CLIENT_TRUSTSTORE_PASSWORD = + "nimbus.thrift.tls.client.truststore.password"; /** * The number of threads that should be used by the nimbus thrift TLS server. @@ -1352,22 +1640,26 @@ public class Config extends HashMap { */ @IsInteger @IsPositiveNumber - public static final String NIMBUS_THRIFT_TLS_MAX_BUFFER_SIZE = "nimbus.thrift.tls.max_buffer_size"; + public static final String NIMBUS_THRIFT_TLS_MAX_BUFFER_SIZE = + "nimbus.thrift.tls.max_buffer_size"; /** * How long before a Thrift TLS Client socket hangs before timeout and restart the socket. */ @IsInteger - public static final String STORM_THRIFT_TLS_SOCKET_TIMEOUT_MS = "storm.thrift.tls.socket.timeout.ms"; + public static final String STORM_THRIFT_TLS_SOCKET_TIMEOUT_MS = + "storm.thrift.tls.socket.timeout.ms"; /** * Whether the workers of the topology should use TLS or non-TLS thrift to connect to nimbus. * It is achieved by changing the value of {@link #NIMBUS_THRIFT_CLIENT_USE_TLS} for workers. - * Nimbus will adjust this value in topology conf during submission, so that the value will only be set true if + * Nimbus will adjust this value in topology conf during submission, so that the value will only + * be set true if * both Nimbus conf and topology conf have it set as true. */ @IsBoolean - public static final String TOPOLOGY_WORKER_NIMBUS_THRIFT_CLIENT_USE_TLS = "topology.worker.nimbus.thrift.client.use.tls"; + public static final String TOPOLOGY_WORKER_NIMBUS_THRIFT_CLIENT_USE_TLS = + "topology.worker.nimbus.thrift.client.use.tls"; /** * Whether the nimbus client should use TLS or non-TLS thrift to connect to nimbus. @@ -1381,77 +1673,89 @@ public class Config extends HashMap { * The default value is set false. */ @IsBoolean - public static final String SUPERVISOR_THRIFT_TLS_CLIENT_AUTH_REQUIRED = "supervisor.thrift.tls.client.auth.required"; + public static final String SUPERVISOR_THRIFT_TLS_CLIENT_AUTH_REQUIRED = + "supervisor.thrift.tls.client.auth.required"; /** * The path to the keystore that the supervisor TLS server uses. */ @IsString - public static final String SUPERVISOR_THRIFT_TLS_SERVER_KEYSTORE_PATH = "supervisor.thrift.tls.server.keystore.path"; + public static final String SUPERVISOR_THRIFT_TLS_SERVER_KEYSTORE_PATH = + "supervisor.thrift.tls.server.keystore.path"; /** * The password of the keystore that the supervisor TLS server uses. */ @IsString @Password - public static final String SUPERVISOR_THRIFT_TLS_SERVER_KEYSTORE_PASSWORD = "supervisor.thrift.tls.server.keystore.password"; + public static final String SUPERVISOR_THRIFT_TLS_SERVER_KEYSTORE_PASSWORD = + "supervisor.thrift.tls.server.keystore.password"; /** * The path to the truststore that the supervisor TLS server uses. */ @IsString - public static final String SUPERVISOR_THRIFT_TLS_SERVER_TRUSTSTORE_PATH = "supervisor.thrift.tls.server.truststore.path"; + public static final String SUPERVISOR_THRIFT_TLS_SERVER_TRUSTSTORE_PATH = + "supervisor.thrift.tls.server.truststore.path"; /** * The password of the truststore that the supervisor TLS server uses. */ @IsString @Password - public static final String SUPERVISOR_THRIFT_TLS_SERVER_TRUSTSTORE_PASSWORD = "supervisor.thrift.tls.server.truststore.password"; + public static final String SUPERVISOR_THRIFT_TLS_SERVER_TRUSTSTORE_PASSWORD = + "supervisor.thrift.tls.server.truststore.password"; /** * The path to the keystore that the supervisor TLS client uses. */ @IsString - public static final String SUPERVISOR_THRIFT_TLS_CLIENT_KEYSTORE_PATH = "supervisor.thrift.tls.client.keystore.path"; + public static final String SUPERVISOR_THRIFT_TLS_CLIENT_KEYSTORE_PATH = + "supervisor.thrift.tls.client.keystore.path"; /** * The password of the keystore that the supervisor TLS client uses. */ @IsString @Password - public static final String SUPERVISOR_THRIFT_TLS_CLIENT_KEYSTORE_PASSWORD = "supervisor.thrift.tls.client.keystore.password"; + public static final String SUPERVISOR_THRIFT_TLS_CLIENT_KEYSTORE_PASSWORD = + "supervisor.thrift.tls.client.keystore.password"; /** * The path to the truststore that the supervisor TLS client uses. */ @IsString - public static final String SUPERVISOR_THRIFT_TLS_CLIENT_TRUSTSTORE_PATH = "supervisor.thrift.tls.client.truststore.path"; + public static final String SUPERVISOR_THRIFT_TLS_CLIENT_TRUSTSTORE_PATH = + "supervisor.thrift.tls.client.truststore.path"; /** * The password of the truststore that the supervisor TLS client uses. */ @IsString @Password - public static final String SUPERVISOR_THRIFT_TLS_CLIENT_TRUSTSTORE_PASSWORD = "supervisor.thrift.tls.client.truststore.password"; + public static final String SUPERVISOR_THRIFT_TLS_CLIENT_TRUSTSTORE_PASSWORD = + "supervisor.thrift.tls.client.truststore.password"; /** * The path to the key that the supervisor TLS client uses. */ @IsString - public static final String SUPERVISOR_THRIFT_TLS_CLIENT_KEY_PATH = "supervisor.thrift.tls.client.key.path"; + public static final String SUPERVISOR_THRIFT_TLS_CLIENT_KEY_PATH = + "supervisor.thrift.tls.client.key.path"; /** * The path of the certificate that the supervisor TLS client uses. */ @IsString - public static final String SUPERVISOR_THRIFT_TLS_CLIENT_CERT_PATH = "supervisor.thrift.tls.client.cert.path"; + public static final String SUPERVISOR_THRIFT_TLS_CLIENT_CERT_PATH = + "supervisor.thrift.tls.client.cert.path"; /** * Whether the supervisor clients should use TLS or non-TLS. */ @IsBoolean - public static final String SUPERVISOR_THRIFT_CLIENT_USE_TLS = "supervisor.thrift.client.use.tls"; + public static final String SUPERVISOR_THRIFT_CLIENT_USE_TLS = + "supervisor.thrift.client.use.tls"; /** * The DRPC transport plug-in for Thrift client/server communication. @@ -1486,7 +1790,8 @@ public class Config extends HashMap { * The DRPC invocations transport plug-in for Thrift client/server communication. */ @IsString - public static final String DRPC_INVOCATIONS_THRIFT_TRANSPORT_PLUGIN = "drpc.invocations.thrift.transport"; + public static final String DRPC_INVOCATIONS_THRIFT_TRANSPORT_PLUGIN = + "drpc.invocations.thrift.transport"; /** * DRPC invocations thrift server worker threads. */ @@ -1495,17 +1800,20 @@ public class Config extends HashMap { public static final String DRPC_INVOCATIONS_THREADS = "drpc.invocations.threads"; /** * Initialization parameters for the group mapping service plugin. Provides a way for a - * {@link #STORM_GROUP_MAPPING_SERVICE_PROVIDER_PLUGIN} implementation to access optional settings. + * {@link #STORM_GROUP_MAPPING_SERVICE_PROVIDER_PLUGIN} implementation to access optional + * settings. */ @IsType(type = Map.class) - public static final String STORM_GROUP_MAPPING_SERVICE_PARAMS = "storm.group.mapping.service.params"; + public static final String STORM_GROUP_MAPPING_SERVICE_PARAMS = + "storm.group.mapping.service.params"; /** * The default transport plug-in for Thrift client/server communication. */ @IsString public static final String STORM_THRIFT_TRANSPORT_PLUGIN = "storm.thrift.transport"; /** - * How long a worker can go without heartbeating before the supervisor tries to restart the worker process. + * How long a worker can go without heartbeating before the supervisor tries to restart the + * worker process. * Can be overridden by {@link #TOPOLOGY_WORKER_TIMEOUT_SECS}, if set. */ @IsInteger @@ -1520,22 +1828,27 @@ public class Config extends HashMap { @NotNull public static final String WORKER_MAX_TIMEOUT_SECS = "worker.max.timeout.secs"; /** - * Topology configurable worker heartbeat timeout before the supervisor tries to restart the worker process. + * Topology configurable worker heartbeat timeout before the supervisor tries to restart the + * worker process. * Maximum value constrained by {@link #WORKER_MAX_TIMEOUT_SECS}. * When topology timeout is greater, the following configs are effectively overridden: - * {@link #SUPERVISOR_WORKER_TIMEOUT_SECS}, SUPERVISOR_WORKER_START_TIMEOUT_SECS, NIMBUS_TASK_TIMEOUT_SECS and NIMBUS_TASK_LAUNCH_SECS. + * {@link #SUPERVISOR_WORKER_TIMEOUT_SECS}, SUPERVISOR_WORKER_START_TIMEOUT_SECS, + * NIMBUS_TASK_TIMEOUT_SECS and NIMBUS_TASK_LAUNCH_SECS. */ @IsInteger @IsPositiveNumber @NotNull public static final String TOPOLOGY_WORKER_TIMEOUT_SECS = "topology.worker.timeout.secs"; /** - * How many seconds to allow for graceful worker shutdown when killing workers before resorting to force kill. - * If a worker fails to shut down gracefully within this delay, it will either suicide or be forcibly killed by the supervisor. + * How many seconds to allow for graceful worker shutdown when killing workers before resorting + * to force kill. + * If a worker fails to shut down gracefully within this delay, it will either suicide or be + * forcibly killed by the supervisor. */ @IsInteger @IsPositiveNumber - public static final String SUPERVISOR_WORKER_SHUTDOWN_SLEEP_SECS = "supervisor.worker.shutdown.sleep.secs"; + public static final String SUPERVISOR_WORKER_SHUTDOWN_SLEEP_SECS = + "supervisor.worker.shutdown.sleep.secs"; /** * A list of hosts of ZooKeeper servers used to manage the cluster. */ @@ -1548,7 +1861,8 @@ public class Config extends HashMap { @IsPositiveNumber public static final String STORM_ZOOKEEPER_PORT = "storm.zookeeper.port"; /** - * This is part of a temporary workaround to a ZK bug, it is the 'scheme:acl' for the user Nimbus and Supervisors use to authenticate + * This is part of a temporary workaround to a ZK bug, it is the 'scheme:acl' for the user + * Nimbus and Supervisors use to authenticate * with ZK. */ @IsString @@ -1561,10 +1875,12 @@ public class Config extends HashMap { @IsString public static final String STORM_ZOOKEEPER_DRPC_ACL = "storm.zookeeper.drpcACL"; /** - * The topology Zookeeper authentication scheme to use, e.g. "digest". It is the internal config and user shouldn't set it. + * The topology Zookeeper authentication scheme to use, e.g. "digest". It is the internal config + * and user shouldn't set it. */ @IsString - public static final String STORM_ZOOKEEPER_TOPOLOGY_AUTH_SCHEME = "storm.zookeeper.topology.auth.scheme"; + public static final String STORM_ZOOKEEPER_TOPOLOGY_AUTH_SCHEME = + "storm.zookeeper.topology.auth.scheme"; /** * Enable SSL/TLS for ZooKeeper client connection. @@ -1575,61 +1891,78 @@ public class Config extends HashMap { * Keystore location for ZooKeeper client connection over SSL. */ @IsString - public static final String STORM_ZOOKEEPER_SSL_KEYSTORE_PATH = "storm.zookeeper.ssl.keystore.path"; + public static final String STORM_ZOOKEEPER_SSL_KEYSTORE_PATH = + "storm.zookeeper.ssl.keystore.path"; /** * Keystore password for ZooKeeper client connection over SSL. */ @IsString @Password - public static final String STORM_ZOOKEEPER_SSL_KEYSTORE_PASSWORD = "storm.zookeeper.ssl.keystore.password"; + public static final String STORM_ZOOKEEPER_SSL_KEYSTORE_PASSWORD = + "storm.zookeeper.ssl.keystore.password"; /** * Truststore location for ZooKeeper client connection over SSL. */ @IsString - public static final String STORM_ZOOKEEPER_SSL_TRUSTSTORE_PATH = "storm.zookeeper.ssl.truststore.path"; + public static final String STORM_ZOOKEEPER_SSL_TRUSTSTORE_PATH = + "storm.zookeeper.ssl.truststore.path"; /** * Truststore password for ZooKeeper client connection over SSL. */ @IsString @Password - public static final String STORM_ZOOKEEPER_SSL_TRUSTSTORE_PASSWORD = "storm.zookeeper.ssl.truststore.password"; + public static final String STORM_ZOOKEEPER_SSL_TRUSTSTORE_PASSWORD = + "storm.zookeeper.ssl.truststore.password"; /** * Enable or disable hostname verification. */ @IsBoolean - public static final String STORM_ZOOKEEPER_SSL_HOSTNAME_VERIFICATION = "storm.zookeeper.ssl.hostnameVerification"; + public static final String STORM_ZOOKEEPER_SSL_HOSTNAME_VERIFICATION = + "storm.zookeeper.ssl.hostnameVerification"; /** - * The delegate for serializing metadata, should be used for serialized objects stored in zookeeper and on disk. This is NOT used for + * The delegate for serializing metadata, should be used for serialized objects stored in + * zookeeper and on disk. This is NOT used for * compressing serialized tuples sent between topologies. */ @IsString - public static final String STORM_META_SERIALIZATION_DELEGATE = "storm.meta.serialization.delegate"; + public static final String STORM_META_SERIALIZATION_DELEGATE = + "storm.meta.serialization.delegate"; /** - * Topology configuration to enable compression of serialized tuples during inter-worker network transfer. + * Topology configuration to enable compression of serialized tuples during inter-worker network + * transfer. * When set to {@code true}, tuples emitted by this component will be compressed prior to being sent - * over the network to remote worker processes. This is highly recommended for topologies exchanging - * large payloads (e.g., entire lines of text or large data blocks) to significantly reduce network I/O. + * over the network to remote worker processes. This is highly recommended for topologies + * exchanging + * large payloads (e.g., entire lines of text or large data blocks) to significantly reduce + * network I/O. * Default: {@code false} (Disabled by default to prevent unexpected CPU overhead). */ @IsBoolean - public static final String TOPOLOGY_TUPLE_COMPRESSION_ENABLE = "topology.tuple.compression.enable"; + public static final String TOPOLOGY_TUPLE_COMPRESSION_ENABLE = + "topology.tuple.compression.enable"; /** - * Topology configuration specifying the minimum size threshold (in bytes) for compressing serialized tuples + * Topology configuration specifying the minimum size threshold (in bytes) for compressing + * serialized tuples * during inter-worker network transfer. * When the serialized byte array of a tuple exceeds this threshold, it will be compressed - * prior to being transmitted over the network to a remote worker process. This optimizes network I/O - * for large payloads (such as text blocks or massive objects) without wasting cycles on small data. + * prior to being transmitted over the network to a remote worker process. This optimizes + * network I/O + * for large payloads (such as text blocks or massive objects) without wasting cycles on small + * data. * Set to 0 to bypass the size check and compress all tuples regardless of size. * Default: {@code 1460} bytes (The typical maximum segment size [MSS] for a standard - * Ethernet TCP payload, preventing compression on packets that already fit within a single network frame). + * Ethernet TCP payload, preventing compression on packets that already fit within a single + * network frame). */ @IsPositiveNumber(includeZero = true) - public static final String TOPOLOGY_TUPLE_COMPRESSION_THRESHOLD = "topology.tuple.compression.threshold"; + public static final String TOPOLOGY_TUPLE_COMPRESSION_THRESHOLD = + "topology.tuple.compression.threshold"; /** * GZIP max decompression bytes. Defaults to 104857600 (100MB). */ @IsPositiveNumber(includeZero = false) - public static final String STORM_COMPRESSION_GZIP_MAX_DECOMPRESSED_BYTES = "storm.compression.gzip.max.decompressed.bytes"; + public static final String STORM_COMPRESSION_GZIP_MAX_DECOMPRESSED_BYTES = + "storm.compression.gzip.max.decompressed.bytes"; /** * Zstandard compression level. * Supported range: 1 to 19. Default: 3. @@ -1642,12 +1975,14 @@ public class Config extends HashMap { * Zstandard max decompression bytes. Defaults to 104857600 (100MB). */ @IsPositiveNumber(includeZero = false) - public static final String STORM_COMPRESSION_ZSTD_MAX_DECOMPRESSED_BYTES = "storm.compression.zstd.max.decompressed.bytes"; + public static final String STORM_COMPRESSION_ZSTD_MAX_DECOMPRESSED_BYTES = + "storm.compression.zstd.max.decompressed.bytes"; /** * Max decompression bytes for tuples. Defaults to 10485760 (10MB). */ @IsPositiveNumber(includeZero = false) - public static final String TOPOLOGY_TUPLE_COMPRESSION_MAX_DECOMPRESSED_BYTES = "topology.tuple.compression.max.decompressed.bytes"; + public static final String TOPOLOGY_TUPLE_COMPRESSION_MAX_DECOMPRESSED_BYTES = + "topology.tuple.compression.max.decompressed.bytes"; /** * Configure the topology metrics reporters to be used on workers. */ @@ -1658,7 +1993,8 @@ public class Config extends HashMap { * A list of system metrics reporters that will get added to each topology. */ @IsListEntryCustom(entryValidatorClasses = {MetricReportersValidator.class}) - public static final String STORM_TOPOLOGY_METRICS_SYSTEM_REPORTERS = "storm.topology.metrics.system.reporters"; + public static final String STORM_TOPOLOGY_METRICS_SYSTEM_REPORTERS = + "storm.topology.metrics.system.reporters"; /** * Configure the topology metrics reporters to be used on workers. @@ -1675,20 +2011,23 @@ public class Config extends HashMap { public static final String CLIENT_BLOBSTORE = "client.blobstore.class"; /** - * What directory to use for the blobstore. The directory is expected to be an absolute path when using HDFS blobstore, for - * LocalFsBlobStore it could be either absolute or relative. If the setting is a relative directory, it is relative to root directory of + * What directory to use for the blobstore. The directory is expected to be an absolute path + * when using HDFS blobstore, for + * LocalFsBlobStore it could be either absolute or relative. If the setting is a relative + * directory, it is relative to root directory of * Storm installation. */ @IsString public static final String BLOBSTORE_DIR = "blobstore.dir"; /** - * Enable the blobstore cleaner. Certain blobstores may only want to run the cleaner on one daemon. Currently Nimbus handles setting + * Enable the blobstore cleaner. Certain blobstores may only want to run the cleaner on one + * daemon. Currently Nimbus handles setting * this. */ @IsBoolean public static final String BLOBSTORE_CLEANUP_ENABLE = "blobstore.cleanup.enable"; /** - * principal for nimbus/supervisor to use to access secure hdfs for the blobstore. + * Principal for nimbus/supervisor to use to access secure hdfs for the blobstore. * The format is generally "primary/instance@REALM", where "instance" field is optional. * If the instance field of the principal is the string "_HOST", it will * be replaced with the host name of the server the daemon is running on @@ -1699,7 +2038,7 @@ public class Config extends HashMap { @IsString public static final String BLOBSTORE_HDFS_PRINCIPAL = "blobstore.hdfs.principal"; /** - * keytab for nimbus/supervisor to use to access secure hdfs for the blobstore. + * Keytab for nimbus/supervisor to use to access secure hdfs for the blobstore. * * @Deprecated Use {@link Config#STORM_HDFS_LOGIN_KEYTAB} instead. */ @@ -1711,7 +2050,8 @@ public class Config extends HashMap { */ @IsPositiveNumber @IsInteger - public static final String STORM_BLOBSTORE_REPLICATION_FACTOR = "storm.blobstore.replication.factor"; + public static final String STORM_BLOBSTORE_REPLICATION_FACTOR = + "storm.blobstore.replication.factor"; /** * The principal for nimbus/supervisor to use to access secure hdfs. * The format is generally "primary/instance@REALM", where "instance" field is optional. @@ -1728,10 +2068,12 @@ public class Config extends HashMap { @IsString public static final String STORM_HDFS_LOGIN_KEYTAB = "storm.hdfs.login.keytab"; /** - * The hostname the supervisors/workers should report to nimbus. If unset, Storm will get the hostname to report by calling + * The hostname the supervisors/workers should report to nimbus. If unset, Storm will get the + * hostname to report by calling * InetAddress.getLocalHost().getCanonicalHostName(). * - *

    You should set this config when you don't have a DNS which supervisors/workers can utilize to find each other + *

    You should set this config when you don't have a DNS which supervisors/workers can utilize + * to find each other * based on hostname got from calls to * InetAddress.getLocalHost().getCanonicalHostName(). */ @@ -1743,13 +2085,15 @@ public class Config extends HashMap { @IsStringList public static final String NIMBUS_SEEDS = "nimbus.seeds"; /** - * A list of users that are the only ones allowed to run user operation on storm cluster. To use this set nimbus.authorizer to + * A list of users that are the only ones allowed to run user operation on storm cluster. To use + * this set nimbus.authorizer to * org.apache.storm.security.auth.authorizer.SimpleACLAuthorizer */ @IsStringList public static final String NIMBUS_USERS = "nimbus.users"; /** - * A list of groups , users belong to these groups are the only ones allowed to run user operation on storm cluster. To use this set + * A list of groups , users belong to these groups are the only ones allowed to run user + * operation on storm cluster. To use this set * nimbus.authorizer to org.apache.storm.security.auth.authorizer.SimpleACLAuthorizer */ @IsStringList @@ -1765,63 +2109,80 @@ public class Config extends HashMap { @IsString public static final String STORM_ZOOKEEPER_ROOT = "storm.zookeeper.root"; /** - * A string representing the payload for topology Zookeeper authentication. It gets serialized using UTF-8 encoding during + * A string representing the payload for topology Zookeeper authentication. It gets serialized + * using UTF-8 encoding during * authentication. */ @IsString @Password - public static final String STORM_ZOOKEEPER_TOPOLOGY_AUTH_PAYLOAD = "storm.zookeeper.topology.auth.payload"; + public static final String STORM_ZOOKEEPER_TOPOLOGY_AUTH_PAYLOAD = + "storm.zookeeper.topology.auth.payload"; /** - * The cluster Zookeeper authentication scheme to use, e.g. "digest". Defaults to no authentication. + * The cluster Zookeeper authentication scheme to use, e.g. "digest". Defaults to no + * authentication. */ @IsString public static final String STORM_ZOOKEEPER_AUTH_SCHEME = "storm.zookeeper.auth.scheme"; /** - * A string representing the payload for cluster Zookeeper authentication. It gets serialized using UTF-8 encoding during - * authentication. Note that if this is set to something with a secret (as when using digest authentication) then it should only be set - * in the storm-cluster-auth.yaml file. This file storm-cluster-auth.yaml should then be protected with appropriate permissions that + * A string representing the payload for cluster Zookeeper authentication. It gets serialized + * using UTF-8 encoding during + * authentication. Note that if this is set to something with a secret (as when using digest + * authentication) then it should only be set + * in the storm-cluster-auth.yaml file. This file storm-cluster-auth.yaml should then be + * protected with appropriate permissions that * deny access from workers. */ @IsString @Password public static final String STORM_ZOOKEEPER_AUTH_PAYLOAD = "storm.zookeeper.auth.payload"; /** - * What Network Topography detection classes should we use. Given a list of supervisor hostnames (or IP addresses), this class would - * return a list of rack names that correspond to the supervisors. This information is stored in Cluster.java, and is used in the + * What Network Topography detection classes should we use. Given a list of supervisor hostnames + * (or IP addresses), this class would + * return a list of rack names that correspond to the supervisors. This information is stored in + * Cluster.java, and is used in the * resource aware scheduler. */ @NotNull - @IsImplementationOfClass(implementsClass = org.apache.storm.networktopography.DNSToSwitchMapping.class) + @IsImplementationOfClass(implementsClass = + org.apache.storm.networktopography.DNSToSwitchMapping.class) public static final String STORM_NETWORK_TOPOGRAPHY_PLUGIN = "storm.network.topography.plugin"; /** - * The jvm opts provided to workers launched by this supervisor for GC. All "%ID%" substrings are replaced with an identifier for this - * worker. Because the JVM complains about multiple GC opts the topology can override this default value by setting + * The jvm opts provided to workers launched by this supervisor for GC. All "%ID%" substrings + * are replaced with an identifier for this + * worker. Because the JVM complains about multiple GC opts the topology can override this + * default value by setting * topology.worker.gc.childopts. */ @IsStringOrStringList public static final String WORKER_GC_CHILDOPTS = "worker.gc.childopts"; /** - * The jvm opts provided to workers launched by this supervisor. All "%ID%", "%WORKER-ID%", "%TOPOLOGY-ID%", "%WORKER-PORT%" and - * "%HEAP-MEM%" substrings are replaced with: %ID% -> port (for backward compatibility), %WORKER-ID% -> worker-id, + * The jvm opts provided to workers launched by this supervisor. All "%ID%", "%WORKER-ID%", + * "%TOPOLOGY-ID%", "%WORKER-PORT%" and + * "%HEAP-MEM%" substrings are replaced with: %ID% -> port (for backward compatibility), + * %WORKER-ID% -> worker-id, * %TOPOLOGY-ID% -> topology-id, %WORKER-PORT% -> port. %HEAP-MEM% -> mem-onheap. */ @IsStringOrStringList public static final String WORKER_CHILDOPTS = "worker.childopts"; /** - * The default heap memory size in MB per worker, used in the jvm -Xmx opts for launching the worker. + * The default heap memory size in MB per worker, used in the jvm -Xmx opts for launching the + * worker. */ @IsInteger @IsPositiveNumber public static final String WORKER_HEAP_MEMORY_MB = "worker.heap.memory.mb"; /** - * The total amount of memory (in MiB) a supervisor is allowed to give to its workers. A default value will be set for this config if + * The total amount of memory (in MiB) a supervisor is allowed to give to its workers. A default + * value will be set for this config if * user does not override */ @IsPositiveNumber public static final String SUPERVISOR_MEMORY_CAPACITY_MB = "supervisor.memory.capacity.mb"; /** - * The total amount of CPU resources a supervisor is allowed to give to its workers. By convention 1 cpu core should be about 100, but - * this can be adjusted if needed using 100 makes it simple to set the desired value to the capacity measurement for single threaded + * The total amount of CPU resources a supervisor is allowed to give to its workers. By + * convention 1 cpu core should be about 100, but + * this can be adjusted if needed using 100 makes it simple to set the desired value to the + * capacity measurement for single threaded * bolts. A default value will be set for this config if user does not override */ @IsPositiveNumber @@ -1851,20 +2212,26 @@ public class Config extends HashMap { public static final String SUPERVISOR_THRIFT_THREADS = "supervisor.thrift.threads"; @IsNumber @IsPositiveNumber - public static final String SUPERVISOR_THRIFT_MAX_BUFFER_SIZE = "supervisor.thrift.max_buffer_size"; + public static final String SUPERVISOR_THRIFT_MAX_BUFFER_SIZE = + "supervisor.thrift.max_buffer_size"; /** - * How long before a supervisor Thrift Client socket hangs before timeout and restart the socket. + * How long before a supervisor Thrift Client socket hangs before timeout and restart the + * socket. */ @IsInteger - public static final String SUPERVISOR_THRIFT_SOCKET_TIMEOUT_MS = "supervisor.thrift.socket.timeout.ms"; + public static final String SUPERVISOR_THRIFT_SOCKET_TIMEOUT_MS = + "supervisor.thrift.socket.timeout.ms"; /** - * A map of resources the Supervisor has e.g {"cpu.pcore.percent" : 200.0. "onheap.memory.mb": 256.0, "gpu.count" : 2.0 } + * A map of resources the Supervisor has e.g {"cpu.pcore.percent" : 200.0. "onheap.memory.mb": + * 256.0, "gpu.count" : 2.0 } */ @IsMapEntryType(keyType = String.class, valueType = Number.class) public static final String SUPERVISOR_RESOURCES_MAP = "supervisor.resources.map"; /** - * Whether or not to use ZeroMQ for messaging in local mode. If this is set to false, then Storm will use a pure-Java messaging system. - * The purpose of this flag is to make it easy to run Storm in local mode by eliminating the need for native dependencies, which can be + * Whether or not to use ZeroMQ for messaging in local mode. If this is set to false, then Storm + * will use a pure-Java messaging system. + * The purpose of this flag is to make it easy to run Storm in local mode by eliminating the + * need for native dependencies, which can be * difficult to install. * *

    Defaults to false. @@ -1877,68 +2244,81 @@ public class Config extends HashMap { @IsString public static final String STORM_MESSAGING_TRANSPORT = "storm.messaging.transport"; /** - * Netty based messaging: Is authentication required for Netty messaging from client worker process to server worker process. + * Netty based messaging: Is authentication required for Netty messaging from client worker + * process to server worker process. * See https://issues.apache.org/jira/browse/STORM-348 for more details */ @IsBoolean - public static final String STORM_MESSAGING_NETTY_AUTHENTICATION = "storm.messaging.netty.authentication"; + public static final String STORM_MESSAGING_NETTY_AUTHENTICATION = + "storm.messaging.netty.authentication"; /** * Netty based messaging: The buffer size for send/recv buffer. */ @IsInteger @IsPositiveNumber - public static final String STORM_MESSAGING_NETTY_BUFFER_SIZE = "storm.messaging.netty.buffer_size"; + public static final String STORM_MESSAGING_NETTY_BUFFER_SIZE = + "storm.messaging.netty.buffer_size"; /** * Netty based messaging: The netty write buffer high watermark in bytes. - *

    - * If the number of bytes queued in the netty's write buffer exceeds this value, the netty {@code Channel.isWritable()} will start to + * + *

    If the number of bytes queued in the netty's write buffer exceeds this value, the netty {@code Channel.isWritable()} will start to * return {@code false}. The client will wait until the value falls below the {@linkplain #STORM_MESSAGING_NETTY_BUFFER_LOW_WATERMARK * low water mark}. *

    */ @IsInteger @IsPositiveNumber - public static final String STORM_MESSAGING_NETTY_BUFFER_HIGH_WATERMARK = "storm.messaging.netty.buffer.high.watermark"; + public static final String STORM_MESSAGING_NETTY_BUFFER_HIGH_WATERMARK = + "storm.messaging.netty.buffer.high.watermark"; /** * Netty based messaging: The netty write buffer low watermark in bytes. - *

    - * Once the number of bytes queued in the write buffer exceeded the {@linkplain #STORM_MESSAGING_NETTY_BUFFER_HIGH_WATERMARK high water + * + *

    Once the number of bytes queued in the write buffer exceeded the {@linkplain + * #STORM_MESSAGING_NETTY_BUFFER_HIGH_WATERMARK high water * mark} and then dropped down below this value, the netty {@code Channel.isWritable()} will start to return true. *

    */ @IsInteger @IsPositiveNumber - public static final String STORM_MESSAGING_NETTY_BUFFER_LOW_WATERMARK = "storm.messaging.netty.buffer.low.watermark"; + public static final String STORM_MESSAGING_NETTY_BUFFER_LOW_WATERMARK = + "storm.messaging.netty.buffer.low.watermark"; /** - * Netty based messaging: Sets the backlog value to specify when the channel binds to a local address. + * Netty based messaging: Sets the backlog value to specify when the channel binds to a local + * address. */ @IsInteger @IsPositiveNumber - public static final String STORM_MESSAGING_NETTY_SOCKET_BACKLOG = "storm.messaging.netty.socket.backlog"; + public static final String STORM_MESSAGING_NETTY_SOCKET_BACKLOG = + "storm.messaging.netty.socket.backlog"; /** * Netty based messaging: The # of worker threads for the server. */ @IsInteger @IsPositiveNumber(includeZero = true) - public static final String STORM_MESSAGING_NETTY_SERVER_WORKER_THREADS = "storm.messaging.netty.server_worker_threads"; + public static final String STORM_MESSAGING_NETTY_SERVER_WORKER_THREADS = + "storm.messaging.netty.server_worker_threads"; /** - * If the Netty messaging layer is busy, the Netty client will try to batch message as more as possible up to the size of + * If the Netty messaging layer is busy, the Netty client will try to batch message as more as + * possible up to the size of * STORM_NETTY_MESSAGE_BATCH_SIZE bytes. */ @IsInteger - public static final String STORM_NETTY_MESSAGE_BATCH_SIZE = "storm.messaging.netty.transfer.batch.size"; + public static final String STORM_NETTY_MESSAGE_BATCH_SIZE = + "storm.messaging.netty.transfer.batch.size"; /** * Netty based messaging: The min # of milliseconds that a peer will wait. */ @IsInteger @IsPositiveNumber(includeZero = true) - public static final String STORM_MESSAGING_NETTY_MIN_SLEEP_MS = "storm.messaging.netty.min_wait_ms"; + public static final String STORM_MESSAGING_NETTY_MIN_SLEEP_MS = + "storm.messaging.netty.min_wait_ms"; /** * Netty based messaging: The max # of milliseconds that a peer will wait. */ @IsInteger @IsPositiveNumber(includeZero = true) - public static final String STORM_MESSAGING_NETTY_MAX_SLEEP_MS = "storm.messaging.netty.max_wait_ms"; + public static final String STORM_MESSAGING_NETTY_MAX_SLEEP_MS = + "storm.messaging.netty.max_wait_ms"; /** * Netty based messaging: The # of worker threads for the client. */ @@ -1950,51 +2330,60 @@ public class Config extends HashMap { * Netty based messaging: Enables TLS connections between workers. */ @IsBoolean - public static final String STORM_MESSAGING_NETTY_TLS_ENABLE = "storm.messaging.netty.tls.enable"; + public static final String STORM_MESSAGING_NETTY_TLS_ENABLE = + "storm.messaging.netty.tls.enable"; /** - * Netty based messaging: When using TLS connections, adds validation that open SSL libraries are available. + * Netty based messaging: When using TLS connections, adds validation that open SSL libraries + * are available. */ @IsBoolean - public static final String STORM_MESSAGING_NETTY_TLS_REQUIRE_OPEN_SSL = "storm.messaging.netty.tls.require.open.ssl"; + public static final String STORM_MESSAGING_NETTY_TLS_REQUIRE_OPEN_SSL = + "storm.messaging.netty.tls.require.open.ssl"; /** * Netty based messaging: Specifies the TLS ciphers to be used when enabled. */ @IsStringOrStringList - public static final String STORM_MESSAGING_NETTY_TLS_CIPHERS = "storm.messaging.netty.tls.ciphers"; + public static final String STORM_MESSAGING_NETTY_TLS_CIPHERS = + "storm.messaging.netty.tls.ciphers"; /** * Netty based messaging: Specifies the truststore when TLS is enabled. */ @IsString - public static final String STORM_MESSAGING_NETTY_TLS_TRUSTSTORE_PATH = "storm.messaging.netty.tls.truststore.path"; + public static final String STORM_MESSAGING_NETTY_TLS_TRUSTSTORE_PATH = + "storm.messaging.netty.tls.truststore.path"; /** * Netty based messaging: Specifies the truststore password when TLS is enabled. */ @IsString @Password - public static final String STORM_MESSAGING_NETTY_TLS_TRUSTSTORE_PASSWORD = "storm.messaging.netty.tls.truststore.password"; + public static final String STORM_MESSAGING_NETTY_TLS_TRUSTSTORE_PASSWORD = + "storm.messaging.netty.tls.truststore.password"; /** * Netty based messaging: Specifies the keystore when TLS is enabled. */ @IsString - public static final String STORM_MESSAGING_NETTY_TLS_KEYSTORE_PATH = "storm.messaging.netty.tls.keystore.path"; + public static final String STORM_MESSAGING_NETTY_TLS_KEYSTORE_PATH = + "storm.messaging.netty.tls.keystore.path"; /** * Netty based messaging: Specifies the keystore password when TLS is enabled. */ @IsString @Password - public static final String STORM_MESSAGING_NETTY_TLS_KEYSTORE_PASSWORD = "storm.messaging.netty.tls.keystore.password"; + public static final String STORM_MESSAGING_NETTY_TLS_KEYSTORE_PASSWORD = + "storm.messaging.netty.tls.keystore.password"; /** * Netty based messaging: Specifies the client truststore when TLS is enabled. */ @IsString - public static final String STORM_MESSAGING_NETTY_TLS_CLIENT_TRUSTSTORE_PATH = "storm.messaging.netty.tls.client.truststore.path"; + public static final String STORM_MESSAGING_NETTY_TLS_CLIENT_TRUSTSTORE_PATH = + "storm.messaging.netty.tls.client.truststore.path"; /** * Netty based messaging: Specifies the client truststore password when TLS is enabled. @@ -2008,7 +2397,8 @@ public class Config extends HashMap { * Netty based messaging: Specifies the client keystore when TLS is enabled. */ @IsString - public static final String STORM_MESSAGING_NETTY_TLS_CLIENT_KEYSTORE_PATH = "storm.messaging.netty.tls.client.keystore.path"; + public static final String STORM_MESSAGING_NETTY_TLS_CLIENT_KEYSTORE_PATH = + "storm.messaging.netty.tls.client.keystore.path"; /** * Netty based messaging: Specifies the client keystore password when TLS is enabled. @@ -2022,11 +2412,14 @@ public class Config extends HashMap { * Netty based messaging: Specifies the protocols TLS is enabled. */ @IsString - public static final String STORM_MESSAGING_NETTY_TLS_SSL_PROTOCOLS = "storm.messaging.netty.tls.ssl.protocols"; + public static final String STORM_MESSAGING_NETTY_TLS_SSL_PROTOCOLS = + "storm.messaging.netty.tls.ssl.protocols"; /** - * Netty based messaging: Specifies whether the client checks that the server certificate matches the host it is - * connecting to when TLS is enabled. Defaults to true. Set this to false only if the worker certificates in use do + * Netty based messaging: Specifies whether the client checks that the server certificate + * matches the host it is + * connecting to when TLS is enabled. Defaults to true. Set this to false only if the worker + * certificates in use do * not carry a host or IP SAN for the address the workers connect to. */ @IsBoolean @@ -2034,25 +2427,30 @@ public class Config extends HashMap { "storm.messaging.netty.tls.hostnameVerification"; /** - * Netty based messaging: The number of milliseconds that a Netty client will retry flushing messages that are already + * Netty based messaging: The number of milliseconds that a Netty client will retry flushing + * messages that are already * buffered to be sent. */ @IsLong @IsPositiveNumber - public static final String STORM_MESSAGING_NETTY_FLUSH_TIMEOUT_MS = "storm.messaging.netty.flush_timeout_ms"; + public static final String STORM_MESSAGING_NETTY_FLUSH_TIMEOUT_MS = + "storm.messaging.netty.flush_timeout_ms"; /** * Should the supervior try to run the worker as the lauching user or not. Defaults to false. */ @IsBoolean public static final String SUPERVISOR_RUN_WORKER_AS_USER = "supervisor.run.worker.as.user"; /** - * max timeout for supervisor reported heartbeats when master gains leadership. + * Max timeout for supervisor reported heartbeats when master gains leadership. */ @IsInteger - public static final String SUPERVISOR_WORKER_HEARTBEATS_MAX_TIMEOUT_SECS = "supervisor.worker.heartbeats.max.timeout.secs"; + public static final String SUPERVISOR_WORKER_HEARTBEATS_MAX_TIMEOUT_SECS = + "supervisor.worker.heartbeats.max.timeout.secs"; /** - * On some systems (windows for example) symlinks require special privileges that not everyone wants to grant a headless user. You can - * completely disable the use of symlinks by setting this config to true, but by doing so you may also lose some features from storm. + * On some systems (windows for example) symlinks require special privileges that not everyone + * wants to grant a headless user. You can + * completely disable the use of symlinks by setting this config to true, but by doing so you + * may also lose some features from storm. * For example the blobstore feature does not currently work without symlinks enabled. */ @IsBoolean @@ -2066,32 +2464,38 @@ public class Config extends HashMap { * The plugin that will provide user groups service. */ @IsString - public static final String STORM_GROUP_MAPPING_SERVICE_PROVIDER_PLUGIN = "storm.group.mapping.service"; + public static final String STORM_GROUP_MAPPING_SERVICE_PROVIDER_PLUGIN = + "storm.group.mapping.service"; /** * A list of credential renewers that nimbus should load. */ @IsStringList public static final String NIMBUS_CREDENTIAL_RENEWERS = "nimbus.credential.renewers.classes"; /** - * A list of plugins that nimbus should load during submit topology to populate credentials on user's behalf. + * A list of plugins that nimbus should load during submit topology to populate credentials on + * user's behalf. */ @IsStringList public static final String NIMBUS_AUTO_CRED_PLUGINS = "nimbus.autocredential.plugins.classes"; /** - * A list of users that run the supervisors and should be authorized to interact with nimbus as a supervisor would. To use this set + * A list of users that run the supervisors and should be authorized to interact with nimbus as + * a supervisor would. To use this set * nimbus.authorizer to org.apache.storm.security.auth.authorizer.SimpleACLAuthorizer. */ @IsStringList public static final String NIMBUS_SUPERVISOR_USERS = "nimbus.supervisor.users"; /** - * A list of users that nimbus runs as and should be authorized to interact with the supervisor as nimbus would. To use this set - * supervisor.authorizer to org.apache.storm.security.auth.authorizer.SupervisorSimpleACLAuthorizer. + * A list of users that nimbus runs as and should be authorized to interact with the supervisor + * as nimbus would. To use this set + * supervisor.authorizer to + * org.apache.storm.security.auth.authorizer.SupervisorSimpleACLAuthorizer. */ @IsStringList public static final String NIMBUS_DAEMON_USERS = "nimbus.daemon.users"; /** - * A list of users that are cluster admins and can run any command. To use this set nimbus.authorizer to + * A list of users that are cluster admins and can run any command. To use this set + * nimbus.authorizer to * org.apache.storm.security.auth.authorizer.SimpleACLAuthorizer */ @IsStringList @@ -2102,16 +2506,19 @@ public class Config extends HashMap { @IsStringList public static final String NIMBUS_ADMINS_GROUPS = "nimbus.admins.groups"; /** - * For secure mode we would want to turn on this config By default this is turned off assuming the default is insecure. + * For secure mode we would want to turn on this config By default this is turned off assuming + * the default is insecure. */ @IsBoolean - public static final String STORM_BLOBSTORE_ACL_VALIDATION_ENABLED = "storm.blobstore.acl.validation.enabled"; + public static final String STORM_BLOBSTORE_ACL_VALIDATION_ENABLED = + "storm.blobstore.acl.validation.enabled"; /** * What buffer size to use for the blobstore uploads. */ @IsPositiveNumber @IsInteger - public static final String STORM_BLOBSTORE_INPUTSTREAM_BUFFER_SIZE_BYTES = "storm.blobstore.inputstream.buffer.size.bytes"; + public static final String STORM_BLOBSTORE_INPUTSTREAM_BUFFER_SIZE_BYTES = + "storm.blobstore.inputstream.buffer.size.bytes"; /** * What chunk size to use for storm client to upload dependency jars. */ @@ -2123,7 +2530,8 @@ public class Config extends HashMap { * FQCN of a class that implements {@code ISubmitterHook} @see ISubmitterHook for details. */ @IsString - public static final String STORM_TOPOLOGY_SUBMISSION_NOTIFIER_PLUGIN = "storm.topology.submission.notifier.plugin.class"; + public static final String STORM_TOPOLOGY_SUBMISSION_NOTIFIER_PLUGIN = + "storm.topology.submission.notifier.plugin.class"; /** * Impersonation user ACL config entries. */ @@ -2131,63 +2539,87 @@ public class Config extends HashMap { valueValidatorClasses = {ConfigValidation.ImpersonationAclUserEntryValidator.class}) public static final String NIMBUS_IMPERSONATION_ACL = "nimbus.impersonation.acl"; /** - * A whitelist of the RAS scheduler strategies allowed by nimbus. Should be a list of fully-qualified class names. When it is not - * set only the scheduler strategies shipped with Storm are allowed, so a custom strategy has to be listed here explicitly. A + * A whitelist of the RAS scheduler strategies allowed by nimbus. Should be a list of + * fully-qualified class names. When it is not + * set only the scheduler strategies shipped with Storm are allowed, so a custom strategy has to + * be listed here explicitly. A * topology that selects a strategy which is not allowed is rejected by nimbus. */ @IsStringList - public static final String NIMBUS_SCHEDULER_STRATEGY_CLASS_WHITELIST = "nimbus.scheduler.strategy.class.whitelist"; + public static final String NIMBUS_SCHEDULER_STRATEGY_CLASS_WHITELIST = + "nimbus.scheduler.strategy.class.whitelist"; /** - * Full path to the worker-laucher executable that will be used to lauch workers when SUPERVISOR_RUN_WORKER_AS_USER is set to true. + * Full path to the worker-laucher executable that will be used to lauch workers when + * SUPERVISOR_RUN_WORKER_AS_USER is set to true. */ @IsString public static final String SUPERVISOR_WORKER_LAUNCHER = "supervisor.worker.launcher"; /** - * Map a version of storm to a worker classpath that can be used to run it. This allows the supervisor to select an available version of + * Map a version of storm to a worker classpath that can be used to run it. This allows the + * supervisor to select an available version of * storm that is compatible with what a topology was launched with. * - *

    Only the major and minor version numbers are used, although this may change in the future. The code will + *

    Only the major and minor version numbers are used, although this may change in the future. + * The code will * first try to find a version - * that is the same or higher than the requested version, but with the same major version number. If it cannot it will fall back to - * using one with a lower minor version, but in some cases this might fail as some features may be missing. + * that is the same or higher than the requested version, but with the same major version + * number. If it cannot it will fall back to + * using one with a lower minor version, but in some cases this might fail as some features may + * be missing. * - *

    Because of how this selection process works please don't include two releases with the same major and minor versions as it is - * undefined which will be selected. Also it is good practice to just include one release for each major version you want to support - * unless the minor versions are truly not compatible with each other. This is to avoid maintenance and testing overhead. + *

    Because of how this selection process works please don't include two releases with the + * same major and minor versions as it is + * undefined which will be selected. Also it is good practice to just include one release for + * each major version you want to support + * unless the minor versions are truly not compatible with each other. This is to avoid + * maintenance and testing overhead. * - *

    This config needs to be set on all supervisors and on nimbus. In general this can be the output of calling storm classpath on the - * version you want and adding in an entry for the config directory for that release. You should modify the storm.yaml of each of these + *

    This config needs to be set on all supervisors and on nimbus. In general this can be the + * output of calling storm classpath on the + * version you want and adding in an entry for the config directory for that release. You should + * modify the storm.yaml of each of these * versions to match the features and settings you want on the main version. */ @IsMapEntryType(keyType = String.class, valueType = String.class) - public static final String SUPERVISOR_WORKER_VERSION_CLASSPATH_MAP = "supervisor.worker.version.classpath.map"; + public static final String SUPERVISOR_WORKER_VERSION_CLASSPATH_MAP = + "supervisor.worker.version.classpath.map"; /** - * Map a version of storm to a worker's main class. In most cases storm should have correct defaults and just setting + * Map a version of storm to a worker's main class. In most cases storm should have correct + * defaults and just setting * SUPERVISOR_WORKER_VERSION_CLASSPATH_MAP is enough. */ @IsMapEntryType(keyType = String.class, valueType = String.class) - public static final String SUPERVISOR_WORKER_VERSION_MAIN_MAP = "supervisor.worker.version.main.map"; + public static final String SUPERVISOR_WORKER_VERSION_MAIN_MAP = + "supervisor.worker.version.main.map"; /** - * Map a version of storm to a worker's logwriter class. In most cases storm should have correct defaults and just setting + * Map a version of storm to a worker's logwriter class. In most cases storm should have correct + * defaults and just setting * SUPERVISOR_WORKER_VERSION_CLASSPATH_MAP is enough. */ @IsMapEntryType(keyType = String.class, valueType = String.class) - public static final String SUPERVISOR_WORKER_VERSION_LOGWRITER_MAP = "supervisor.worker.version.logwriter.map"; + public static final String SUPERVISOR_WORKER_VERSION_LOGWRITER_MAP = + "supervisor.worker.version.logwriter.map"; /** - * The version of storm to assume a topology should run as if not version is given by the client when submitting the topology. + * The version of storm to assume a topology should run as if not version is given by the client + * when submitting the topology. */ @IsString - public static final String SUPERVISOR_WORKER_DEFAULT_VERSION = "supervisor.worker.default.version"; + public static final String SUPERVISOR_WORKER_DEFAULT_VERSION = + "supervisor.worker.default.version"; /** - * A directory on the local filesystem used by Storm for any local filesystem usage it needs. The directory must exist and the Storm - * daemons must have permission to read/write from this location. It could be either absolute or relative. If the setting is a relative + * A directory on the local filesystem used by Storm for any local filesystem usage it needs. + * The directory must exist and the Storm + * daemons must have permission to read/write from this location. It could be either absolute or + * relative. If the setting is a relative * directory, it is relative to root directory of Storm installation. */ @IsString public static final String STORM_LOCAL_DIR = "storm.local.dir"; /** - * The workers-artifacts directory (where we place all workers' logs), can be either absolute or relative. By default, - * ${storm.log.dir}/workers-artifacts is where worker logs go. If the setting is a relative directory, it is relative to storm.log.dir. + * The workers-artifacts directory (where we place all workers' logs), can be either absolute or + * relative. By default, + * ${storm.log.dir}/workers-artifacts is where worker logs go. If the setting is a relative + * directory, it is relative to storm.log.dir. */ @IsString public static final String STORM_WORKERS_ARTIFACTS_DIR = "storm.workers.artifacts.dir"; @@ -2195,7 +2627,8 @@ public class Config extends HashMap { * The connection timeout for clients to ZooKeeper. */ @IsInteger - public static final String STORM_ZOOKEEPER_CONNECTION_TIMEOUT = "storm.zookeeper.connection.timeout"; + public static final String STORM_ZOOKEEPER_CONNECTION_TIMEOUT = + "storm.zookeeper.connection.timeout"; /** * The session timeout for clients to ZooKeeper. */ @@ -2210,14 +2643,16 @@ public class Config extends HashMap { * The ceiling of the interval between retries of a Zookeeper operation. */ @IsInteger - public static final String STORM_ZOOKEEPER_RETRY_INTERVAL_CEILING = "storm.zookeeper.retry.intervalceiling.millis"; + public static final String STORM_ZOOKEEPER_RETRY_INTERVAL_CEILING = + "storm.zookeeper.retry.intervalceiling.millis"; /** * The number of times to retry a Zookeeper operation. */ @IsInteger public static final String STORM_ZOOKEEPER_RETRY_TIMES = "storm.zookeeper.retry.times"; /** - * The ClusterState factory that worker will use to create a ClusterState to store state in. Defaults to ZooKeeper. + * The ClusterState factory that worker will use to create a ClusterState to store state in. + * Defaults to ZooKeeper. */ @IsString public static final String STORM_CLUSTER_STATE_STORE = "storm.cluster.state.store"; @@ -2241,8 +2676,10 @@ public class Config extends HashMap { @IsPositiveNumber public static final String TASK_HEARTBEAT_FREQUENCY_SECS = "task.heartbeat.frequency.secs"; /** - * How often a task should sync its connections with other tasks (if a task is reassigned, the other tasks sending messages to it need - * to refresh their connections). In general though, when a reassignment happens other tasks will be notified almost immediately. This + * How often a task should sync its connections with other tasks (if a task is reassigned, the + * other tasks sending messages to it need + * to refresh their connections). In general though, when a reassignment happens other tasks + * will be notified almost immediately. This * configuration is here just in case that notification doesn't come through. */ @IsInteger @@ -2263,9 +2700,12 @@ public class Config extends HashMap { @IsString public static final String DRPC_AUTHORIZER_ACL_FILENAME = "drpc.authorizer.acl.filename"; /** - * Whether the DRPCSimpleAclAuthorizer should deny requests for operations involving functions that have no explicit ACL entry. When set - * to false (the default) DRPC functions that have no entry in the ACL will be permitted, which is appropriate for a development - * environment. When set to true, explicit ACL entries are required for every DRPC function, and any request for functions will be + * Whether the DRPCSimpleAclAuthorizer should deny requests for operations involving functions + * that have no explicit ACL entry. When set + * to false (the default) DRPC functions that have no entry in the ACL will be permitted, which + * is appropriate for a development + * environment. When set to true, explicit ACL entries are required for every DRPC function, and + * any request for functions will be * denied. * * @see org.apache.storm.security.auth.authorizer.DRPCSimpleACLAuthorizer @@ -2273,7 +2713,7 @@ public class Config extends HashMap { @IsBoolean public static final String DRPC_AUTHORIZER_ACL_STRICT = "drpc.authorizer.acl.strict"; /** - * root directory of the storm cgroup hierarchy. + * Root directory of the storm cgroup hierarchy. */ @IsString public static final String STORM_CGROUP_HIERARCHY_DIR = "storm.cgroup.hierarchy.dir"; @@ -2295,37 +2735,46 @@ public class Config extends HashMap { @IsString public static String TOPOLOGY_OCI_IMAGE = "topology.oci.image"; /** - * Interval to check for the worker to check for updated blobs and refresh worker state accordingly. The default is 10 seconds + * Interval to check for the worker to check for updated blobs and refresh worker state + * accordingly. The default is 10 seconds */ @IsInteger @IsPositiveNumber - public static final String WORKER_BLOB_UPDATE_POLL_INTERVAL_SECS = "worker.blob.update.poll.interval.secs"; + public static final String WORKER_BLOB_UPDATE_POLL_INTERVAL_SECS = + "worker.blob.update.poll.interval.secs"; /** - * Specify the Locale for daemon metrics reporter plugin. Use the specified IETF BCP 47 language tag string for a Locale. - * This config should have been placed in the DaemonConfig class since it is intended only for use by daemons. + * Specify the Locale for daemon metrics reporter plugin. Use the specified IETF BCP 47 language + * tag string for a Locale. + * This config should have been placed in the DaemonConfig class since it is intended only for + * use by daemons. * Keeping it here only for backwards compatibility. */ @IsString - public static final String STORM_DAEMON_METRICS_REPORTER_PLUGIN_LOCALE = "storm.daemon.metrics.reporter.plugin.locale"; + public static final String STORM_DAEMON_METRICS_REPORTER_PLUGIN_LOCALE = + "storm.daemon.metrics.reporter.plugin.locale"; /** * Specify the rate unit in TimeUnit for daemon metrics reporter plugin. - * This config should have been placed in the DaemonConfig class since it is intended only for use by daemons. + * This config should have been placed in the DaemonConfig class since it is intended only for + * use by daemons. * Keeping it here only for backwards compatibility. */ @IsString - public static final String STORM_DAEMON_METRICS_REPORTER_PLUGIN_RATE_UNIT = "storm.daemon.metrics.reporter.plugin.rate.unit"; + public static final String STORM_DAEMON_METRICS_REPORTER_PLUGIN_RATE_UNIT = + "storm.daemon.metrics.reporter.plugin.rate.unit"; /** * Specify the duration unit in TimeUnit for daemon metrics reporter plugin. - * This config should have been placed in the DaemonConfig class since it is intended only for use by daemons. + * This config should have been placed in the DaemonConfig class since it is intended only for + * use by daemons. * Keeping it here only for backwards compatibility. */ @IsString - public static final String STORM_DAEMON_METRICS_REPORTER_PLUGIN_DURATION_UNIT = "storm.daemon.metrics.reporter.plugin.duration.unit"; + public static final String STORM_DAEMON_METRICS_REPORTER_PLUGIN_DURATION_UNIT = + "storm.daemon.metrics.reporter.plugin.duration.unit"; - //DO NOT CHANGE UNLESS WE ADD IN STATE NOT STORED IN THE PARENT CLASS + // DO NOT CHANGE UNLESS WE ADD IN STATE NOT STORED IN THE PARENT CLASS private static final long serialVersionUID = -1550278723792864455L; public static void setClasspath(Map conf, String cp) { @@ -2364,18 +2813,21 @@ public static void registerSerialization(Map conf, Class klass) getRegisteredSerializations(conf).add(klass.getName()); } - public static void registerSerialization(Map conf, Class klass, Class serializerClass) { + public static void registerSerialization(Map conf, Class klass, + Class serializerClass) { Map register = new HashMap(); register.put(klass.getName(), serializerClass.getName()); getRegisteredSerializations(conf).add(register); } - public static void registerEventLogger(Map conf, Class klass, Map argument) { + public static void registerEventLogger(Map conf, + Class klass, Map argument) { Map m = new HashMap<>(); m.put("class", klass.getCanonicalName()); m.put("arguments", argument); - List> l = (List>) conf.get(TOPOLOGY_EVENT_LOGGER_REGISTER); + List> l = (List>) conf + .get(TOPOLOGY_EVENT_LOGGER_REGISTER); if (l == null) { l = new ArrayList<>(); } @@ -2384,11 +2836,13 @@ public static void registerEventLogger(Map conf, Class conf, Class klass) { + public static void registerEventLogger(Map conf, + Class klass) { registerEventLogger(conf, klass, null); } - public static void registerMetricsConsumer(Map conf, Class klass, Object argument, long parallelismHint) { + public static void registerMetricsConsumer(Map conf, Class klass, + Object argument, long parallelismHint) { HashMap m = new HashMap<>(); m.put("class", klass.getCanonicalName()); m.put("parallelism.hint", parallelismHint); @@ -2402,7 +2856,8 @@ public static void registerMetricsConsumer(Map conf, Class klass conf.put(TOPOLOGY_METRICS_CONSUMER_REGISTER, l); } - public static void registerMetricsConsumer(Map conf, Class klass, long parallelismHint) { + public static void registerMetricsConsumer(Map conf, Class klass, + long parallelismHint) { registerMetricsConsumer(conf, klass, null, parallelismHint); } @@ -2410,11 +2865,13 @@ public static void registerMetricsConsumer(Map conf, Class klass registerMetricsConsumer(conf, klass, null, 1L); } - public static void registerDecorator(Map conf, Class klass) { + public static void registerDecorator(Map conf, + Class klass) { getRegisteredDecorators(conf).add(klass.getName()); } - public static void setKryoFactory(Map conf, Class klass) { + public static void setKryoFactory(Map conf, + Class klass) { conf.put(Config.TOPOLOGY_KRYO_FACTORY, klass.getName()); } @@ -2510,7 +2967,8 @@ public void registerSerialization(Class klass, Class seria } @SuppressWarnings("checkstyle:OverloadMethodsDeclarationOrder") - public void registerEventLogger(Class klass, Map argument) { + public void registerEventLogger(Class klass, Map argument) { registerEventLogger(this, klass, argument); } @@ -2578,9 +3036,12 @@ public void setTopologyWorkerMaxHeapSize(Number size) { } /** - * Declares executors of component1 cannot be on the same worker as executors of component2. This function is additive. Thus a user can - * setTopologyComponentWorkerConstraints("A", "B") and then setTopologyComponentWorkerConstraints("B", "C") Which means executors form - * component A cannot be on the same worker with executors of component B and executors of Component B cannot be on workers with + * Declares executors of component1 cannot be on the same worker as executors of component2. + * This function is additive. Thus a user can + * setTopologyComponentWorkerConstraints("A", "B") and then + * setTopologyComponentWorkerConstraints("B", "C") Which means executors form + * component A cannot be on the same worker with executors of component B and executors of + * Component B cannot be on workers with * executors of component C * * @param component1 a component that should not coexist with component2 @@ -2589,8 +3050,9 @@ public void setTopologyWorkerMaxHeapSize(Number size) { public void setTopologyComponentWorkerConstraints(String component1, String component2) { if (component1 != null && component2 != null) { List constraintPair = Arrays.asList(component1, component2); - List> constraints = (List>) computeIfAbsent(Config.TOPOLOGY_RAS_CONSTRAINTS, - (k) -> new ArrayList<>(1)); + List> constraints = + (List>) computeIfAbsent(Config.TOPOLOGY_RAS_CONSTRAINTS, + (k) -> new ArrayList<>(1)); constraints.add(constraintPair); } } @@ -2654,7 +3116,7 @@ public static String getHdfsPrincipal(Map conf) throws UnknownHo Config.BLOBSTORE_HDFS_PRINCIPAL, Config.STORM_HDFS_LOGIN_PRINCIPAL); ret = blobstorePrincipal; } else { - //both not null; + // both not null; LOG.warn("Both {} and {} are set. Use {} only.", Config.BLOBSTORE_HDFS_PRINCIPAL, Config.STORM_HDFS_LOGIN_PRINCIPAL, Config.STORM_HDFS_LOGIN_PRINCIPAL); ret = hdfsPrincipal; @@ -2682,7 +3144,7 @@ public static String getHdfsKeytab(Map conf) { Config.BLOBSTORE_HDFS_KEYTAB, Config.STORM_HDFS_LOGIN_KEYTAB); ret = blobstoreKeyTab; } else { - //both not null; + // both not null; LOG.warn("Both {} and {} are set. Use {} only.", Config.BLOBSTORE_HDFS_KEYTAB, Config.STORM_HDFS_LOGIN_KEYTAB, Config.STORM_HDFS_LOGIN_KEYTAB); ret = hdfsKeyTab; diff --git a/storm-client/src/jvm/org/apache/storm/Constants.java b/storm-client/src/jvm/org/apache/storm/Constants.java index 72abcb74a59..fa36ce3ff88 100644 --- a/storm-client/src/jvm/org/apache/storm/Constants.java +++ b/storm-client/src/jvm/org/apache/storm/Constants.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -17,9 +23,9 @@ import java.util.Set; import org.apache.storm.coordination.CoordinatedBolt; - public class Constants { - public static final String COORDINATED_STREAM_ID = CoordinatedBolt.class.getName() + "/coord-stream"; + public static final String COORDINATED_STREAM_ID = CoordinatedBolt.class.getName() + + "/coord-stream"; public static final long SYSTEM_TASK_ID = -1; public static final List SYSTEM_EXECUTOR_ID = Arrays.asList(-1L, -1L); @@ -33,19 +39,25 @@ public class Constants { public static final String FEEDBACK_TICK_STREAM_ID = "__feedback_tick"; /** - * System streams carrying low-volume, time-driven control signals that may be routed to the executor receive + * System streams carrying low-volume, time-driven control signals that may be routed to the + * executor receive * queue's control lane (see {@code Config.TOPOLOGY_EXECUTOR_RECEIVE_CONTROL_QUEUE_ENABLE}). Tuples on these - * streams are wall-clock signals with no ordering contract against data tuples, so delivering them ahead of - * co-enqueued data tuples is safe, and dropping one when the control lane is full is self-healing because the + * streams are wall-clock signals with no ordering contract against data tuples, so delivering + * them ahead of + * co-enqueued data tuples is safe, and dropping one when the control lane is full is + * self-healing because the * next one arrives within the signal's period. */ public static final Set SYSTEM_CONTROL_STREAM_IDS = - Set.of(SYSTEM_TICK_STREAM_ID, SYSTEM_FLUSH_STREAM_ID, METRICS_TICK_STREAM_ID, FEEDBACK_STREAM_ID, FEEDBACK_TICK_STREAM_ID); + Set.of(SYSTEM_TICK_STREAM_ID, SYSTEM_FLUSH_STREAM_ID, METRICS_TICK_STREAM_ID, + FEEDBACK_STREAM_ID, FEEDBACK_TICK_STREAM_ID); public static boolean isControlStreamId(String streamId) { - // All control stream ids start with "__"; the prefix check cheaply rejects the common data-stream + // All control stream ids start with "__"; the prefix check cheaply rejects the common + // data-stream // case before falling through to the set lookup. - return streamId != null && streamId.startsWith("__") && SYSTEM_CONTROL_STREAM_IDS.contains(streamId); + return streamId != null && streamId.startsWith("__") && SYSTEM_CONTROL_STREAM_IDS + .contains(streamId); } public static final Object TOPOLOGY = "topology"; @@ -74,8 +86,10 @@ public static boolean isControlStreamId(String streamId) { public static final String COMMON_OFFHEAP_MEMORY_RESOURCE_NAME = "offheap.memory.mb"; public static final String COMMON_TOTAL_MEMORY_RESOURCE_NAME = "memory.mb"; - public static final String NIMBUS_SEND_ASSIGNMENT_EXCEPTIONS = "nimbus:num-send-assignment-exceptions"; - public static final String SUPERVISOR_HEALTH_CHECK_TIMEOUTS = "supervisor:health-check-timeouts"; + public static final String NIMBUS_SEND_ASSIGNMENT_EXCEPTIONS = + "nimbus:num-send-assignment-exceptions"; + public static final String SUPERVISOR_HEALTH_CHECK_TIMEOUTS = + "supervisor:health-check-timeouts"; public static final String WORKER_METRICS_REGISTRY = "worker-metrics-registry"; } diff --git a/storm-client/src/jvm/org/apache/storm/ICredentialsListener.java b/storm-client/src/jvm/org/apache/storm/ICredentialsListener.java index bb25b6a754c..e3a5fcf42f0 100644 --- a/storm-client/src/jvm/org/apache/storm/ICredentialsListener.java +++ b/storm-client/src/jvm/org/apache/storm/ICredentialsListener.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/ILocalCluster.java b/storm-client/src/jvm/org/apache/storm/ILocalCluster.java index 8e7cfa8f4fe..927eef0d7ec 100644 --- a/storm-client/src/jvm/org/apache/storm/ILocalCluster.java +++ b/storm-client/src/jvm/org/apache/storm/ILocalCluster.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -40,7 +46,8 @@ public interface ILocalCluster extends AutoCloseable { * @return an AutoCloseable that will kill the topology. * @throws TException on any error from nimbus */ - ILocalTopology submitTopology(String topologyName, Map conf, StormTopology topology) throws TException; + ILocalTopology submitTopology(String topologyName, Map conf, + StormTopology topology) throws TException; /** * Submit a topology to be run in local mode. @@ -52,7 +59,8 @@ public interface ILocalCluster extends AutoCloseable { * @return an AutoCloseable that will kill the topology. * @throws TException on any error from nimbus */ - ILocalTopology submitTopologyWithOpts(String topologyName, Map conf, StormTopology topology, + ILocalTopology submitTopologyWithOpts(String topologyName, Map conf, + StormTopology topology, SubmitOptions submitOpts) throws TException; /** @@ -134,6 +142,7 @@ ILocalTopology submitTopologyWithOpts(String topologyName, Map c /** * Get cluster information. + * * @return a summary of the current state of the cluster * @throws TException on any error from nimbus */ @@ -181,7 +190,8 @@ ILocalTopology submitTopologyWithOpts(String topologyName, Map c * @return the state of a topology * @throws TException on any error from nimbus */ - TopologyInfo getTopologyInfoByNameWithOpts(String name, GetInfoOptions options) throws TException; + TopologyInfo getTopologyInfoByNameWithOpts(String name, + GetInfoOptions options) throws TException; /** * This is intended for internal testing only. @@ -191,14 +201,16 @@ ILocalTopology submitTopologyWithOpts(String topologyName, Map c IStormClusterState getClusterState(); /** - * Advance the cluster time when the cluster is using SimulatedTime. This is intended for internal testing only. + * Advance the cluster time when the cluster is using SimulatedTime. This is intended for + * internal testing only. * * @param secs the number of seconds to advance time */ void advanceClusterTime(int secs) throws InterruptedException; /** - * Advance the cluster time when the cluster is using SimulatedTime. This is intended for internal testing only. + * Advance the cluster time when the cluster is using SimulatedTime. This is intended for + * internal testing only. * * @param secs the number of seconds to advance time * @param steps the number of steps we should take when advancing simulated time @@ -206,7 +218,8 @@ ILocalTopology submitTopologyWithOpts(String topologyName, Map c void advanceClusterTime(int secs, int steps) throws InterruptedException; /** - * If the cluster is tracked get the id for the tracked cluster. This is intended for internal testing only. + * If the cluster is tracked get the id for the tracked cluster. This is intended for internal + * testing only. * * @return the id of the tracked cluster */ diff --git a/storm-client/src/jvm/org/apache/storm/ILocalDRPC.java b/storm-client/src/jvm/org/apache/storm/ILocalDRPC.java index 037239ba14d..f91f13ea707 100644 --- a/storm-client/src/jvm/org/apache/storm/ILocalDRPC.java +++ b/storm-client/src/jvm/org/apache/storm/ILocalDRPC.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -19,12 +25,14 @@ @SuppressWarnings("checkstyle:AbbreviationAsWordInName") public interface ILocalDRPC extends DistributedRPC.Iface, DistributedRPCInvocations.Iface, Shutdownable, AutoCloseable { /** - * Get the ID of the service. This is used internally if multiple local DRPC clusters are in use at one time. + * Get the ID of the service. This is used internally if multiple local DRPC clusters are in use + * at one time. */ String getServiceId(); /** * Shutdown. + * * @deprecated use {@link #close()} instead */ @Deprecated diff --git a/storm-client/src/jvm/org/apache/storm/ISubmitterHook.java b/storm-client/src/jvm/org/apache/storm/ISubmitterHook.java index b4f9268e73a..a2e7af13cd9 100644 --- a/storm-client/src/jvm/org/apache/storm/ISubmitterHook.java +++ b/storm-client/src/jvm/org/apache/storm/ISubmitterHook.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -17,9 +23,12 @@ import org.apache.storm.generated.TopologyInfo; /** - * if FQCN of an implementation of this class is specified by setting the config storm.topology.submission.notifier.plugin.class, that - * class's notify method will be invoked when a topology is successfully submitted via StormSubmitter class. + * If FQCN of an implementation of this class is specified by setting the config + * storm.topology.submission.notifier.plugin.class, that + * class's notify method will be invoked when a topology is successfully submitted via + * StormSubmitter class. */ public interface ISubmitterHook { - void notify(TopologyInfo topologyInfo, Map topoConf, StormTopology topology) throws IllegalAccessException; + void notify(TopologyInfo topologyInfo, Map topoConf, + StormTopology topology) throws IllegalAccessException; } diff --git a/storm-client/src/jvm/org/apache/storm/LogWriter.java b/storm-client/src/jvm/org/apache/storm/LogWriter.java index 0c032191c91..b8ac8adf6d4 100644 --- a/storm-client/src/jvm/org/apache/storm/LogWriter.java +++ b/storm-client/src/jvm/org/apache/storm/LogWriter.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -19,7 +25,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; - /** * Launch a sub process and write files out to logs. */ diff --git a/storm-client/src/jvm/org/apache/storm/StormSubmitter.java b/storm-client/src/jvm/org/apache/storm/StormSubmitter.java index d522fd95bfa..aae2232a3cc 100644 --- a/storm-client/src/jvm/org/apache/storm/StormSubmitter.java +++ b/storm-client/src/jvm/org/apache/storm/StormSubmitter.java @@ -26,7 +26,6 @@ import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; - import org.apache.storm.dependency.DependencyPropertiesParser; import org.apache.storm.dependency.DependencyUploader; import org.apache.storm.generated.AlreadyAliveException; @@ -54,7 +53,8 @@ import org.slf4j.LoggerFactory; /** - * Use this class to submit topologies to run on the Storm cluster. You should run your program with the "storm jar" command from the + * Use this class to submit topologies to run on the Storm cluster. You should run your program with + * the "storm jar" command from the * command-line, and then use this class to submit your topologies. */ public class StormSubmitter { @@ -81,7 +81,8 @@ public static Map prepareZookeeperAuthentication(Map prepareZookeeperAuthentication(Map populateCredentials(Map conf, Map creds) { + private static Map populateCredentials(Map conf, Map creds) { Map ret = new HashMap<>(); for (IAutoCredentials autoCred : ClientAuthUtils.getAutoCredentials(conf)) { LOG.info("Running " + autoCred); @@ -114,7 +116,8 @@ private static Map populateCredentials(Map conf, * @throws NotAliveException if the topology is not alive * @throws InvalidTopologyException if any other error happens */ - public static boolean pushCredentials(String name, Map topoConf, Map credentials) + public static boolean pushCredentials(String name, Map topoConf, Map credentials) throws AuthorizationException, NotAliveException, InvalidTopologyException { return pushCredentials(name, topoConf, credentials, null); } @@ -132,7 +135,8 @@ public static boolean pushCredentials(String name, Map topoConf, * @throws NotAliveException if the topology is not alive * @throws InvalidTopologyException if any other error happens */ - public static boolean pushCredentials(String name, Map topoConf, Map credentials, String expectedUser) + public static boolean pushCredentials(String name, Map topoConf, Map credentials, String expectedUser) throws AuthorizationException, NotAliveException, InvalidTopologyException { topoConf = new HashMap<>(topoConf); topoConf.putAll(Utils.readCommandLineOpts()); @@ -159,7 +163,6 @@ public static boolean pushCredentials(String name, Map topoConf, return true; } - /** * Submits a topology to run on the cluster. A topology runs forever or until explicitly killed. * @@ -169,9 +172,11 @@ public static boolean pushCredentials(String name, Map topoConf, * @throws AlreadyAliveException if a topology with this name is already running * @throws InvalidTopologyException if an invalid topology was submitted * @throws AuthorizationException if authorization is failed - * @throws SubmitterHookException if any Exception occurs during initialization or invocation of registered {@link ISubmitterHook} + * @throws SubmitterHookException if any Exception occurs during initialization or invocation of + * registered {@link ISubmitterHook} */ - public static void submitTopology(String name, Map topoConf, StormTopology topology) + public static void submitTopology(String name, Map topoConf, + StormTopology topology) throws AlreadyAliveException, InvalidTopologyException, AuthorizationException { submitTopology(name, topoConf, topology, null, null); } @@ -186,9 +191,11 @@ public static void submitTopology(String name, Map topoConf, Sto * @throws AlreadyAliveException if a topology with this name is already running * @throws InvalidTopologyException if an invalid topology was submitted * @throws AuthorizationException if authorization is failed - * @throws SubmitterHookException if any Exception occurs during initialization or invocation of registered {@link ISubmitterHook} + * @throws SubmitterHookException if any Exception occurs during initialization or invocation of + * registered {@link ISubmitterHook} */ - public static void submitTopology(String name, Map topoConf, StormTopology topology, SubmitOptions opts) + public static void submitTopology(String name, Map topoConf, + StormTopology topology, SubmitOptions opts) throws AlreadyAliveException, InvalidTopologyException, AuthorizationException { submitTopology(name, topoConf, topology, opts, null); } @@ -200,31 +207,38 @@ public static void submitTopology(String name, Map topoConf, Sto * @param topoConf the topology-specific configuration. See {@link Config}. * @param topology the processing to execute. * @param opts to manipulate the starting of the topology - * @param progressListener to track the progress of the jar upload process {@link ProgressListener} + * @param progressListener to track the progress of the jar upload process {@link + * ProgressListener} * @throws AlreadyAliveException if a topology with this name is already running * @throws InvalidTopologyException if an invalid topology was submitted * @throws AuthorizationException if authorization is failed - * @throws SubmitterHookException if any Exception occurs during initialization or invocation of registered {@link ISubmitterHook} + * @throws SubmitterHookException if any Exception occurs during initialization or invocation of + * registered {@link ISubmitterHook} */ @SuppressWarnings("unchecked") - public static void submitTopology(String name, Map topoConf, StormTopology topology, SubmitOptions opts, + public static void submitTopology(String name, Map topoConf, + StormTopology topology, SubmitOptions opts, ProgressListener progressListener) throws AlreadyAliveException, InvalidTopologyException, AuthorizationException { submitTopologyAs(name, topoConf, topology, opts, progressListener, null); } /** - * Submits a topology to run on the cluster as a particular user. A topology runs forever or until explicitly killed. + * Submits a topology to run on the cluster as a particular user. A topology runs forever or + * until explicitly killed. * * @param asUser The user as which this topology should be submitted. - * @throws IllegalArgumentException thrown if configs will yield an unschedulable topology. validateConfs validates confs - * @throws SubmitterHookException if any Exception occurs during initialization or invocation of registered {@link ISubmitterHook} + * @throws IllegalArgumentException thrown if configs will yield an unschedulable topology. + * validateConfs validates confs + * @throws SubmitterHookException if any Exception occurs during initialization or invocation of + * registered {@link ISubmitterHook} */ - public static void submitTopologyAs(String name, Map topoConf, StormTopology topology, SubmitOptions opts, + public static void submitTopologyAs(String name, Map topoConf, + StormTopology topology, SubmitOptions opts, ProgressListener progressListener, String asUser) throws AlreadyAliveException, InvalidTopologyException, AuthorizationException, IllegalArgumentException { - //validate topology name first; nothing else should be done if it's invalid. + // validate topology name first; nothing else should be done if it's invalid. Utils.validateTopologyName(name); if (!Utils.isValidConf(topoConf)) { @@ -232,7 +246,8 @@ public static void submitTopologyAs(String name, Map topoConf, S } if (topology.get_spouts_size() == 0) { - throw new WrappedInvalidTopologyException("Topology " + name + " does not have any spout"); + throw new WrappedInvalidTopologyException("Topology " + name + + " does not have any spout"); } topoConf = new HashMap<>(topoConf); @@ -268,7 +283,8 @@ public static void submitTopologyAs(String name, Map topoConf, S String maskedSerConf = JSONValue.toJSONString(ConfigUtils.maskCredentials(topoConf)); try (NimbusClient client = NimbusClient.Builder.withConf(conf).asUser(asUser).build()) { if (!isTopologyNameAllowed(name, client)) { - throw new RuntimeException("Topology name " + name + " is either not allowed or it already exists on the cluster"); + throw new RuntimeException("Topology name " + name + + " is either not allowed or it already exists on the cluster"); } // Dependency uploading only makes sense for distributed mode @@ -283,7 +299,8 @@ public static void submitTopologyAs(String name, Map topoConf, S artifactsBlobKeys = uploadDependencyArtifactsToBlobStore(uploader); } catch (Throwable e) { - // every uploaded blob carries a key unique to this submission, and no topology refers to + // every uploaded blob carries a key unique to this submission, and no topology + // refers to // them yet, so nothing else can be using them uploader.deleteBlobs(jarsBlobKeys); uploader.deleteBlobs(artifactsBlobKeys); @@ -293,10 +310,12 @@ public static void submitTopologyAs(String name, Map topoConf, S try { setDependencyBlobsToTopology(topology, jarsBlobKeys, artifactsBlobKeys); - submitTopologyInDistributeMode(name, topology, opts, progressListener, asUser, conf, serConf, + submitTopologyInDistributeMode(name, topology, opts, progressListener, asUser, + conf, serConf, maskedSerConf, client); } catch (AlreadyAliveException | InvalidTopologyException | AuthorizationException e) { - // the topology was rejected, so the blobs it refers to are unreachable; their keys are + // the topology was rejected, so the blobs it refers to are unreachable; their + // keys are // unique to this submission, so nothing else can be using them // Note that we don't handle TException to delete the blobs // because it's safer to leave some blobs instead of topology not running @@ -335,7 +354,8 @@ private static List uploadDependencyArtifactsToBlobStore(DependencyUploa DependencyPropertiesParser propertiesParser = new DependencyPropertiesParser(); String depArtifactsProp = System.getProperty("storm.dependency.artifacts", "{}"); - Map depArtifacts = propertiesParser.parseArtifactsProperties(depArtifactsProp); + Map depArtifacts = propertiesParser + .parseArtifactsProperties(depArtifactsProp); try { return uploader.uploadArtifacts(depArtifacts); @@ -344,19 +364,24 @@ private static List uploadDependencyArtifactsToBlobStore(DependencyUploa } } - private static void setDependencyBlobsToTopology(StormTopology topology, List jarsBlobKeys, List artifactsBlobKeys) { - LOG.info("Dependency Blob keys - jars : {} / artifacts : {}", jarsBlobKeys, artifactsBlobKeys); + private static void setDependencyBlobsToTopology(StormTopology topology, + List jarsBlobKeys, List artifactsBlobKeys) { + LOG.info("Dependency Blob keys - jars : {} / artifacts : {}", jarsBlobKeys, + artifactsBlobKeys); topology.set_dependency_jars(jarsBlobKeys); topology.set_dependency_artifacts(artifactsBlobKeys); } - private static void submitTopologyInDistributeMode(String name, StormTopology topology, SubmitOptions opts, + private static void submitTopologyInDistributeMode(String name, StormTopology topology, + SubmitOptions opts, ProgressListener progressListener, String asUser, Map conf, String serConf, String maskedSerConf, NimbusClient client) throws TException { try { - String jar = submitJarAs(conf, System.getProperty("storm.jar"), progressListener, client); - LOG.info("Submitting topology {} in distributed mode with conf {}", name, maskedSerConf); + String jar = submitJarAs(conf, System.getProperty("storm.jar"), progressListener, + client); + LOG.info("Submitting topology {} in distributed mode with conf {}", name, + maskedSerConf); Utils.addVersions(topology); if (opts != null) { client.getClient().submitTopologyWithOpts(name, jar, serConf, topology, opts); @@ -376,34 +401,44 @@ private static void submitTopologyInDistributeMode(String name, StormTopology to /** * Invoke submitter hook. - * @throws SubmitterHookException This is thrown when any Exception occurs during initialization or invocation of registered {@link + * + * @throws SubmitterHookException This is thrown when any Exception occurs during initialization + * or invocation of registered {@link * ISubmitterHook} */ - private static void invokeSubmitterHook(String name, String asUser, Map topoConf, StormTopology topology) { + private static void invokeSubmitterHook(String name, String asUser, Map topoConf, StormTopology topology) { String submissionNotifierClassName = null; try { if (topoConf.containsKey(Config.STORM_TOPOLOGY_SUBMISSION_NOTIFIER_PLUGIN)) { - submissionNotifierClassName = topoConf.get(Config.STORM_TOPOLOGY_SUBMISSION_NOTIFIER_PLUGIN).toString(); - LOG.info("Initializing the registered ISubmitterHook [{}]", submissionNotifierClassName); + submissionNotifierClassName = topoConf + .get(Config.STORM_TOPOLOGY_SUBMISSION_NOTIFIER_PLUGIN).toString(); + LOG.info("Initializing the registered ISubmitterHook [{}]", + submissionNotifierClassName); if (submissionNotifierClassName == null || submissionNotifierClassName.isEmpty()) { throw new IllegalArgumentException( - Config.STORM_TOPOLOGY_SUBMISSION_NOTIFIER_PLUGIN + " property must be a non empty string."); + Config.STORM_TOPOLOGY_SUBMISSION_NOTIFIER_PLUGIN + + " property must be a non empty string."); } - ISubmitterHook submitterHook = (ISubmitterHook) Class.forName(submissionNotifierClassName).newInstance(); + ISubmitterHook submitterHook = (ISubmitterHook) Class + .forName(submissionNotifierClassName).newInstance(); TopologyInfo topologyInfo = Utils.getTopologyInfo(name, asUser, topoConf); - LOG.info("Invoking the registered ISubmitterHook [{}]", submissionNotifierClassName); + LOG.info("Invoking the registered ISubmitterHook [{}]", + submissionNotifierClassName); submitterHook.notify(topologyInfo, topoConf, topology); } } catch (Exception e) { - LOG.warn("Error occurred in invoking submitter hook:[{}] ", submissionNotifierClassName, e); + LOG.warn("Error occurred in invoking submitter hook:[{}] ", submissionNotifierClassName, + e); throw new SubmitterHookException(e); } } /** - * Submits a topology to run on the cluster with a progress bar. A topology runs forever or until explicitly killed. + * Submits a topology to run on the cluster with a progress bar. A topology runs forever or + * until explicitly killed. * * @param name the name of the storm. * @param topoConf the topology-specific configuration. See {@link Config}. @@ -413,13 +448,15 @@ private static void invokeSubmitterHook(String name, String asUser, Map topoConf, StormTopology topology) throws + public static void submitTopologyWithProgressBar(String name, Map topoConf, + StormTopology topology) throws AlreadyAliveException, InvalidTopologyException, AuthorizationException { submitTopologyWithProgressBar(name, topoConf, topology, null); } /** - * Submits a topology to run on the cluster with a progress bar. A topology runs forever or until explicitly killed. + * Submits a topology to run on the cluster with a progress bar. A topology runs forever or + * until explicitly killed. * * @param name the name of the storm. * @param topoConf the topology-specific configuration. See {@link Config}. @@ -428,20 +465,24 @@ public static void submitTopologyWithProgressBar(String name, Map topoConf, StormTopology topology, + public static void submitTopologyWithProgressBar(String name, Map topoConf, + StormTopology topology, SubmitOptions opts) throws AlreadyAliveException, InvalidTopologyException, AuthorizationException { // show a progress bar so we know we're not stuck (especially on slow connections) submitTopology(name, topoConf, topology, opts, new StormSubmitter.ProgressListener() { @Override public void onStart(String srcFile, String targetFile, long totalBytes) { - System.out.printf("Start uploading file '%s' to '%s' (%d bytes)\n", srcFile, targetFile, totalBytes); + System.out.printf("Start uploading file '%s' to '%s' (%d bytes)\n", srcFile, + targetFile, totalBytes); } @Override - public void onProgress(String srcFile, String targetFile, long bytesUploaded, long totalBytes) { + public void onProgress(String srcFile, String targetFile, long bytesUploaded, + long totalBytes) { int length = 50; int p = (int) ((length * bytesUploaded) / totalBytes); String progress = StringUtils.repeat("=", p); @@ -452,7 +493,8 @@ public void onProgress(String srcFile, String targetFile, long bytesUploaded, lo @Override public void onCompleted(String srcFile, String targetFile, long totalBytes) { - System.out.printf("\nFile '%s' uploaded to '%s' (%d bytes)\n", srcFile, targetFile, totalBytes); + System.out.printf("\nFile '%s' uploaded to '%s' (%d bytes)\n", srcFile, targetFile, + totalBytes); } }); } @@ -484,19 +526,23 @@ public static String submitJar(Map conf, String localJar) { * @param listener progress listener to track the jar file upload * @return the remote location of the submitted jar */ - public static String submitJar(Map conf, String localJar, ProgressListener listener) { + public static String submitJar(Map conf, String localJar, + ProgressListener listener) { return submitJarAs(conf, localJar, listener, (String) null); } - public static String submitJarAs(Map conf, String localJar, ProgressListener listener, NimbusClient client) { + public static String submitJarAs(Map conf, String localJar, + ProgressListener listener, NimbusClient client) { if (localJar == null) { throw new RuntimeException( - "Must submit topologies using the 'storm' client script so that StormSubmitter knows which jar to upload."); + "Must submit topologies using the 'storm' client script so that StormSubmitter " + + "knows which jar to upload."); } try { String uploadLocation = client.getClient().beginFileUpload(); - LOG.info("Uploading topology jar " + localJar + " to assigned location: " + uploadLocation); + LOG.info("Uploading topology jar " + localJar + " to assigned location: " + + uploadLocation); BufferFileInputStream is = new BufferFileInputStream(localJar, THRIFT_CHUNK_SIZE_BYTES); long totalSize = new File(localJar).length(); @@ -530,10 +576,12 @@ public static String submitJarAs(Map conf, String localJar, Prog } } - public static String submitJarAs(Map conf, String localJar, ProgressListener listener, String asUser) { + public static String submitJarAs(Map conf, String localJar, + ProgressListener listener, String asUser) { if (localJar == null) { throw new RuntimeException( - "Must submit topologies using the 'storm' client script so that StormSubmitter knows which jar to upload."); + "Must submit topologies using the 'storm' client script so that StormSubmitter " + + "knows which jar to upload."); } try (NimbusClient client = NimbusClient.Builder.withConf(conf).asUser(asUser).build()) { @@ -552,7 +600,7 @@ private static void validateConfs(Map topoConf) throws IllegalAr */ public interface ProgressListener { /** - * called before file is uploaded. + * Called before file is uploaded. * * @param srcFile - jar file to be uploaded * @param targetFile - destination file @@ -561,7 +609,7 @@ public interface ProgressListener { void onStart(String srcFile, String targetFile, long totalBytes); /** - * called whenever a chunk of bytes is uploaded. + * Called whenever a chunk of bytes is uploaded. * * @param srcFile - jar file to be uploaded * @param targetFile - destination file @@ -571,7 +619,7 @@ public interface ProgressListener { void onProgress(String srcFile, String targetFile, long bytesUploaded, long totalBytes); /** - * called when the file is uploaded. + * Called when the file is uploaded. * * @param srcFile - jar file to be uploaded * @param targetFile - destination file diff --git a/storm-client/src/jvm/org/apache/storm/StormTimer.java b/storm-client/src/jvm/org/apache/storm/StormTimer.java index be7f7c11424..69873b5a1be 100644 --- a/storm-client/src/jvm/org/apache/storm/StormTimer.java +++ b/storm-client/src/jvm/org/apache/storm/StormTimer.java @@ -1,12 +1,17 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -21,13 +26,14 @@ import org.apache.storm.utils.Utils; /** - * The timer defined in this file is very similar to java.util.Timer, except it integrates with Storm's time simulation capabilities. This + * The timer defined in this file is very similar to java.util.Timer, except it integrates with + * Storm's time simulation capabilities. This * lets us test code that does asynchronous work on the timer thread. */ public class StormTimer implements AutoCloseable { - //task to run + // task to run private final StormTimerTask task = new StormTimerTask(); /** @@ -140,7 +146,8 @@ public void run() { * @param jitterMs jitter added to the run * @param func the function to run */ - public void scheduleRecurringWithJitter(int delaySecs, final int recurSecs, final int jitterMs, final Runnable func) { + public void scheduleRecurringWithJitter(int delaySecs, final int recurSecs, final int jitterMs, + final Runnable func) { schedule(delaySecs, new Runnable() { @Override public void run() { @@ -152,7 +159,7 @@ public void run() { } /** - * check if timer is active. + * Check if timer is active. */ private void checkActive() { if (!this.task.isActive()) { @@ -161,7 +168,7 @@ private void checkActive() { } /** - * cancel timer. + * Cancel timer. */ @Override @@ -174,7 +181,7 @@ public void close() throws InterruptedException { } /** - * is timer waiting. Used in timer simulation. + * Is timer waiting. Used in timer simulation. */ public boolean isTimerWaiting() { return Time.isThreadWaiting(task); @@ -194,13 +201,14 @@ public QueueEntry(Long endTimeMs, Runnable func, String id) { public static class StormTimerTask extends Thread { - //initialCapacity set to 11 since its the default inital capacity of PriorityBlockingQueue - private PriorityBlockingQueue queue = new PriorityBlockingQueue(11, new Comparator() { - @Override + // initialCapacity set to 11 since its the default inital capacity of PriorityBlockingQueue + private PriorityBlockingQueue queue = new PriorityBlockingQueue(11, + new Comparator() { + @Override public int compare(QueueEntry o1, QueueEntry o2) { - return o1.endTimeMs.intValue() - o2.endTimeMs.intValue(); - } - }); + return o1.endTimeMs.intValue() - o2.endTimeMs.intValue(); + } + }); // boolean to indicate whether timer is active private AtomicBoolean active = new AtomicBoolean(false); @@ -208,7 +216,7 @@ public int compare(QueueEntry o1, QueueEntry o2) { // function to call when timer is killed private Thread.UncaughtExceptionHandler onKill; - //random number generator + // random number generator private Random random = new Random(); @Override @@ -217,7 +225,8 @@ public void run() { QueueEntry queueEntry = null; try { queueEntry = this.queue.peek(); - if ((queueEntry != null) && (Time.currentTimeMillis() >= queueEntry.endTimeMs)) { + if ((queueEntry != null) && (Time + .currentTimeMillis() >= queueEntry.endTimeMs)) { // It is imperative to not run the function // inside the timer lock. Otherwise, it is // possible to deadlock if the fn deals with @@ -233,7 +242,8 @@ public void run() { // an upper bound, e.g. 1000 millis, to the // sleeping time, to limit the response time // for detecting any new event within 1 secs. - Time.sleep(Math.min(1000, (queueEntry.endTimeMs - Time.currentTimeMillis()))); + Time.sleep(Math.min(1000, (queueEntry.endTimeMs - Time + .currentTimeMillis()))); } else { // Otherwise poll to see if any new event // was scheduled. This is, in essence, the @@ -247,8 +257,10 @@ public void run() { } } catch (Throwable e) { if (!(Utils.exceptionCauseIsInstanceOf(InterruptedException.class, e)) - && !(Utils.exceptionCauseIsInstanceOf(ClosedByInterruptException.class, e))) { - // need to set active false before calling onKill() - current implementation does not return. + && !(Utils.exceptionCauseIsInstanceOf(ClosedByInterruptException.class, + e))) { + // need to set active false before calling onKill() - current implementation + // does not return. this.setActive(false); this.onKill.uncaughtException(this, e); } diff --git a/storm-client/src/jvm/org/apache/storm/Thrift.java b/storm-client/src/jvm/org/apache/storm/Thrift.java index 79b169a566b..3f87c36afad 100644 --- a/storm-client/src/jvm/org/apache/storm/Thrift.java +++ b/storm-client/src/jvm/org/apache/storm/Thrift.java @@ -24,8 +24,8 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Map.Entry; +import java.util.Map; import java.util.Set; import org.apache.storm.generated.Bolt; import org.apache.storm.generated.ComponentCommon; @@ -37,8 +37,8 @@ import org.apache.storm.generated.NullStruct; import org.apache.storm.generated.SpoutSpec; import org.apache.storm.generated.StateSpoutSpec; -import org.apache.storm.generated.StormTopology; import org.apache.storm.generated.StormTopology._Fields; +import org.apache.storm.generated.StormTopology; import org.apache.storm.generated.StreamInfo; import org.apache.storm.shade.net.minidev.json.JSONValue; import org.apache.storm.task.IBolt; @@ -160,7 +160,8 @@ public static Grouping._Fields groupingType(Grouping grouping) { public static List fieldGrouping(Grouping grouping) { if (!Grouping._Fields.FIELDS.equals(groupingType(grouping))) { - throw new IllegalArgumentException("Tried to get grouping fields from non fields grouping"); + throw new IllegalArgumentException("Tried to get grouping fields from non fields " + + "grouping"); } return grouping.get_fields(); } @@ -192,12 +193,14 @@ public static Object deserializeComponentObject(ComponentObject obj) { return Utils.javaDeserialize(obj.get_serialized_java(), Serializable.class); } - public static ComponentCommon prepareComponentCommon(Map inputs, Map inputs, + Map outputs, Integer parallelismHint) { return prepareComponentCommon(inputs, outputs, parallelismHint, null); } - public static ComponentCommon prepareComponentCommon(Map inputs, Map outputs, + public static ComponentCommon prepareComponentCommon(Map inputs, + Map outputs, Integer parallelismHint, Map conf) { Map mappedInputs = new HashMap<>(); Map mappedOutputs = new HashMap<>(); @@ -217,18 +220,21 @@ public static ComponentCommon prepareComponentCommon(Map outputs) { + public static SpoutSpec prepareSerializedSpoutDetails(IRichSpout spout, Map outputs) { return new SpoutSpec(ComponentObject.serialized_java(Utils.javaSerialize(spout)), prepareComponentCommon(new HashMap<>(), outputs, null, null)); } - public static Bolt prepareSerializedBoltDetails(Map inputs, IBolt bolt, Map outputs, + public static Bolt prepareSerializedBoltDetails(Map inputs, + IBolt bolt, Map outputs, Integer parallelismHint, Map conf) { ComponentCommon common = prepareComponentCommon(inputs, outputs, parallelismHint, conf); return new Bolt(ComponentObject.serialized_java(Utils.javaSerialize(bolt)), common); } - public static BoltDetails prepareBoltDetails(Map inputs, Object bolt) { + public static BoltDetails prepareBoltDetails(Map inputs, + Object bolt) { return prepareBoltDetails(inputs, bolt, null, null); } @@ -251,7 +257,8 @@ public static SpoutDetails prepareSpoutDetails(IRichSpout spout, Integer paralle return prepareSpoutDetails(spout, parallelismHint, null); } - public static SpoutDetails prepareSpoutDetails(IRichSpout spout, Integer parallelismHint, Map conf) { + public static SpoutDetails prepareSpoutDetails(IRichSpout spout, Integer parallelismHint, + Map conf) { SpoutDetails details = new SpoutDetails(spout, parallelismHint, conf); return details; } @@ -267,12 +274,14 @@ public static StormTopology buildTopology(HashMap spoutMap return buildTopology(spoutMap, boltMap); } - public static StormTopology buildTopology(Map spoutMap, Map boltMap) { + public static StormTopology buildTopology(Map spoutMap, Map boltMap) { TopologyBuilder builder = new TopologyBuilder(); for (Entry entry : spoutMap.entrySet()) { String spoutId = entry.getKey(); SpoutDetails spec = entry.getValue(); - SpoutDeclarer spoutDeclarer = builder.setSpout(spoutId, spec.getSpout(), spec.getParallelism()); + SpoutDeclarer spoutDeclarer = builder.setSpout(spoutId, spec.getSpout(), spec + .getParallelism()); spoutDeclarer.addConfigurations(spec.getConf()); } for (Entry entry : boltMap.entrySet()) { @@ -280,9 +289,11 @@ public static StormTopology buildTopology(Map spoutMap, Ma BoltDetails spec = entry.getValue(); BoltDeclarer boltDeclarer = null; if (spec.bolt instanceof IRichBolt) { - boltDeclarer = builder.setBolt(spoutId, (IRichBolt) spec.getBolt(), spec.getParallelism()); + boltDeclarer = builder.setBolt(spoutId, (IRichBolt) spec.getBolt(), spec + .getParallelism()); } else { - boltDeclarer = builder.setBolt(spoutId, (IBasicBolt) spec.getBolt(), spec.getParallelism()); + boltDeclarer = builder.setBolt(spoutId, (IBasicBolt) spec.getBolt(), spec + .getParallelism()); } boltDeclarer.addConfigurations(spec.getConf()); addInputs(boltDeclarer, spec.getInputs()); diff --git a/storm-client/src/jvm/org/apache/storm/annotation/InterfaceStability.java b/storm-client/src/jvm/org/apache/storm/annotation/InterfaceStability.java index 0a57ac1a6c4..56a7b610efd 100644 --- a/storm-client/src/jvm/org/apache/storm/annotation/InterfaceStability.java +++ b/storm-client/src/jvm/org/apache/storm/annotation/InterfaceStability.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -17,12 +23,14 @@ import java.lang.annotation.RetentionPolicy; /** - * Annotation to inform users of how much to rely on a particular package, class or method not changing over time. + * Annotation to inform users of how much to rely on a particular package, class or method not + * changing over time. */ @InterfaceStability.Evolving public class InterfaceStability { /** - * Can evolve while retaining compatibility for minor release boundaries.; can break compatibility only at major release (ie. at m.0). + * Can evolve while retaining compatibility for minor release boundaries.; can break + * compatibility only at major release (ie. at m.0). */ @Documented @Retention(RetentionPolicy.RUNTIME) @@ -42,7 +50,8 @@ public class InterfaceStability { ; /** - * No guarantee is provided as to reliability or stability across any level of release granularity. + * No guarantee is provided as to reliability or stability across any level of release + * granularity. */ @Documented @Retention(RetentionPolicy.RUNTIME) diff --git a/storm-client/src/jvm/org/apache/storm/assignments/ILocalAssignmentsBackend.java b/storm-client/src/jvm/org/apache/storm/assignments/ILocalAssignmentsBackend.java index 762842ecc1e..80387379ef9 100644 --- a/storm-client/src/jvm/org/apache/storm/assignments/ILocalAssignmentsBackend.java +++ b/storm-client/src/jvm/org/apache/storm/assignments/ILocalAssignmentsBackend.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -72,7 +78,8 @@ public interface ILocalAssignmentsBackend extends AutoCloseable { /** * Sync remote assignments to local, if remote is null, we will sync it from zk. * - * @param remote specific remote assignments, if it is null, it will sync from zookeeper[only used for nimbus] + * @param remote specific remote assignments, if it is null, it will sync from zookeeper[only + * used for nimbus] */ void syncRemoteAssignments(Map remote); diff --git a/storm-client/src/jvm/org/apache/storm/assignments/InMemoryAssignmentBackend.java b/storm-client/src/jvm/org/apache/storm/assignments/InMemoryAssignmentBackend.java index 8854e386fbd..4cdb2a21885 100644 --- a/storm-client/src/jvm/org/apache/storm/assignments/InMemoryAssignmentBackend.java +++ b/storm-client/src/jvm/org/apache/storm/assignments/InMemoryAssignmentBackend.java @@ -1,12 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -23,13 +28,15 @@ import org.slf4j.LoggerFactory; /** - * An assignment backend which will keep all assignments and id-info in memory. Only used if no backend is specified internal. + * An assignment backend which will keep all assignments and id-info in memory. Only used if no + * backend is specified internal. * *

    About thread safe: idToAssignment,idToName,nameToId are all memory cache in nimbus local, for *

      *
    • idToAssignment: nimbus will modify it and supervisors will sync it at fixed interval, * so the assignments would come to eventual consistency.
    • - *
    • idToName: storm submitting/killing is guarded by the same lock, a {@link ConcurrentHashMap} is ok.
    • + *
    • idToName: storm submitting/killing is guarded by the same lock, a {@link ConcurrentHashMap} + * is ok.
    • *
    • nameToId: same as idToName. *
    */ @@ -96,7 +103,8 @@ public Map assignmentsInfo() { public void syncRemoteAssignments(Map remote) { Map tmp = new ConcurrentHashMap<>(); for (Map.Entry entry : remote.entrySet()) { - tmp.put(entry.getKey(), ClusterUtils.maybeDeserialize(entry.getValue(), Assignment.class)); + tmp.put(entry.getKey(), ClusterUtils.maybeDeserialize(entry.getValue(), + Assignment.class)); } this.idToAssignment = tmp; } diff --git a/storm-client/src/jvm/org/apache/storm/assignments/LocalAssignmentsBackendFactory.java b/storm-client/src/jvm/org/apache/storm/assignments/LocalAssignmentsBackendFactory.java index ba92187177a..9890a14d0f8 100644 --- a/storm-client/src/jvm/org/apache/storm/assignments/LocalAssignmentsBackendFactory.java +++ b/storm-client/src/jvm/org/apache/storm/assignments/LocalAssignmentsBackendFactory.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -25,8 +31,10 @@ public class LocalAssignmentsBackendFactory { public static ILocalAssignmentsBackend getBackend(Map conf) { if (conf.get(Config.NIMBUS_LOCAL_ASSIGNMENTS_BACKEND_CLASS) != null) { - Object targetObj = ReflectionUtils.newInstance((String) conf.get(Config.NIMBUS_LOCAL_ASSIGNMENTS_BACKEND_CLASS)); - Preconditions.checkState(targetObj instanceof ILocalAssignmentsBackend, "{} must implements ILocalAssignmentsBackend", + Object targetObj = ReflectionUtils.newInstance((String) conf + .get(Config.NIMBUS_LOCAL_ASSIGNMENTS_BACKEND_CLASS)); + Preconditions.checkState(targetObj instanceof ILocalAssignmentsBackend, + "{} must implements ILocalAssignmentsBackend", Config.NIMBUS_LOCAL_ASSIGNMENTS_BACKEND_CLASS); ((ILocalAssignmentsBackend) targetObj).prepare(conf); return (ILocalAssignmentsBackend) targetObj; diff --git a/storm-client/src/jvm/org/apache/storm/blobstore/AtomicOutputStream.java b/storm-client/src/jvm/org/apache/storm/blobstore/AtomicOutputStream.java index d74f47ca65b..d0309b88a88 100644 --- a/storm-client/src/jvm/org/apache/storm/blobstore/AtomicOutputStream.java +++ b/storm-client/src/jvm/org/apache/storm/blobstore/AtomicOutputStream.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/blobstore/BlobStore.java b/storm-client/src/jvm/org/apache/storm/blobstore/BlobStore.java index 367141ae509..aae3659d705 100644 --- a/storm-client/src/jvm/org/apache/storm/blobstore/BlobStore.java +++ b/storm-client/src/jvm/org/apache/storm/blobstore/BlobStore.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -37,16 +43,21 @@ import org.slf4j.LoggerFactory; /** - * Provides a way to store blobs that can be downloaded. Blobs must be able to be uploaded and listed from Nimbus, and - * downloaded from the Supervisors. It is a key value based store. Key being a string and value being the blob data. + * Provides a way to store blobs that can be downloaded. Blobs must be able to be uploaded and + * listed from Nimbus, and + * downloaded from the Supervisors. It is a key value based store. Key being a string and value + * being the blob data. * - *

    ACL checking must take place against the provided subject. If the blob store does not support Security it must + *

    ACL checking must take place against the provided subject. If the blob store does not support + * Security it must * validate that all ACLs set are always WORLD, everything. * - *

    The users can upload their blobs through the blob store command line. The command line also allows us to update + *

    The users can upload their blobs through the blob store command line. The command line also + * allows us to update * and delete blobs. * - *

    Modifying the replication factor only works for HdfsBlobStore as for the LocalFsBlobStore the replication is + *

    Modifying the replication factor only works for HdfsBlobStore as for the LocalFsBlobStore the + * replication is * dependent on the number of Nimbodes available. */ public abstract class BlobStore implements Shutdownable, AutoCloseable { @@ -72,7 +83,8 @@ public static final void validateKey(String key) throws IllegalArgumentException * @param baseDir The directory path to store the blobs * @param nimbusInfo Contains the nimbus host, port and leadership information */ - public abstract void prepare(Map conf, String baseDir, NimbusInfo nimbusInfo, ILeaderElector leaderElector); + public abstract void prepare(Map conf, String baseDir, NimbusInfo nimbusInfo, + ILeaderElector leaderElector); /** * Start the syncing blobs between the local running instance of the BlobStore and others. @@ -91,7 +103,8 @@ public void startSyncBlobs() throws KeyNotFoundException, AuthorizationException * @param who Is the subject creating the blob * @return AtomicOutputStream returns a stream into which the data can be written */ - public abstract AtomicOutputStream createBlob(String key, SettableBlobMeta meta, Subject who) throws AuthorizationException, + public abstract AtomicOutputStream createBlob(String key, SettableBlobMeta meta, + Subject who) throws AuthorizationException, KeyAlreadyExistsException; /** @@ -102,7 +115,8 @@ public abstract AtomicOutputStream createBlob(String key, SettableBlobMeta meta, * @param meta Metadata which contains the acls information * @param who Is the subject creating the blob */ - public void createBlob(String key, byte[] data, SettableBlobMeta meta, Subject who) throws AuthorizationException, + public void createBlob(String key, byte[] data, SettableBlobMeta meta, + Subject who) throws AuthorizationException, KeyAlreadyExistsException, IOException { AtomicOutputStream out = null; try { @@ -125,7 +139,8 @@ public void createBlob(String key, byte[] data, SettableBlobMeta meta, Subject w * @param meta Metadata which contains the acls information * @param who Is the subject creating the blob */ - public void createBlob(String key, InputStream in, SettableBlobMeta meta, Subject who) throws AuthorizationException, + public void createBlob(String key, InputStream in, SettableBlobMeta meta, + Subject who) throws AuthorizationException, KeyAlreadyExistsException, IOException { AtomicOutputStream out = null; try { @@ -156,7 +171,8 @@ public void createBlob(String key, InputStream in, SettableBlobMeta meta, Subjec * @param who Is the subject having the write privilege for the blob * @return AtomicOutputStream returns a stream into which the data can be written */ - public abstract AtomicOutputStream updateBlob(String key, Subject who) throws AuthorizationException, KeyNotFoundException; + public abstract AtomicOutputStream updateBlob(String key, + Subject who) throws AuthorizationException, KeyNotFoundException; /** * Wrapper called to create the blob which contains the byte data. @@ -165,7 +181,8 @@ public void createBlob(String key, InputStream in, SettableBlobMeta meta, Subjec * @param data Byte data that needs to be uploaded * @param who Is the subject creating the blob */ - public void updateBlob(String key, byte[] data, Subject who) throws AuthorizationException, IOException, KeyNotFoundException { + public void updateBlob(String key, byte[] data, + Subject who) throws AuthorizationException, IOException, KeyNotFoundException { AtomicOutputStream out = null; try { out = updateBlob(key, who); @@ -180,13 +197,15 @@ public void updateBlob(String key, byte[] data, Subject who) throws Authorizatio } /** - * Gets the current version of metadata for a blob to be viewed by the user or downloaded by the supervisor. + * Gets the current version of metadata for a blob to be viewed by the user or downloaded by the + * supervisor. * * @param key Key for the blob * @param who Is the subject having the read privilege for the blob * @return AtomicOutputStream returns a stream into which the data can be written */ - public abstract ReadableBlobMeta getBlobMeta(String key, Subject who) throws AuthorizationException, KeyNotFoundException; + public abstract ReadableBlobMeta getBlobMeta(String key, + Subject who) throws AuthorizationException, KeyNotFoundException; /** * Sets leader elector (only used by LocalFsBlobStore to help sync blobs between Nimbi. @@ -201,7 +220,8 @@ public void updateBlob(String key, byte[] data, Subject who) throws Authorizatio * @param meta Metadata which contains the updated acls information * @param who Is the subject having the write privilege for the blob */ - public abstract void setBlobMeta(String key, SettableBlobMeta meta, Subject who) throws AuthorizationException, KeyNotFoundException; + public abstract void setBlobMeta(String key, SettableBlobMeta meta, + Subject who) throws AuthorizationException, KeyNotFoundException; /** * Deletes the blob data and metadata. @@ -209,7 +229,8 @@ public void updateBlob(String key, byte[] data, Subject who) throws Authorizatio * @param key Key for the blob * @param who Is the subject having write privilege for the blob */ - public abstract void deleteBlob(String key, Subject who) throws AuthorizationException, KeyNotFoundException; + public abstract void deleteBlob(String key, + Subject who) throws AuthorizationException, KeyNotFoundException; /** * Gets the InputStream to read the blob details. @@ -218,7 +239,8 @@ public void updateBlob(String key, byte[] data, Subject who) throws Authorizatio * @param who Is the subject having the read privilege for the blob * @return InputStreamWithMeta has the additional file length and version information */ - public abstract InputStreamWithMeta getBlob(String key, Subject who) throws AuthorizationException, KeyNotFoundException; + public abstract InputStreamWithMeta getBlob(String key, + Subject who) throws AuthorizationException, KeyNotFoundException; /** * Returns an iterator with all the list of keys currently available on the blob store. @@ -244,7 +266,8 @@ public void updateBlob(String key, byte[] data, Subject who) throws Authorizatio * @param who Is the subject having the update privilege for the blob * @return BlobReplication object containing the updated replication factor for the blob */ - public abstract int updateBlobReplication(String key, int replication, Subject who) throws AuthorizationException, KeyNotFoundException, + public abstract int updateBlobReplication(String key, int replication, + Subject who) throws AuthorizationException, KeyNotFoundException, IOException; @Override @@ -278,7 +301,8 @@ public Set filterAndListKeys(KeyFilter filter) { * @param out Output stream * @param who Is the subject having read privilege for the blob */ - public void readBlobTo(String key, OutputStream out, Subject who) throws IOException, KeyNotFoundException, AuthorizationException { + public void readBlobTo(String key, OutputStream out, + Subject who) throws IOException, KeyNotFoundException, AuthorizationException { InputStreamWithMeta in = getBlob(key, who); if (in == null) { throw new IOException("Could not find " + key); @@ -301,7 +325,8 @@ public void readBlobTo(String key, OutputStream out, Subject who) throws IOExcep * @param key Key for the blob * @param who Is the subject having the read privilege for the blob */ - public byte[] readBlob(String key, Subject who) throws IOException, KeyNotFoundException, AuthorizationException { + public byte[] readBlob(String key, + Subject who) throws IOException, KeyNotFoundException, AuthorizationException { ByteArrayOutputStream out = new ByteArrayOutputStream(); readBlobTo(key, out, who); byte[] bytes = out.toByteArray(); @@ -311,6 +336,7 @@ public byte[] readBlob(String key, Subject who) throws IOException, KeyNotFoundE /** * Get IDs stored in blob store. + * * @return a set of all of the topology ids with special data stored in the blob store. */ public Set storedTopoIds() { @@ -327,7 +353,8 @@ public void updateLastBlobUpdateTime() throws IOException { } /** - * Validates that the blob update time of the blobstore is up to date with the current existing blobs. + * Validates that the blob update time of the blobstore is up to date with the current existing + * blobs. * * @throws IOException on any error */ @@ -396,7 +423,7 @@ public BlobStoreFileOutputStream(BlobStoreFile part) throws IOException { @Override public void close() throws IOException { try { - //close means commit + // close means commit out.close(); part.commit(); } catch (IOException | RuntimeException e) { @@ -431,7 +458,8 @@ public void write(byte[] b, int offset, int len) throws IOException { } /** - * Input stream implementation used for writing both the metadata containing the acl information and the blob data. + * Input stream implementation used for writing both the metadata containing the acl information + * and the blob data. */ protected class BlobStoreFileInputStream extends InputStreamWithMeta { private BlobStoreFile part; diff --git a/storm-client/src/jvm/org/apache/storm/blobstore/BlobStoreAclHandler.java b/storm-client/src/jvm/org/apache/storm/blobstore/BlobStoreAclHandler.java index a61fb662405..02552089f23 100644 --- a/storm-client/src/jvm/org/apache/storm/blobstore/BlobStoreAclHandler.java +++ b/storm-client/src/jvm/org/apache/storm/blobstore/BlobStoreAclHandler.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -36,7 +42,8 @@ import org.slf4j.LoggerFactory; /** - * Provides common handling of acls for Blobstores. Also contains some static utility functions related to Blobstores. + * Provides common handling of acls for Blobstores. Also contains some static utility functions + * related to Blobstores. */ public class BlobStoreAclHandler { public static final Logger LOG = LoggerFactory.getLogger(BlobStoreAclHandler.class); @@ -56,7 +63,8 @@ public class BlobStoreAclHandler { public BlobStoreAclHandler(Map conf) { ptol = ClientAuthUtils.getPrincipalToLocalPlugin(conf); if (conf.get(Config.STORM_GROUP_MAPPING_SERVICE_PROVIDER_PLUGIN) != null) { - groupMappingServiceProvider = ClientAuthUtils.getGroupMappingServiceProviderPlugin(conf); + groupMappingServiceProvider = ClientAuthUtils + .getGroupMappingServiceProviderPlugin(conf); } else { groupMappingServiceProvider = null; } @@ -96,7 +104,7 @@ private static int parseAccess(String access) { } else if ('a' == c) { ret = ret | ADMIN; } else if ('-' == c) { - //ignored + // ignored } else { throw new IllegalArgumentException(""); } @@ -110,7 +118,8 @@ public static AccessControl parseAccessControl(String str) { String name = ""; String access = "-"; if (parts.length > 3) { - throw new IllegalArgumentException("Don't know how to parse " + str + " into an ACL value"); + throw new IllegalArgumentException("Don't know how to parse " + str + + " into an ACL value"); } else if (parts.length == 1) { type = "other"; name = ""; @@ -149,7 +158,8 @@ public static String accessControlToString(AccessControl ac) { ret.append("u"); break; default: - throw new IllegalArgumentException("Don't know what a type of " + ac.get_type() + " means "); + throw new IllegalArgumentException("Don't know what a type of " + ac.get_type() + + " means "); } ret.append(":"); if (ac.is_set_name()) { @@ -161,7 +171,8 @@ public static String accessControlToString(AccessControl ac) { } @SuppressWarnings("checkstyle:AbbreviationAsWordInName") - public static void validateSettableACLs(String key, List acls) throws AuthorizationException { + public static void validateSettableACLs(String key, + List acls) throws AuthorizationException { Set aclUsers = new HashSet<>(); List duplicateUsers = new ArrayList<>(); for (AccessControl acl : acls) { @@ -173,7 +184,8 @@ public static void validateSettableACLs(String key, List acls) th } if (duplicateUsers.size() > 0) { String errorMessage = "user " + Arrays.toString(duplicateUsers.toArray()) - + " can't appear more than once in the ACLs for key [" + key + "]."; + + " can't appear more than once in the ACLs for key [" + key + + "]."; throw new WrappedAuthorizationException(errorMessage); } } @@ -251,9 +263,11 @@ public boolean checkForValidUsers(Subject who, int mask) { } /** - * The user should be able to see the metadata if and only if they have any of READ, WRITE, or ADMIN. + * The user should be able to see the metadata if and only if they have any of READ, WRITE, or + * ADMIN. */ - public void validateUserCanReadMeta(List acl, Subject who, String key) throws AuthorizationException { + public void validateUserCanReadMeta(List acl, Subject who, + String key) throws AuthorizationException { hasAnyPermissions(acl, (READ | WRITE | ADMIN), who, key); } @@ -261,12 +275,15 @@ public void validateUserCanReadMeta(List acl, Subject who, String * Validates if the user has any of the permissions mentioned in the mask. * * @param acl ACL for the key. - * @param mask mask holds the cumulative value of READ = 1, WRITE = 2 or ADMIN = 4 permissions. mask = 1 implies READ privilege. mask = + * @param mask mask holds the cumulative value of READ = 1, WRITE = 2 or ADMIN = 4 permissions. + * mask = 1 implies READ privilege. mask = * 5 implies READ and ADMIN privileges. - * @param who Is the user against whom the permissions are validated for a key using the ACL and the mask. + * @param who Is the user against whom the permissions are validated for a key using the ACL and + * the mask. * @param key Key used to identify the blob. */ - public void hasAnyPermissions(List acl, int mask, Subject who, String key) throws AuthorizationException { + public void hasAnyPermissions(List acl, int mask, Subject who, + String key) throws AuthorizationException { if (!doAclValidation) { return; } @@ -290,12 +307,15 @@ public void hasAnyPermissions(List acl, int mask, Subject who, St * Validates if the user has at least the set of permissions mentioned in the mask. * * @param acl ACL for the key. - * @param mask mask holds the cumulative value of READ = 1, WRITE = 2 or ADMIN = 4 permissions. mask = 1 implies READ privilege. mask = + * @param mask mask holds the cumulative value of READ = 1, WRITE = 2 or ADMIN = 4 permissions. + * mask = 1 implies READ privilege. mask = * 5 implies READ and ADMIN privileges. - * @param who Is the user against whom the permissions are validated for a key using the ACL and the mask. + * @param who Is the user against whom the permissions are validated for a key using the ACL and + * the mask. * @param key Key used to identify the blob. */ - public void hasPermissions(List acl, int mask, Subject who, String key) throws AuthorizationException { + public void hasPermissions(List acl, int mask, Subject who, + String key) throws AuthorizationException { if (!doAclValidation) { return; } @@ -316,7 +336,8 @@ public void hasPermissions(List acl, int mask, Subject who, Strin user + " does not have " + namedPerms(mask) + " access to " + key); } - public void normalizeSettableBlobMeta(String key, SettableBlobMeta meta, Subject who, int opMask) { + public void normalizeSettableBlobMeta(String key, SettableBlobMeta meta, Subject who, + int opMask) { meta.set_acl(normalizeSettableAcls(key, meta.get_acl(), who, opMask)); } @@ -363,7 +384,8 @@ private List removeBadAcls(List accessControls) { return resultAcl; } - private List normalizeSettableAcls(String key, List acls, Subject who, + private List normalizeSettableAcls(String key, List acls, + Subject who, int opMask) { List cleanAcls = removeBadAcls(acls); Set userNames = getUserNamesFromSubject(who); @@ -373,9 +395,11 @@ private List normalizeSettableAcls(String key, List normalizeSettableAcls(String key, List acls) { boolean isWorldEverything = false; for (AccessControl acl : acls) { - if (acl.get_type() == AccessControlType.OTHER && acl.get_access() == (READ | WRITE | ADMIN)) { + if (acl.get_type() == AccessControlType.OTHER && acl + .get_access() == (READ | WRITE | ADMIN)) { isWorldEverything = true; break; } diff --git a/storm-client/src/jvm/org/apache/storm/blobstore/BlobStoreFile.java b/storm-client/src/jvm/org/apache/storm/blobstore/BlobStoreFile.java index bd901a0c234..49dca739cb9 100644 --- a/storm-client/src/jvm/org/apache/storm/blobstore/BlobStoreFile.java +++ b/storm-client/src/jvm/org/apache/storm/blobstore/BlobStoreFile.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/blobstore/ClientBlobStore.java b/storm-client/src/jvm/org/apache/storm/blobstore/ClientBlobStore.java index 02b284fcb2f..d782a54c7c1 100644 --- a/storm-client/src/jvm/org/apache/storm/blobstore/ClientBlobStore.java +++ b/storm-client/src/jvm/org/apache/storm/blobstore/ClientBlobStore.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -28,12 +34,16 @@ /** * The ClientBlobStore has two concrete implementations 1. NimbusBlobStore 2. HdfsClientBlobStore. * - *

    Create, update, read and delete are some of the basic operations defined by this interface. Each operation is - * validated for permissions against an user. We currently have NIMBUS_ADMINS and SUPERVISOR_ADMINS configuration. - * NIMBUS_ADMINS are given READ, WRITE and ADMIN access whereas the SUPERVISOR_ADMINS are given READ access in order to + *

    Create, update, read and delete are some of the basic operations defined by this interface. + * Each operation is + * validated for permissions against an user. We currently have NIMBUS_ADMINS and SUPERVISOR_ADMINS + * configuration. + * NIMBUS_ADMINS are given READ, WRITE and ADMIN access whereas the SUPERVISOR_ADMINS are given READ + * access in order to * read and download the blobs form the nimbus. * - *

    The ACLs for the blob store are validated against whether the subject is a NIMBUS_ADMIN, SUPERVISOR_ADMIN or USER + *

    The ACLs for the blob store are validated against whether the subject is a NIMBUS_ADMIN, + * SUPERVISOR_ADMIN or USER * who has read, write or admin privileges in order to perform respective operations on the blob. * *

    For more detailed implementation @@ -63,7 +73,8 @@ public static void withConfiguredClient(WithBlobstore withBlobstore) throws Exce * @param meta contains ACL information * @return AtomicOutputStream returns an output stream into which data can be written */ - protected abstract AtomicOutputStream createBlobToExtend(String key, SettableBlobMeta meta) throws AuthorizationException, + protected abstract AtomicOutputStream createBlobToExtend(String key, + SettableBlobMeta meta) throws AuthorizationException, KeyAlreadyExistsException; /** @@ -84,6 +95,7 @@ protected abstract AtomicOutputStream createBlobToExtend(String key, SettableBlo /** * Decide if the blob is deleted from cluster. + * * @param blobKey blob key */ public abstract boolean isRemoteBlobExists(String blobKey) throws AuthorizationException; @@ -94,7 +106,8 @@ protected abstract AtomicOutputStream createBlobToExtend(String key, SettableBlo * @param key blob key name * @param meta contains ACL information */ - protected abstract void setBlobMetaToExtend(String key, SettableBlobMeta meta) throws AuthorizationException, KeyNotFoundException; + protected abstract void setBlobMetaToExtend(String key, + SettableBlobMeta meta) throws AuthorizationException, KeyNotFoundException; /** * Client facing API to delete a blob. @@ -113,6 +126,7 @@ protected abstract AtomicOutputStream createBlobToExtend(String key, SettableBlo /** * List keys. + * * @return Iterator for a list of keys currently present in the blob store. */ public abstract Iterator listKeys(); @@ -132,7 +146,8 @@ protected abstract AtomicOutputStream createBlobToExtend(String key, SettableBlo * @param replication int indicates the replication factor a blob has to be set * @return int indicates the replication factor of a blob */ - public abstract int updateBlobReplication(String key, int replication) throws AuthorizationException, KeyNotFoundException; + public abstract int updateBlobReplication(String key, + int replication) throws AuthorizationException, KeyNotFoundException; /** * Client facing API to set a nimbus client. @@ -144,7 +159,8 @@ protected abstract AtomicOutputStream createBlobToExtend(String key, SettableBlo public abstract boolean setClient(Map conf, NimbusClient client); /** - * Creates state inside a zookeeper. Required for blobstore to write to zookeeper when Nimbus HA is turned on in + * Creates state inside a zookeeper. Required for blobstore to write to zookeeper when Nimbus HA + * is turned on in * order to maintain state consistency. */ public abstract void createStateInZookeeper(String key); @@ -159,7 +175,8 @@ protected abstract AtomicOutputStream createBlobToExtend(String key, SettableBlo * @param meta contains ACL information * @return AtomicOutputStream returns an output stream into which data can be written */ - public final AtomicOutputStream createBlob(String key, SettableBlobMeta meta) throws AuthorizationException, KeyAlreadyExistsException { + public final AtomicOutputStream createBlob(String key, + SettableBlobMeta meta) throws AuthorizationException, KeyAlreadyExistsException { if (meta != null && meta.is_set_acl()) { BlobStoreAclHandler.validateSettableACLs(key, meta.get_acl()); } @@ -172,7 +189,8 @@ public final AtomicOutputStream createBlob(String key, SettableBlobMeta meta) th * @param key blob key name * @param meta contains ACL information */ - public final void setBlobMeta(String key, SettableBlobMeta meta) throws AuthorizationException, KeyNotFoundException { + public final void setBlobMeta(String key, + SettableBlobMeta meta) throws AuthorizationException, KeyNotFoundException { if (meta != null && meta.is_set_acl()) { BlobStoreAclHandler.validateSettableACLs(key, meta.get_acl()); } @@ -184,7 +202,8 @@ public interface WithBlobstore { } /** - * Client facing API to get the last update time of existing blobs in a blobstore. This is only required for use on + * Client facing API to get the last update time of existing blobs in a blobstore. This is only + * required for use on * supervisors. * * @return the timestamp of when the blobstore was last updated. -1L if the blobstore diff --git a/storm-client/src/jvm/org/apache/storm/blobstore/InputStreamWithMeta.java b/storm-client/src/jvm/org/apache/storm/blobstore/InputStreamWithMeta.java index 745bbfac689..3654e88cf9d 100644 --- a/storm-client/src/jvm/org/apache/storm/blobstore/InputStreamWithMeta.java +++ b/storm-client/src/jvm/org/apache/storm/blobstore/InputStreamWithMeta.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/blobstore/KeyFilter.java b/storm-client/src/jvm/org/apache/storm/blobstore/KeyFilter.java index 3c841011066..708600d0907 100644 --- a/storm-client/src/jvm/org/apache/storm/blobstore/KeyFilter.java +++ b/storm-client/src/jvm/org/apache/storm/blobstore/KeyFilter.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/blobstore/LocalModeClientBlobStore.java b/storm-client/src/jvm/org/apache/storm/blobstore/LocalModeClientBlobStore.java index 5bf0199dbae..9369963dbc5 100644 --- a/storm-client/src/jvm/org/apache/storm/blobstore/LocalModeClientBlobStore.java +++ b/storm-client/src/jvm/org/apache/storm/blobstore/LocalModeClientBlobStore.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -39,11 +45,12 @@ public void shutdown() { @Override public void prepare(Map conf) { - //NOOP prepare should have already been called + // NOOP prepare should have already been called } @Override - protected AtomicOutputStream createBlobToExtend(String key, SettableBlobMeta meta) throws AuthorizationException, + protected AtomicOutputStream createBlobToExtend(String key, + SettableBlobMeta meta) throws AuthorizationException, KeyAlreadyExistsException { return wrapped.createBlob(key, meta, null); } @@ -69,7 +76,8 @@ public boolean isRemoteBlobExists(String blobKey) throws AuthorizationException } @Override - protected void setBlobMetaToExtend(String key, SettableBlobMeta meta) throws AuthorizationException, KeyNotFoundException { + protected void setBlobMetaToExtend(String key, + SettableBlobMeta meta) throws AuthorizationException, KeyNotFoundException { wrapped.setBlobMeta(key, meta, null); } @@ -100,7 +108,8 @@ public int getBlobReplication(String key) throws AuthorizationException, KeyNotF } @Override - public int updateBlobReplication(String key, int replication) throws AuthorizationException, KeyNotFoundException { + public int updateBlobReplication(String key, + int replication) throws AuthorizationException, KeyNotFoundException { try { return wrapped.updateBlobReplication(key, replication, null); } catch (AuthorizationException | KeyNotFoundException rethrow) { @@ -117,7 +126,7 @@ public boolean setClient(Map conf, NimbusClient client) { @Override public void createStateInZookeeper(String key) { - //NOOP + // NOOP } @Override diff --git a/storm-client/src/jvm/org/apache/storm/blobstore/NimbusBlobStore.java b/storm-client/src/jvm/org/apache/storm/blobstore/NimbusBlobStore.java index e76c7176555..a6b70547a59 100644 --- a/storm-client/src/jvm/org/apache/storm/blobstore/NimbusBlobStore.java +++ b/storm-client/src/jvm/org/apache/storm/blobstore/NimbusBlobStore.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -32,10 +38,12 @@ import org.slf4j.LoggerFactory; /** - * NimbusBlobStore is a USER facing client API to perform basic operations such as create, update, delete and read for local and hdfs blob + * NimbusBlobStore is a USER facing client API to perform basic operations such as create, update, + * delete and read for local and hdfs blob * store. * - *

    For local blob store it is also the client facing API for supervisor in order to download blobs from nimbus. + *

    For local blob store it is also the client facing API for supervisor in order to download + * blobs from nimbus. */ public class NimbusBlobStore extends ClientBlobStore implements AutoCloseable { private static final Logger LOG = LoggerFactory.getLogger(NimbusBlobStore.class); @@ -46,7 +54,8 @@ public class NimbusBlobStore extends ClientBlobStore implements AutoCloseable { public void prepare(Map conf) { this.client = NimbusClient.Builder.withConf(conf).build(); if (conf != null) { - this.bufferSize = ObjectReader.getInt(conf.get(Config.STORM_BLOBSTORE_INPUTSTREAM_BUFFER_SIZE_BYTES), bufferSize); + this.bufferSize = ObjectReader.getInt(conf + .get(Config.STORM_BLOBSTORE_INPUTSTREAM_BUFFER_SIZE_BYTES), bufferSize); } } @@ -55,7 +64,8 @@ protected AtomicOutputStream createBlobToExtend(String key, SettableBlobMeta met throws AuthorizationException, KeyAlreadyExistsException { try { synchronized (client) { - return new NimbusUploadAtomicOutputStream(client.getClient().beginCreateBlob(key, meta), this.bufferSize, key); + return new NimbusUploadAtomicOutputStream(client.getClient().beginCreateBlob(key, + meta), this.bufferSize, key); } } catch (AuthorizationException | KeyAlreadyExistsException exp) { throw exp; @@ -69,7 +79,8 @@ public AtomicOutputStream updateBlob(String key) throws AuthorizationException, KeyNotFoundException { try { synchronized (client) { - return new NimbusUploadAtomicOutputStream(client.getClient().beginUpdateBlob(key), this.bufferSize, key); + return new NimbusUploadAtomicOutputStream(client.getClient().beginUpdateBlob(key), + this.bufferSize, key); } } catch (AuthorizationException | KeyNotFoundException exp) { throw exp; @@ -176,7 +187,8 @@ public int getBlobReplication(String key) throws AuthorizationException, KeyNotF } @Override - public int updateBlobReplication(String key, int replication) throws AuthorizationException, KeyNotFoundException { + public int updateBlobReplication(String key, + int replication) throws AuthorizationException, KeyNotFoundException { try { return client.getClient().updateBlobReplication(key, replication); } catch (AuthorizationException | KeyNotFoundException exp) { @@ -193,7 +205,8 @@ public boolean setClient(Map conf, NimbusClient client) { } this.client = client; if (conf != null) { - this.bufferSize = ObjectReader.getInt(conf.get(Config.STORM_BLOBSTORE_INPUTSTREAM_BUFFER_SIZE_BYTES), bufferSize); + this.bufferSize = ObjectReader.getInt(conf + .get(Config.STORM_BLOBSTORE_INPUTSTREAM_BUFFER_SIZE_BYTES), bufferSize); } return true; } @@ -393,7 +406,8 @@ public void cancel() throws IOException { public void write(int b) throws IOException { try { synchronized (client) { - client.getClient().uploadBlobChunk(session, ByteBuffer.wrap(new byte[]{ (byte) b })); + client.getClient().uploadBlobChunk(session, ByteBuffer + .wrap(new byte[]{ (byte) b })); } } catch (TException e) { throw new RuntimeException(e); @@ -413,7 +427,8 @@ public void write(byte[] b, int offset, int len) throws IOException { int realLen = Math.min(end - realOffset, maxChunkSize); LOG.debug("Writing {} bytes of {} remaining", realLen, (end - realOffset)); synchronized (client) { - client.getClient().uploadBlobChunk(session, ByteBuffer.wrap(b, realOffset, realLen)); + client.getClient().uploadBlobChunk(session, ByteBuffer.wrap(b, realOffset, + realLen)); } } } catch (TException e) { diff --git a/storm-client/src/jvm/org/apache/storm/bolt/JoinBolt.java b/storm-client/src/jvm/org/apache/storm/bolt/JoinBolt.java index 354723e704f..d676787a3d4 100644 --- a/storm-client/src/jvm/org/apache/storm/bolt/JoinBolt.java +++ b/storm-client/src/jvm/org/apache/storm/bolt/JoinBolt.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -34,14 +40,16 @@ public class JoinBolt extends BaseWindowedBolt { // Map[StreamName -> JoinInfo] protected LinkedHashMap joinCriteria = new LinkedHashMap<>(); protected FieldSelector[] outputFields; // specified via bolt.select() ... used in declaring Output fields - // protected String[] dotSeparatedOutputFieldNames; // fieldNames in x.y.z format w/o stream name, used for naming output fields + // protected String[] dotSeparatedOutputFieldNames; // fieldNames in x.y.z format w/o stream + // name, used for naming output fields protected String outputStreamName; // Map[StreamName -> Map[Key -> List] ] - HashMap>> hashedInputs = new HashMap<>(); // holds remaining streams + HashMap>> hashedInputs = + new HashMap<>(); // holds remaining streams private OutputCollector collector; /** - * Calls JoinBolt(Selector.SOURCE, sourceId, fieldName) + * Calls JoinBolt(Selector.SOURCE, sourceId, fieldName). * * @param sourceId Id of source component (spout/bolt) from which this bolt is receiving data * @param fieldName the field to use for joining the stream (x.y.z format) @@ -50,9 +58,9 @@ public JoinBolt(String sourceId, String fieldName) { this(Selector.SOURCE, sourceId, fieldName); } - /** - * Introduces the first stream to start the join with. Equivalent SQL ... select .... from srcOrStreamId ... + * Introduces the first stream to start the join with. Equivalent SQL ... select .... from + * srcOrStreamId ... * * @param type Specifies whether 'srcOrStreamId' refers to stream name/source component * @param srcOrStreamId name of stream OR source component @@ -65,7 +73,8 @@ public JoinBolt(Selector type, String srcOrStreamId, String fieldName) { } /** - * Optional. Allows naming the output stream of this bolt. If not specified, the emits will happen on 'default' stream. + * Optional. Allows naming the output stream of this bolt. If not specified, the emits will + * happen on 'default' stream. */ public JoinBolt withOutputStream(String streamName) { this.outputStreamName = streamName; @@ -91,10 +100,12 @@ public JoinBolt join(String newStream, String field, String priorStream) { } /** - * Performs left Join with the newStream. SQL : from stream1 left join stream2 on stream2.field = stream1.field1 same as: new + * Performs left Join with the newStream. SQL : from stream1 left join stream2 on stream2.field + * = stream1.field1 same as: new * WindowedQueryBolt(stream1, field1). leftJoin(stream2, field, stream1); * - *

    Note: priorStream must be previously joined Valid ex: new WindowedQueryBolt(s1,k1). leftJoin(s2,k2, s1). leftJoin(s3,k3, s2); + *

    Note: priorStream must be previously joined Valid ex: new WindowedQueryBolt(s1,k1). + * leftJoin(s2,k2, s1). leftJoin(s3,k3, s2); * Invalid ex: new WindowedQueryBolt(s1,k1). leftJoin(s3,k3, s2). leftJoin(s2,k2, s1); * * @param newStream Either a name of a stream or an upstream component @@ -104,14 +115,17 @@ public JoinBolt leftJoin(String newStream, String field, String priorStream) { return joinCommon(newStream, field, priorStream, JoinType.LEFT); } - private JoinBolt joinCommon(String newStream, String fieldDescriptor, String priorStream, JoinType joinType) { + private JoinBolt joinCommon(String newStream, String fieldDescriptor, String priorStream, + JoinType joinType) { if (hashedInputs.containsKey(newStream)) { - throw new IllegalArgumentException("'" + newStream + "' is already part of join. Cannot join with it more than once."); + throw new IllegalArgumentException("'" + newStream + + "' is already part of join. Cannot join with it more than once."); } hashedInputs.put(newStream, new HashMap>()); JoinInfo joinInfo = joinCriteria.get(priorStream); if (joinInfo == null) { - throw new IllegalArgumentException("Stream '" + priorStream + "' was not previously declared"); + throw new IllegalArgumentException("Stream '" + priorStream + + "' was not previously declared"); } FieldSelector field = new FieldSelector(newStream, fieldDescriptor); @@ -120,9 +134,12 @@ private JoinBolt joinCommon(String newStream, String fieldDescriptor, String pri } /** - * Specify projection fields. i.e. Specifies the fields to include in the output. e.g: .select("field1, stream2:field2, field3") Nested - * Key names are supported for nested types: e.g: .select("outerKey1.innerKey1, outerKey1.innerKey2, stream3:outerKey2.innerKey3)" Inner - * types (non leaf) must be Map<> in order to support nested lookup using this dot notation This selected fields implicitly declare the + * Specify projection fields. i.e. Specifies the fields to include in the output. e.g: + * .select("field1, stream2:field2, field3") Nested + * Key names are supported for nested types: e.g: .select("outerKey1.innerKey1, + * outerKey1.innerKey2, stream3:outerKey2.innerKey3)" Inner + * types (non leaf) must be Map<> in order to support nested lookup using this dot notation This + * selected fields implicitly declare the * output fieldNames for the bolt based. */ public JoinBolt select(String commaSeparatedKeys) { @@ -149,7 +166,8 @@ public void declareOutputFields(OutputFieldsDeclarer declarer) { } @Override - public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) { + public void prepare(Map topoConf, TopologyContext context, + OutputCollector collector) { this.collector = collector; // initialize the hashedInputs data structure int i = 0; @@ -174,11 +192,13 @@ public void execute(TupleWindow inputWindow) { for (ResultRecord resultRecord : joinResult.getRecords()) { ArrayList outputTuple = resultRecord.getOutputFields(); if (outputStreamName == null) { - // explicit anchoring emits to corresponding input tuples only, as default window anchoring will anchor them to all + // explicit anchoring emits to corresponding input tuples only, as default window + // anchoring will anchor them to all // tuples in window collector.emit(resultRecord.tupleList, outputTuple); } else { - // explicitly anchor emits to corresponding input tuples only, as default window anchoring will anchor them to all tuples + // explicitly anchor emits to corresponding input tuples only, as default window + // anchoring will anchor them to all tuples // in window collector.emit(outputStreamName, resultRecord.tupleList, outputTuple); } @@ -221,7 +241,8 @@ protected JoinAccumulator hashJoin(List tuples) { for (String streamName : joinCriteria.keySet()) { boolean finalJoin = (i == joinCriteria.size() - 1); if (i > 0) { - probe = doJoin(probe, hashedInputs.get(streamName), joinCriteria.get(streamName), finalJoin); + probe = doJoin(probe, hashedInputs.get(streamName), joinCriteria.get(streamName), + finalJoin); } ++i; } @@ -231,7 +252,8 @@ protected JoinAccumulator hashJoin(List tuples) { } // Dispatches to the right join method (inner/left/right/outer) based on the joinInfo.joinType - protected JoinAccumulator doJoin(JoinAccumulator probe, HashMap> buildInput, JoinInfo joinInfo, + protected JoinAccumulator doJoin(JoinAccumulator probe, HashMap> buildInput, JoinInfo joinInfo, boolean finalJoin) { final JoinType joinType = joinInfo.getJoinType(); switch (joinType) { @@ -247,11 +269,13 @@ protected JoinAccumulator doJoin(JoinAccumulator probe, HashMap> buildInput, JoinInfo joinInfo, + protected JoinAccumulator doInnerJoin(JoinAccumulator probe, Map> buildInput, JoinInfo joinInfo, boolean finalJoin) { String[] probeKeyName = joinInfo.getOtherField(); JoinAccumulator result = new JoinAccumulator(); - FieldSelector fieldSelector = new FieldSelector(joinInfo.other.getStreamName(), probeKeyName); + FieldSelector fieldSelector = new FieldSelector(joinInfo.other.getStreamName(), + probeKeyName); for (ResultRecord rec : probe.getRecords()) { Object probeKey = rec.getField(fieldSelector); if (probeKey != null) { @@ -268,15 +292,18 @@ protected JoinAccumulator doInnerJoin(JoinAccumulator probe, Map> buildInput, JoinInfo joinInfo, + protected JoinAccumulator doLeftJoin(JoinAccumulator probe, Map> buildInput, JoinInfo joinInfo, boolean finalJoin) { String[] probeKeyName = joinInfo.getOtherField(); JoinAccumulator result = new JoinAccumulator(); - FieldSelector fieldSelector = new FieldSelector(joinInfo.other.getStreamName(), probeKeyName); + FieldSelector fieldSelector = new FieldSelector(joinInfo.other.getStreamName(), + probeKeyName); for (ResultRecord rec : probe.getRecords()) { Object probeKey = rec.getField(fieldSelector); if (probeKey != null) { - ArrayList matchingBuildRecs = buildInput.get(probeKey); // ok if its return null + ArrayList matchingBuildRecs = buildInput + .get(probeKey); // ok if its return null if (matchingBuildRecs != null && !matchingBuildRecs.isEmpty()) { for (Tuple matchingRec : matchingBuildRecs) { ResultRecord mergedRecord = new ResultRecord(rec, matchingRec, finalJoin); @@ -292,11 +319,13 @@ protected JoinAccumulator doLeftJoin(JoinAccumulator probe, Map doProjection(ArrayList tuples, FieldSelector[] projectionFields) { + protected ArrayList doProjection(ArrayList tuples, + FieldSelector[] projectionFields) { ArrayList result = new ArrayList<>(projectionFields.length); - // TODO: optimize this computation... perhaps inner loop can be outside to avoid rescanning tuples + // TODO: optimize this computation... perhaps inner loop can be outside to avoid rescanning + // tuples for (int i = 0; i < projectionFields.length; i++) { boolean missingField = true; for (Tuple tuple : tuples) { @@ -451,14 +483,14 @@ protected static class JoinInfo implements Serializable { private FieldSelector field; // field for the current stream private FieldSelector other; // field for the other (2nd) stream - public JoinInfo(FieldSelector field) { this.joinType = null; this.field = field; this.other = null; } - public JoinInfo(FieldSelector field, String otherStream, JoinInfo otherStreamJoinInfo, JoinType joinType) { + public JoinInfo(FieldSelector field, String otherStream, JoinInfo otherStreamJoinInfo, + JoinType joinType) { this.joinType = joinType; this.field = field; this.other = new FieldSelector(otherStream, otherStreamJoinInfo.field.getOutputName()); @@ -512,13 +544,16 @@ public FieldSelector(String fieldDescriptor) { // sample fieldDescriptor = "str * Constructor. * * @param stream name of stream - * @param fieldDescriptor Simple fieldDescriptor like "x.y.z" and w/o a 'stream1:' stream qualifier. + * @param fieldDescriptor Simple fieldDescriptor like "x.y.z" and w/o a 'stream1:' stream + * qualifier. */ public FieldSelector(String stream, String fieldDescriptor) { this(fieldDescriptor); if (fieldDescriptor.indexOf(":") >= 0) { - throw new IllegalArgumentException("Not expecting stream qualifier ':' in '" + fieldDescriptor - + "'. Stream name '" + stream + "' is implicit in this context"); + throw new IllegalArgumentException("Not expecting stream qualifier ':' in '" + + fieldDescriptor + + "'. Stream name '" + stream + + "' is implicit in this context"); } this.streamName = stream; } @@ -527,7 +562,6 @@ public FieldSelector(String stream, String[] field) { this(stream, String.join(".", field)); } - public String getStreamName() { return streamName; } @@ -549,10 +583,12 @@ public String toString() { // Join helper to concat fields to the record protected class ResultRecord { - ArrayList tupleList = new ArrayList<>(); // contains one Tuple per Stream being joined + ArrayList tupleList = + new ArrayList<>(); // contains one Tuple per Stream being joined ArrayList outFields = null; // refs to fields that will be part of output fields - // 'generateOutputFields' enables us to avoid projection unless it is the final stream being joined + // 'generateOutputFields' enables us to avoid projection unless it is the final stream being + // joined public ResultRecord(Tuple tuple, boolean generateOutputFields) { tupleList.add(tuple); if (generateOutputFields) { @@ -576,7 +612,6 @@ public ArrayList getOutputFields() { return outFields; } - // 'stream' cannot be null, public Object getField(FieldSelector fieldSelector) { for (Tuple tuple : tupleList) { diff --git a/storm-client/src/jvm/org/apache/storm/callback/DefaultWatcherCallBack.java b/storm-client/src/jvm/org/apache/storm/callback/DefaultWatcherCallBack.java index 1b16fed8c07..525c227f6ec 100644 --- a/storm-client/src/jvm/org/apache/storm/callback/DefaultWatcherCallBack.java +++ b/storm-client/src/jvm/org/apache/storm/callback/DefaultWatcherCallBack.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -23,8 +29,10 @@ public class DefaultWatcherCallBack implements WatcherCallBack { private static Logger LOG = LoggerFactory.getLogger(DefaultWatcherCallBack.class); @Override - public void execute(Watcher.Event.KeeperState state, Watcher.Event.EventType type, String path) { - LOG.debug("Zookeeper state update: {}, {}, {}", ZkKeeperStates.getStateName(state), ZkEventTypes.getTypeName(type), path); + public void execute(Watcher.Event.KeeperState state, Watcher.Event.EventType type, + String path) { + LOG.debug("Zookeeper state update: {}, {}, {}", ZkKeeperStates.getStateName(state), + ZkEventTypes.getTypeName(type), path); } } diff --git a/storm-client/src/jvm/org/apache/storm/callback/WatcherCallBack.java b/storm-client/src/jvm/org/apache/storm/callback/WatcherCallBack.java index f6b548ac90b..ea23bcd1714 100644 --- a/storm-client/src/jvm/org/apache/storm/callback/WatcherCallBack.java +++ b/storm-client/src/jvm/org/apache/storm/callback/WatcherCallBack.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/callback/ZKStateChangedCallback.java b/storm-client/src/jvm/org/apache/storm/callback/ZKStateChangedCallback.java index 1a1599734c0..6a1ba971479 100644 --- a/storm-client/src/jvm/org/apache/storm/callback/ZKStateChangedCallback.java +++ b/storm-client/src/jvm/org/apache/storm/callback/ZKStateChangedCallback.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/cluster/ClusterStateContext.java b/storm-client/src/jvm/org/apache/storm/cluster/ClusterStateContext.java index 9f5f8c626f5..2fab8820fe9 100644 --- a/storm-client/src/jvm/org/apache/storm/cluster/ClusterStateContext.java +++ b/storm-client/src/jvm/org/apache/storm/cluster/ClusterStateContext.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -19,7 +25,8 @@ import org.apache.storm.shade.org.apache.zookeeper.data.ACL; /** - * This class is intended to provide runtime-context to StateStorageFactory implementors, giving information such as what daemon is creating + * This class is intended to provide runtime-context to StateStorageFactory implementors, giving + * information such as what daemon is creating * it. */ public class ClusterStateContext { diff --git a/storm-client/src/jvm/org/apache/storm/cluster/ClusterStateListener.java b/storm-client/src/jvm/org/apache/storm/cluster/ClusterStateListener.java index f0968a14d51..7f64e07e01b 100644 --- a/storm-client/src/jvm/org/apache/storm/cluster/ClusterStateListener.java +++ b/storm-client/src/jvm/org/apache/storm/cluster/ClusterStateListener.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/cluster/ClusterUtils.java b/storm-client/src/jvm/org/apache/storm/cluster/ClusterUtils.java index 0c322d9ace6..4477a1c970b 100644 --- a/storm-client/src/jvm/org/apache/storm/cluster/ClusterUtils.java +++ b/storm-client/src/jvm/org/apache/storm/cluster/ClusterUtils.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -45,7 +51,8 @@ public class ClusterUtils { public static final String LEADERINFO_ROOT = "leader-info"; public static final String ERRORS_ROOT = "errors"; public static final String BLOBSTORE_ROOT = "blobstore"; - public static final String BLOBSTORE_MAX_KEY_SEQUENCE_NUMBER_ROOT = "blobstoremaxkeysequencenumber"; + public static final String BLOBSTORE_MAX_KEY_SEQUENCE_NUMBER_ROOT = + "blobstoremaxkeysequencenumber"; public static final String NIMBUSES_ROOT = "nimbuses"; public static final String CREDENTIALS_ROOT = "credentials"; public static final String LOGCONFIG_ROOT = "logconfigs"; @@ -60,7 +67,8 @@ public class ClusterUtils { public static final String LEADERINFO_SUBTREE = ZK_SEPERATOR + LEADERINFO_ROOT; public static final String ERRORS_SUBTREE = ZK_SEPERATOR + ERRORS_ROOT; public static final String BLOBSTORE_SUBTREE = ZK_SEPERATOR + BLOBSTORE_ROOT; - public static final String BLOBSTORE_MAX_KEY_SEQUENCE_NUMBER_SUBTREE = ZK_SEPERATOR + BLOBSTORE_MAX_KEY_SEQUENCE_NUMBER_ROOT; + public static final String BLOBSTORE_MAX_KEY_SEQUENCE_NUMBER_SUBTREE = ZK_SEPERATOR + + BLOBSTORE_MAX_KEY_SEQUENCE_NUMBER_ROOT; public static final String NIMBUSES_SUBTREE = ZK_SEPERATOR + NIMBUSES_ROOT; public static final String CREDENTIALS_SUBTREE = ZK_SEPERATOR + CREDENTIALS_ROOT; public static final String LOGCONFIG_SUBTREE = ZK_SEPERATOR + LOGCONFIG_ROOT; @@ -73,7 +81,8 @@ public class ClusterUtils { private static ClusterUtils _instance = INSTANCE; /** - * Provide an instance of this class for delegates to use. To mock out delegated methods, provide an instance of a subclass that + * Provide an instance of this class for delegates to use. To mock out delegated methods, + * provide an instance of a subclass that * overrides the implementation of the delegated method. * * @param u a Cluster instance @@ -83,7 +92,8 @@ public static void setInstance(ClusterUtils u) { } /** - * Resets the singleton instance to the default. This is helpful to reset the class to its original functionality when mocking is no + * Resets the singleton instance to the default. This is helpful to reset the class to its + * original functionality when mocking is no * longer desired. */ public static void resetInstance() { @@ -118,10 +128,11 @@ private static List mkTopoAcls(Map topoConf, int perms) { ACL acl1 = ZooDefs.Ids.CREATOR_ALL_ACL.get(0); aclList.add(acl1); try { - ACL acl2 = new ACL(perms, new Id("digest", DigestAuthenticationProvider.generateDigest(payload))); + ACL acl2 = new ACL(perms, new Id("digest", DigestAuthenticationProvider + .generateDigest(payload))); aclList.add(acl2); } catch (NoSuchAlgorithmException e) { - //Should only happen on a badly configured system + // Should only happen on a badly configured system throw new RuntimeException(e); } } @@ -209,7 +220,8 @@ public static String profilerConfigPath(String stormId) { return PROFILERCONFIG_SUBTREE + ZK_SEPERATOR + stormId; } - public static String profilerConfigPath(String stormId, String host, Long port, ProfileAction requestType) { + public static String profilerConfigPath(String stormId, String host, Long port, + ProfileAction requestType) { return profilerConfigPath(stormId) + ZK_SEPERATOR + host + "_" + port + "_" + requestType; } @@ -242,7 +254,8 @@ public static String secretKeysPath(WorkerTokenServiceType type, String topology * @param version the version the secret is for. * @return the path to the secret. */ - public static String secretKeysPath(WorkerTokenServiceType type, String topologyId, long version) { + public static String secretKeysPath(WorkerTokenServiceType type, String topologyId, + long version) { return secretKeysPath(type, topologyId) + ZK_SEPERATOR + version; } @@ -272,18 +285,22 @@ public static Map convertExecutorBeats(List config, Map authConf, + public static IStateStorage mkStateStorage(Map config, Map authConf, ClusterStateContext context) throws Exception { return _instance.mkStateStorageImpl(config, authConf, context); } - public static IStormClusterState mkStormClusterState(Object stateStorage, ILocalAssignmentsBackend backend, + public static IStormClusterState mkStormClusterState(Object stateStorage, + ILocalAssignmentsBackend backend, ClusterStateContext context) throws Exception { return _instance.mkStormClusterStateImpl(stateStorage, backend, context); } - public static IStormClusterState mkStormClusterState(Object stateStorage, ClusterStateContext context) throws Exception { - return _instance.mkStormClusterStateImpl(stateStorage, LocalAssignmentsBackendFactory.getDefault(), context); + public static IStormClusterState mkStormClusterState(Object stateStorage, + ClusterStateContext context) throws Exception { + return _instance.mkStormClusterStateImpl(stateStorage, LocalAssignmentsBackendFactory + .getDefault(), context); } public static String stringifyError(Throwable error) { @@ -293,18 +310,21 @@ public static String stringifyError(Throwable error) { return result.toString(); } - public IStormClusterState mkStormClusterStateImpl(Object stateStorage, ILocalAssignmentsBackend backend, + public IStormClusterState mkStormClusterStateImpl(Object stateStorage, + ILocalAssignmentsBackend backend, ClusterStateContext context) throws Exception { if (stateStorage instanceof IStateStorage) { return new StormClusterStateImpl((IStateStorage) stateStorage, backend, context, false); } else { IStateStorage storage = _instance.mkStateStorageImpl((Map) stateStorage, - (Map) stateStorage, context); + (Map) stateStorage, context); return new StormClusterStateImpl(storage, backend, context, true); } } - public IStateStorage mkStateStorageImpl(Map config, Map authConf, ClusterStateContext context) throws + public IStateStorage mkStateStorageImpl(Map config, Map authConf, ClusterStateContext context) throws Exception { String className = null; IStateStorage stateStorage = null; diff --git a/storm-client/src/jvm/org/apache/storm/cluster/ConnectionState.java b/storm-client/src/jvm/org/apache/storm/cluster/ConnectionState.java index 0fa75382d7b..18dffe35f30 100644 --- a/storm-client/src/jvm/org/apache/storm/cluster/ConnectionState.java +++ b/storm-client/src/jvm/org/apache/storm/cluster/ConnectionState.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/cluster/DaemonType.java b/storm-client/src/jvm/org/apache/storm/cluster/DaemonType.java index 27df05b26fa..31054ecf5e8 100644 --- a/storm-client/src/jvm/org/apache/storm/cluster/DaemonType.java +++ b/storm-client/src/jvm/org/apache/storm/cluster/DaemonType.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -25,7 +31,8 @@ import org.slf4j.LoggerFactory; /** - * The type of process/daemon that this server is running as. This is used with the IStormClusterState implementation to know how to + * The type of process/daemon that this server is running as. This is used with the + * IStormClusterState implementation to know how to * properly secure data stored in it. */ public enum DaemonType { @@ -44,13 +51,14 @@ public List getDefaultZkAcls(Map conf) { @Override public List getZkSecretAcls(WorkerTokenServiceType type, Map conf) { if (!Utils.isZkAuthenticationConfiguredStormServer(conf)) { - //This is here only for testing. - LOG.error("Will Store Worker Token Keys in ZK without ACLs. If you are not running tests STOP NOW!"); + // This is here only for testing. + LOG.error("Will Store Worker Token Keys in ZK without ACLs. If you are not " + + "running tests STOP NOW!"); return null; } switch (type) { case NIMBUS: - //Fall through on purpose + // Fall through on purpose case SUPERVISOR: return ZooDefs.Ids.CREATOR_ALL_ACL; case DRPC: @@ -59,10 +67,11 @@ public List getZkSecretAcls(WorkerTokenServiceType type, Maphttp://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/cluster/IStateStorage.java b/storm-client/src/jvm/org/apache/storm/cluster/IStateStorage.java index 61b4c82d34a..5635a616952 100644 --- a/storm-client/src/jvm/org/apache/storm/cluster/IStateStorage.java +++ b/storm-client/src/jvm/org/apache/storm/cluster/IStateStorage.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -19,15 +25,20 @@ import org.apache.storm.shade.org.apache.zookeeper.data.ACL; /** - * StateStorage provides the API for the pluggable state store used by the Storm daemons. Data is stored in path/value - * format, and the store supports listing sub-paths at a given path. All data should be available across all nodes with + * StateStorage provides the API for the pluggable state store used by the Storm daemons. Data is + * stored in path/value + * format, and the store supports listing sub-paths at a given path. All data should be available + * across all nodes with * eventual consistency. * - *

    IMPORTANT NOTE: Heartbeats have different api calls used to interact with them. The root path (/) may or may not + *

    IMPORTANT NOTE: Heartbeats have different api calls used to interact with them. The root path + * (/) may or may not * be the same as the root path for the other api calls. * - *

    For example, performing these two calls: set_data("/path", data, acls); void set_worker_hb("/path", heartbeat, - * acls); may or may not cause a collision in "/path". Never use the same paths with the *_hb* methods as you do with + *

    For example, performing these two calls: set_data("/path", data, acls); void + * set_worker_hb("/path", heartbeat, + * acls); may or may not cause a collision in "/path". Never use the same paths with the *_hb* + * methods as you do with * the others. */ public interface IStateStorage extends Closeable { @@ -35,7 +46,8 @@ public interface IStateStorage extends Closeable { /** * Registers a callback function that gets called when CuratorEvents happen. * - * @param callback is a clojure IFn that accepts the type - translated to clojure keyword as in zookeeper - and the path: (callback type + * @param callback is a clojure IFn that accepts the type - translated to clojure keyword as in + * zookeeper - and the path: (callback type * path) * @return is an id that can be passed to unregister(...) to unregister the callback. */ @@ -49,7 +61,8 @@ public interface IStateStorage extends Closeable { void unregister(String id); /** - * Path will be appended with a monotonically increasing integer, a new node will be created there, and data will be put at that node. + * Path will be appended with a monotonically increasing integer, a new node will be created + * there, and data will be put at that node. * * @param path The path that the monotonically increasing integer suffix will be added to. * @param data The data that will be written at the suffixed path's node. @@ -59,7 +72,8 @@ public interface IStateStorage extends Closeable { String create_sequential(String path, byte[] data, List acls); /** - * Creates nodes for path and all its parents. Path elements are separated by a "/", as in *nix filesystem notation. Equivalent to mkdir + * Creates nodes for path and all its parents. Path elements are separated by a "/", as in *nix + * filesystem notation. Equivalent to mkdir * -p in *nix. * * @param path The path to create, along with all its parents. @@ -75,7 +89,8 @@ public interface IStateStorage extends Closeable { void delete_node(String path); /** - * Creates an ephemeral node at path. Ephemeral nodes are destroyed by the store when the client disconnects. + * Creates an ephemeral node at path. Ephemeral nodes are destroyed by the store when the client + * disconnects. * * @param path The path where a node will be created. * @param data The data to be written at the node. @@ -84,11 +99,13 @@ public interface IStateStorage extends Closeable { void set_ephemeral_node(String path, byte[] data, List acls); /** - * Gets the 'version' of the node at a path. Optionally sets a watch on that node. The version should increase whenever a write + * Gets the 'version' of the node at a path. Optionally sets a watch on that node. The version + * should increase whenever a write * happens. * * @param path The path to get the version of. - * @param watch Whether or not to set a watch on the path. Watched paths emit events which are consumed by functions registered with the + * @param watch Whether or not to set a watch on the path. Watched paths emit events which are + * consumed by functions registered with the * register method. Very useful for catching updates to nodes. * @return The integer version of this node. */ @@ -98,7 +115,8 @@ public interface IStateStorage extends Closeable { * Check if a node exists and optionally set a watch on the path. * * @param path The path to check for the existence of a node. - * @param watch Whether or not to set a watch on the path. Watched paths emit events which are consumed by functions registered with the + * @param watch Whether or not to set a watch on the path. Watched paths emit events which are + * consumed by functions registered with the * register method. Very useful for catching updates to nodes. * @return Whether or not a node exists at path. */ @@ -108,7 +126,8 @@ public interface IStateStorage extends Closeable { * Get a list of paths of all the child nodes which exist immediately under path. * * @param path The path to look under - * @param watch Whether or not to set a watch on the path. Watched paths emit events which are consumed by functions registered with the + * @param watch Whether or not to set a watch on the path. Watched paths emit events which are + * consumed by functions registered with the * register method. Very useful for catching updates to nodes. * @return list of string paths under path. */ @@ -130,20 +149,23 @@ public interface IStateStorage extends Closeable { void set_data(String path, byte[] data, List acls); /** - * Get the data from the node at path + * Get the data from the node at path. * * @param path The path to look under - * @param watch Whether or not to set a watch on the path. Watched paths emit events which are consumed by functions registered with the + * @param watch Whether or not to set a watch on the path. Watched paths emit events which are + * consumed by functions registered with the * register method. Very useful for catching updates to nodes. * @return The data at the node. */ byte[] get_data(String path, boolean watch); /** - * Get the data at the node along with its version. Data is returned in an Map with the keys data and version. + * Get the data at the node along with its version. Data is returned in an Map with the keys + * data and version. * * @param path The path to look under - * @param watch Whether or not to set a watch on the path. Watched paths emit events which are consumed by functions registered with the + * @param watch Whether or not to set a watch on the path. Watched paths emit events which are + * consumed by functions registered with the * register method. Very useful for catching updates to nodes. * @return the data with a version */ @@ -159,21 +181,24 @@ public interface IStateStorage extends Closeable { void set_worker_hb(String path, byte[] data, List acls); /** - * Get the heartbeat from the node at path + * Get the heartbeat from the node at path. * * @param path The path to look under - * @param watch Whether or not to set a watch on the path. Watched paths emit events which are consumed by functions registered with the + * @param watch Whether or not to set a watch on the path. Watched paths emit events which are + * consumed by functions registered with the * register method. Very useful for catching updates to nodes. * @return The heartbeat at the node. */ byte[] get_worker_hb(String path, boolean watch); /** - * Get a list of paths of all the child nodes which exist immediately under path. This is similar to get_children, but must be used for + * Get a list of paths of all the child nodes which exist immediately under path. This is + * similar to get_children, but must be used for * any nodes * * @param path The path to look under - * @param watch Whether or not to set a watch on the path. Watched paths emit events which are consumed by functions registered with the + * @param watch Whether or not to set a watch on the path. Watched paths emit events which are + * consumed by functions registered with the * register method. Very useful for catching updates to nodes. * @return list of string paths under path. */ @@ -194,14 +219,16 @@ public interface IStateStorage extends Closeable { void add_listener(ConnectionStateListener listener); /** - * Force consistency on a path. Any writes committed on the path before this call will be completely propagated when it returns. + * Force consistency on a path. Any writes committed on the path before this call will be + * completely propagated when it returns. * * @param path The path to synchronize. */ void sync_path(String path); /** - * Allows us to delete the znodes within /storm/blobstore/key_name whose znodes start with the corresponding nimbusHostPortInfo. + * Allows us to delete the znodes within /storm/blobstore/key_name whose znodes start with the + * corresponding nimbusHostPortInfo. * * @param path /storm/blobstore/key_name * @param nimbusHostPortInfo Contains the host port information of a nimbus node. diff --git a/storm-client/src/jvm/org/apache/storm/cluster/IStormClusterState.java b/storm-client/src/jvm/org/apache/storm/cluster/IStormClusterState.java index d0987002c3d..79ee9815573 100644 --- a/storm-client/src/jvm/org/apache/storm/cluster/IStormClusterState.java +++ b/storm-client/src/jvm/org/apache/storm/cluster/IStormClusterState.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -64,12 +70,14 @@ public interface IStormClusterState { * Sync the remote state store assignments to local backend, used when master gains leadership, * see {@code org.apache.storm.nimbus.LeaderListenerCallback}. * - * @param remote assigned assignments for a specific {@link IStormClusterState} instance, usually a supervisor/node. + * @param remote assigned assignments for a specific {@link IStormClusterState} instance, + * usually a supervisor/node. */ void syncRemoteAssignments(Map remote); /** - * Flag to indicate if the assignments synced successfully, see {@link IStormClusterState#syncRemoteAssignments(Map)}. + * Flag to indicate if the assignments synced successfully, see {@link + * IStormClusterState#syncRemoteAssignments(Map)}. * * @return true if is synced successfully */ @@ -83,7 +91,8 @@ public interface IStormClusterState { boolean isPacemakerStateStore(); /** - * Mark the assignments as synced successfully, see {@link IStormClusterState#isAssignmentsBackendSynchronized()}. + * Mark the assignments as synced successfully, see {@link + * IStormClusterState#isAssignmentsBackendSynchronized()}. */ void setAssignmentsBackendSynchronized(); @@ -133,7 +142,8 @@ public interface IStormClusterState { void deleteTopologyProfileRequests(String stormId, ProfileRequest profileRequest); - Map executorBeats(String stormId, Map, NodeInfo> executorNodePort); + Map executorBeats(String stormId, Map, + NodeInfo> executorNodePort); List supervisors(Runnable callback); @@ -159,7 +169,8 @@ public interface IStormClusterState { /** * Get leader info from state store, which was written when a master gains leadership. * - *

    Caution: it can not be used for fencing and is only for informational purposes because we use ZK as our + *

    Caution: it can not be used for fencing and is only for informational purposes because we + * use ZK as our * backend now, which could have a overdue info of nodes. * * @param callback callback func @@ -240,7 +251,8 @@ public interface IStormClusterState { void disconnect(); /** - * Get a private key used to validate a token is correct. This is expected to be called from a privileged daemon, and the ACLs should be + * Get a private key used to validate a token is correct. This is expected to be called from a + * privileged daemon, and the ACLs should be * set up to only allow nimbus and these privileged daemons access to these private keys. * * @param type the type of service the key is for. @@ -248,10 +260,12 @@ public interface IStormClusterState { * @param keyVersion the version of the key this is for. * @return the private key or null if it could not be found. */ - PrivateWorkerKey getPrivateWorkerKey(WorkerTokenServiceType type, String topologyId, long keyVersion); + PrivateWorkerKey getPrivateWorkerKey(WorkerTokenServiceType type, String topologyId, + long keyVersion); /** - * Store a new version of a private key. This is expected to only ever be called from nimbus. All ACLs however need to be setup to + * Store a new version of a private key. This is expected to only ever be called from nimbus. + * All ACLs however need to be setup to * allow the given services access to the stored information. * * @param type the type of service this key is for. @@ -259,11 +273,14 @@ public interface IStormClusterState { * @param keyVersion the version of the key this is for. * @param key the key to store. */ - void addPrivateWorkerKey(WorkerTokenServiceType type, String topologyId, long keyVersion, PrivateWorkerKey key); + void addPrivateWorkerKey(WorkerTokenServiceType type, String topologyId, long keyVersion, + PrivateWorkerKey key); /** - * Get the next key version number that should be used for this topology id. This is expected to only ever be called from nimbus, but it - * is acceptable if the ACLs are setup so that it can work from a privileged daemon for the given service. + * Get the next key version number that should be used for this topology id. This is expected to + * only ever be called from nimbus, but it + * is acceptable if the ACLs are setup so that it can work from a privileged daemon for the + * given service. * * @param type the type of service this is for. * @param topologyId the topology id this is for. @@ -272,9 +289,12 @@ public interface IStormClusterState { long getNextPrivateWorkerKeyVersion(WorkerTokenServiceType type, String topologyId); /** - * Remove all keys for the given topology that have expired. The number of keys should be small enough that doing an exhaustive scan of - * them all is acceptable as there is no guarantee that expiration time and version number are related. This should be for all service - * types. This is expected to only ever be called from nimbus and some ACLs may be setup so being called from other daemons will cause + * Remove all keys for the given topology that have expired. The number of keys should be small + * enough that doing an exhaustive scan of + * them all is acceptable as there is no guarantee that expiration time and version number are + * related. This should be for all service + * types. This is expected to only ever be called from nimbus and some ACLs may be setup so + * being called from other daemons will cause * it to fail. * * @param topologyId the id of the topology to scan. @@ -282,7 +302,8 @@ public interface IStormClusterState { void removeExpiredPrivateWorkerKeys(String topologyId); /** - * Remove all of the worker keys for a given topology. Used to clean up after a topology finishes. This is expected to only ever be + * Remove all of the worker keys for a given topology. Used to clean up after a topology + * finishes. This is expected to only ever be * called from nimbus and ideally should only ever work from nimbus. * * @param topologyId the topology to clean up after. @@ -290,7 +311,8 @@ public interface IStormClusterState { void removeAllPrivateWorkerKeys(String topologyId); /** - * Get a list of all topologyIds that currently have private worker keys stored, of any kind. This is expected to only ever be called + * Get a list of all topologyIds that currently have private worker keys stored, of any kind. + * This is expected to only ever be called * from nimbus. * * @return the list of topology ids with any kind of private worker key stored. @@ -306,6 +328,7 @@ default Map allSupervisorInfo() { /** * Get all supervisor info. + * * @param callback be alerted if the list of supervisors change * @return All of the supervisors with the ID as the key */ @@ -334,7 +357,7 @@ default Map topologyBases() { Map stormBases = new HashMap<>(); for (String topologyId : activeStorms()) { StormBase base = stormBase(topologyId, null); - if (base != null) { //race condition with delete + if (base != null) { // race condition with delete stormBases.put(topologyId, base); } } diff --git a/storm-client/src/jvm/org/apache/storm/cluster/PaceMakerStateStorage.java b/storm-client/src/jvm/org/apache/storm/cluster/PaceMakerStateStorage.java index effaeea11c3..d5009269e0d 100644 --- a/storm-client/src/jvm/org/apache/storm/cluster/PaceMakerStateStorage.java +++ b/storm-client/src/jvm/org/apache/storm/cluster/PaceMakerStateStorage.java @@ -40,8 +40,10 @@ /** * State storage that keeps worker heartbeats in Pacemaker and everything else in ZooKeeper. * - * @deprecated Pacemaker is deprecated and only kept for backward compatibility; it will be removed in a future release. - * Use the default heartbeat path instead: workers heartbeat to their supervisor, which reports them to Nimbus over + * @deprecated Pacemaker is deprecated and only kept for backward compatibility; it will be removed + * in a future release. + * Use the default heartbeat path instead: workers heartbeat to their supervisor, which reports + * them to Nimbus over * Thrift, with the default ZooKeeper-based cluster state store ({@code org.apache.storm.cluster.ZKStateStorageFactory}). */ @Deprecated @@ -52,7 +54,8 @@ public class PaceMakerStateStorage implements IStateStorage { private PacemakerClientPool pacemakerClientPool; private IStateStorage stateStorage; - public PaceMakerStateStorage(PacemakerClientPool pacemakerClientPool, IStateStorage stateStorage) throws Exception { + public PaceMakerStateStorage(PacemakerClientPool pacemakerClientPool, + IStateStorage stateStorage) throws Exception { this.pacemakerClientPool = pacemakerClientPool; this.stateStorage = stateStorage; } @@ -131,7 +134,8 @@ public void set_worker_hb(String path, byte[] data, List acls) { HBPulse hbPulse = new HBPulse(); hbPulse.set_id(path); hbPulse.set_details(data); - HBMessage message = new HBMessage(HBServerMessageType.SEND_PULSE, HBMessageData.pulse(hbPulse)); + HBMessage message = new HBMessage(HBServerMessageType.SEND_PULSE, HBMessageData + .pulse(hbPulse)); HBMessage response = pacemakerClientPool.send(message); if (response.get_type() != HBServerMessageType.SEND_PULSE_RESPONSE) { throw new WrappedHBExecutionException("Invalid Response Type"); @@ -143,7 +147,8 @@ public void set_worker_hb(String path, byte[] data, List acls) { throw new RuntimeException(e); } retry--; - LOG.error("{} Failed to set_worker_hb. Will make {} more attempts.", e.getMessage(), retry); + LOG.error("{} Failed to set_worker_hb. Will make {} more attempts.", e.getMessage(), + retry); } catch (InterruptedException e) { LOG.debug("set_worker_hb got interrupted: {}", e); throw new RuntimeException(e); @@ -160,7 +165,8 @@ public byte[] get_worker_hb(String path, boolean watch) { long latestTimeSecs = 0; boolean gotResponse = false; - HBMessage message = new HBMessage(HBServerMessageType.GET_PULSE, HBMessageData.path(path)); + HBMessage message = new HBMessage(HBServerMessageType.GET_PULSE, HBMessageData + .path(path)); List responses = pacemakerClientPool.sendAll(message); for (HBMessage response : responses) { if (response.get_type() != HBServerMessageType.GET_PULSE_RESPONSE) { @@ -173,7 +179,8 @@ public byte[] get_worker_hb(String path, boolean watch) { if (details == null) { continue; } - ClusterWorkerHeartbeat cwh = Utils.deserialize(details, ClusterWorkerHeartbeat.class); + ClusterWorkerHeartbeat cwh = Utils.deserialize(details, + ClusterWorkerHeartbeat.class); if (cwh != null && cwh.get_time_secs() > latestTimeSecs) { latestTimeSecs = cwh.get_time_secs(); ret = details; @@ -188,7 +195,8 @@ public byte[] get_worker_hb(String path, boolean watch) { throw new RuntimeException(e); } retry--; - LOG.error("{} Failed to get_worker_hb. Will make {} more attempts.", e.getMessage(), retry); + LOG.error("{} Failed to get_worker_hb. Will make {} more attempts.", e.getMessage(), + retry); } catch (InterruptedException e) { LOG.debug("get_worker_hb got interrupted: {}", e); throw new RuntimeException(e); @@ -203,10 +211,12 @@ public List get_worker_hb_children(String path, boolean watch) { try { HashSet retSet = new HashSet<>(); - HBMessage message = new HBMessage(HBServerMessageType.GET_ALL_NODES_FOR_PATH, HBMessageData.path(path)); + HBMessage message = new HBMessage(HBServerMessageType.GET_ALL_NODES_FOR_PATH, + HBMessageData.path(path)); List responses = pacemakerClientPool.sendAll(message); for (HBMessage response : responses) { - if (response.get_type() != HBServerMessageType.GET_ALL_NODES_FOR_PATH_RESPONSE) { + if (response + .get_type() != HBServerMessageType.GET_ALL_NODES_FOR_PATH_RESPONSE) { LOG.error("get_worker_hb_children: Invalid Response Type"); continue; } @@ -222,7 +232,8 @@ public List get_worker_hb_children(String path, boolean watch) { throw new RuntimeException(e); } retry--; - LOG.error("{} Failed to get_worker_hb_children. Will make {} more attempts.", e.getMessage(), retry); + LOG.error("{} Failed to get_worker_hb_children. Will make {} more attempts.", e + .getMessage(), retry); } catch (InterruptedException e) { LOG.debug("get_worker_hb_children got interrupted: {}", e); throw new RuntimeException(e); @@ -237,7 +248,8 @@ public void delete_worker_hb(String path) { while (true) { someSucceeded = false; try { - HBMessage message = new HBMessage(HBServerMessageType.DELETE_PATH, HBMessageData.path(path)); + HBMessage message = new HBMessage(HBServerMessageType.DELETE_PATH, HBMessageData + .path(path)); List responses = pacemakerClientPool.sendAll(message); boolean allSucceeded = true; for (HBMessage response : responses) { @@ -264,7 +276,8 @@ public void delete_worker_hb(String path) { } } retry--; - LOG.debug("{} Failed to delete_worker_hb. Will make {} more attempts.", e.getMessage(), retry); + LOG.debug("{} Failed to delete_worker_hb. Will make {} more attempts.", e + .getMessage(), retry); } catch (InterruptedException e) { LOG.debug("delete_worker_hb got interrupted: {}", e); throw new RuntimeException(e); diff --git a/storm-client/src/jvm/org/apache/storm/cluster/PaceMakerStateStorageFactory.java b/storm-client/src/jvm/org/apache/storm/cluster/PaceMakerStateStorageFactory.java index 6b8e3bf3284..41995086e4a 100644 --- a/storm-client/src/jvm/org/apache/storm/cluster/PaceMakerStateStorageFactory.java +++ b/storm-client/src/jvm/org/apache/storm/cluster/PaceMakerStateStorageFactory.java @@ -25,14 +25,17 @@ /** * Factory for {@link PaceMakerStateStorage}. * - * @deprecated Pacemaker is deprecated and only kept for backward compatibility; it will be removed in a future release. - * Use the default heartbeat path instead: workers heartbeat to their supervisor, which reports them to Nimbus over + * @deprecated Pacemaker is deprecated and only kept for backward compatibility; it will be removed + * in a future release. + * Use the default heartbeat path instead: workers heartbeat to their supervisor, which reports + * them to Nimbus over * Thrift, with the default ZooKeeper-based cluster state store ({@code org.apache.storm.cluster.ZKStateStorageFactory}). */ @Deprecated public class PaceMakerStateStorageFactory implements StateStorageFactory { @Override - public IStateStorage mkStore(Map config, Map authConf, ClusterStateContext context) { + public IStateStorage mkStore(Map config, Map authConf, + ClusterStateContext context) { try { ZKStateStorageFactory zkfact = new ZKStateStorageFactory(); IStateStorage zkState = zkfact.mkStore(config, authConf, context); diff --git a/storm-client/src/jvm/org/apache/storm/cluster/StateStorageFactory.java b/storm-client/src/jvm/org/apache/storm/cluster/StateStorageFactory.java index fb321f73672..f46c5164ec7 100644 --- a/storm-client/src/jvm/org/apache/storm/cluster/StateStorageFactory.java +++ b/storm-client/src/jvm/org/apache/storm/cluster/StateStorageFactory.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -16,5 +22,6 @@ public interface StateStorageFactory { - IStateStorage mkStore(Map config, Map authConf, ClusterStateContext context); + IStateStorage mkStore(Map config, Map authConf, + ClusterStateContext context); } diff --git a/storm-client/src/jvm/org/apache/storm/cluster/StormClusterStateImpl.java b/storm-client/src/jvm/org/apache/storm/cluster/StormClusterStateImpl.java index 8be795c2ba2..3aebaae8575 100644 --- a/storm-client/src/jvm/org/apache/storm/cluster/StormClusterStateImpl.java +++ b/storm-client/src/jvm/org/apache/storm/cluster/StormClusterStateImpl.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -73,7 +79,8 @@ public class StormClusterStateImpl implements IStormClusterState { private ConcurrentHashMap credentialsCallback; private ConcurrentHashMap logConfigCallback; - public StormClusterStateImpl(IStateStorage stateStorage, ILocalAssignmentsBackend assignmentsassignmentsBackend, + public StormClusterStateImpl(IStateStorage stateStorage, + ILocalAssignmentsBackend assignmentsassignmentsBackend, ClusterStateContext context, boolean shouldCloseStateStorageOnDisconnect) throws Exception { this.stateStorage = stateStorage; @@ -126,7 +133,8 @@ public void changed(Watcher.Event.EventType type, String path) { } else if (root.equals(ClusterUtils.LEADERINFO_ROOT)) { issueCallback(leaderInfoCallback); } else { - LOG.error("{} Unknown callback for subtree {}", new RuntimeException("Unknown callback for this path"), path); + LOG.error("{} Unknown callback for subtree {}", + new RuntimeException("Unknown callback for this path"), path); Runtime.getRuntime().exit(30); } @@ -172,7 +180,8 @@ protected void issueCallback(AtomicReference cb) { } } - protected void issueMapCallback(ConcurrentHashMap callbackConcurrentHashMap, String key) { + protected void issueMapCallback(ConcurrentHashMap callbackConcurrentHashMap, + String key) { Runnable callback = callbackConcurrentHashMap.remove(key); if (callback != null) { callback.run(); @@ -200,7 +209,8 @@ public Assignment remoteAssignmentInfo(String stormId, Runnable callback) { if (callback != null) { assignmentInfoCallback.put(stormId, callback); } - byte[] serialized = stateStorage.get_data(ClusterUtils.assignmentPath(stormId), callback != null); + byte[] serialized = stateStorage.get_data(ClusterUtils.assignmentPath(stormId), + callback != null); return ClusterUtils.maybeDeserialize(serialized, Assignment.class); } @@ -215,9 +225,11 @@ public void syncRemoteAssignments(Map remote) { this.assignmentsBackend.syncRemoteAssignments(remote); } else { Map tmp = new HashMap<>(); - List stormIds = this.stateStorage.get_children(ClusterUtils.ASSIGNMENTS_SUBTREE, false); + List stormIds = this.stateStorage.get_children(ClusterUtils.ASSIGNMENTS_SUBTREE, + false); for (String stormId : stormIds) { - byte[] assignment = this.stateStorage.get_data(ClusterUtils.assignmentPath(stormId), false); + byte[] assignment = this.stateStorage.get_data(ClusterUtils.assignmentPath(stormId), + false); tmp.put(stormId, assignment); } this.assignmentsBackend.syncRemoteAssignments(tmp); @@ -250,7 +262,8 @@ public VersionedData assignmentInfoWithVersion(String stormId, Runna } Assignment assignment = null; Integer version = 0; - VersionedData dataWithVersion = stateStorage.get_data_with_version(ClusterUtils.assignmentPath(stormId), callback != null); + VersionedData dataWithVersion = stateStorage.get_data_with_version(ClusterUtils + .assignmentPath(stormId), callback != null); if (dataWithVersion != null) { assignment = ClusterUtils.maybeDeserialize(dataWithVersion.getData(), Assignment.class); version = dataWithVersion.getVersion(); @@ -280,10 +293,12 @@ public List nimbuses() { List nimbusIds = stateStorage.get_children(ClusterUtils.NIMBUSES_SUBTREE, false); for (String nimbusId : nimbusIds) { byte[] serialized = stateStorage.get_data(ClusterUtils.nimbusPath(nimbusId), false); - // check for null which can exist because of a race condition in which nimbus nodes in zk may have been + // check for null which can exist because of a race condition in which nimbus nodes in + // zk may have been // removed when connections are reconnected after getting children in the above line if (serialized != null) { - NimbusSummary nimbusSummary = ClusterUtils.maybeDeserialize(serialized, NimbusSummary.class); + NimbusSummary nimbusSummary = ClusterUtils.maybeDeserialize(serialized, + NimbusSummary.class); nimbusSummaries.add(nimbusSummary); } } @@ -295,16 +310,20 @@ public void addNimbusHost(final String nimbusId, final NimbusSummary nimbusSumma // explicit delete for ephmeral node to ensure this session creates the entry. stateStorage.delete_node(ClusterUtils.nimbusPath(nimbusId)); stateStorage.add_listener((curatorFramework, connectionState) -> { - LOG.info("Connection state listener invoked, zookeeper connection state has changed to {}", connectionState); + LOG.info("Connection state listener invoked, zookeeper connection state has changed " + + "to {}", connectionState); if (connectionState.equals(ConnectionState.RECONNECTED)) { - LOG.info("Connection state has changed to reconnected so setting nimbuses entry one more time"); + LOG.info("Connection state has changed to reconnected so setting nimbuses entry " + + "one more time"); // explicit delete for ephemeral node to ensure this session creates the entry. stateStorage.delete_node(ClusterUtils.nimbusPath(nimbusId)); - stateStorage.set_ephemeral_node(ClusterUtils.nimbusPath(nimbusId), Utils.serialize(nimbusSummary), defaultAcls); + stateStorage.set_ephemeral_node(ClusterUtils.nimbusPath(nimbusId), Utils + .serialize(nimbusSummary), defaultAcls); } }); - stateStorage.set_ephemeral_node(ClusterUtils.nimbusPath(nimbusId), Utils.serialize(nimbusSummary), defaultAcls); + stateStorage.set_ephemeral_node(ClusterUtils.nimbusPath(nimbusId), Utils + .serialize(nimbusSummary), defaultAcls); } @Override @@ -317,7 +336,8 @@ public StormBase stormBase(String stormId, Runnable callback) { if (callback != null) { stormBaseCallback.put(stormId, callback); } - return ClusterUtils.maybeDeserialize(stateStorage.get_data(ClusterUtils.stormPath(stormId), callback != null), StormBase.class); + return ClusterUtils.maybeDeserialize(stateStorage.get_data(ClusterUtils.stormPath(stormId), + callback != null), StormBase.class); } @Override @@ -341,7 +361,8 @@ public void syncRemoteIds(Map remote) { @Override public ClusterWorkerHeartbeat getWorkerHeartbeat(String stormId, String node, Long port) { - byte[] bytes = stateStorage.get_worker_hb(ClusterUtils.workerbeatPath(stormId, node, port), false); + byte[] bytes = stateStorage.get_worker_hb(ClusterUtils.workerbeatPath(stormId, node, port), + false); return ClusterUtils.maybeDeserialize(bytes, ClusterWorkerHeartbeat.class); } @@ -396,8 +417,10 @@ public void deleteTopologyProfileRequests(String stormId, ProfileRequest profile } /** - * need to take executor->node+port in explicitly so that we don't run into a situation where a long dead worker with a skewed clock - * overrides all the timestamps. By only checking heartbeats with an assigned node+port, and only reading executors from that heartbeat + * Need to take executor->node+port in explicitly so that we don't run into a situation where a + * long dead worker with a skewed clock + * overrides all the timestamps. By only checking heartbeats with an assigned node+port, and + * only reading executors from that heartbeat * that are actually assigned, we avoid situations like that. * * @param stormId topology id @@ -405,7 +428,8 @@ public void deleteTopologyProfileRequests(String stormId, ProfileRequest profile * @return mapping of executorInfo -> executor beat */ @Override - public Map executorBeats(String stormId, Map, NodeInfo> executorNodePort) { + public Map executorBeats(String stormId, Map, + NodeInfo> executorNodePort) { Map executorWhbs = new HashMap<>(); Map>> nodePortExecutors = Utils.reverseMap(executorNodePort); @@ -417,7 +441,8 @@ public Map executorBeats(String stormId, Map executorInfoList = new ArrayList<>(); for (List list : entry.getValue()) { - executorInfoList.add(new ExecutorInfo(list.get(0).intValue(), list.get(list.size() - 1).intValue())); + executorInfoList.add(new ExecutorInfo(list.get(0).intValue(), list.get(list + .size() - 1).intValue())); } if (whb != null) { executorWhbs.putAll(ClusterUtils.convertExecutorBeats(executorInfoList, whb)); @@ -437,13 +462,15 @@ public List supervisors(Runnable callback) { @Override public SupervisorInfo supervisorInfo(String supervisorId) { String path = ClusterUtils.supervisorPath(supervisorId); - return ClusterUtils.maybeDeserialize(stateStorage.get_data(path, false), SupervisorInfo.class); + return ClusterUtils.maybeDeserialize(stateStorage.get_data(path, false), + SupervisorInfo.class); } @Override public void setupHeatbeats(String stormId, Map topoConf) { stateStorage.mkdirs(ClusterUtils.WORKERBEATS_SUBTREE, defaultAcls); - stateStorage.mkdirs(ClusterUtils.workerbeatStormRoot(stormId), ClusterUtils.mkTopoReadWriteAcls(topoConf)); + stateStorage.mkdirs(ClusterUtils.workerbeatStormRoot(stormId), ClusterUtils + .mkTopoReadWriteAcls(topoConf)); } @Override @@ -479,7 +506,8 @@ public NimbusInfo getLeader(Runnable callback) { if (null != callback) { this.leaderInfoCallback.set(callback); } - return Utils.javaDeserialize(this.stateStorage.get_data(ClusterUtils.LEADERINFO_SUBTREE, callback != null), NimbusInfo.class); + return Utils.javaDeserialize(this.stateStorage.get_data(ClusterUtils.LEADERINFO_SUBTREE, + callback != null), NimbusInfo.class); } @Override @@ -498,9 +526,11 @@ public List errorTopologies() { } @Override - public void setTopologyLogConfig(String stormId, LogConfig logConfig, Map topoConf) { + public void setTopologyLogConfig(String stormId, LogConfig logConfig, Map topoConf) { stateStorage.mkdirs(ClusterUtils.LOGCONFIG_SUBTREE, defaultAcls); - stateStorage.set_data(ClusterUtils.logConfigPath(stormId), Utils.serialize(logConfig), ClusterUtils.mkTopoReadOnlyAcls(topoConf)); + stateStorage.set_data(ClusterUtils.logConfigPath(stormId), Utils.serialize(logConfig), + ClusterUtils.mkTopoReadOnlyAcls(topoConf)); } @Override @@ -509,11 +539,13 @@ public LogConfig topologyLogConfig(String stormId, Runnable cb) { logConfigCallback.put(stormId, cb); } String path = ClusterUtils.logConfigPath(stormId); - return ClusterUtils.maybeDeserialize(stateStorage.get_data(path, cb != null), LogConfig.class); + return ClusterUtils.maybeDeserialize(stateStorage.get_data(path, cb != null), + LogConfig.class); } @Override - public void workerHeartbeat(String stormId, String node, Long port, ClusterWorkerHeartbeat info) { + public void workerHeartbeat(String stormId, String node, Long port, + ClusterWorkerHeartbeat info) { if (info != null) { String path = ClusterUtils.workerbeatPath(stormId, node, port); stateStorage.set_worker_hb(path, Utils.serialize(info), defaultAcls); @@ -533,14 +565,18 @@ public void supervisorHeartbeat(String supervisorId, SupervisorInfo info) { } /** - * Check whether a topology is in throttle-on status or not: if the backpresure/storm-id dir is not empty, this topology has - * throttle-on, otherwise throttle-off. But if the backpresure/storm-id dir is not empty and has not been updated for more than - * timeoutMs, we treat it as throttle-off. This will prevent the spouts from getting stuck indefinitely if something wrong happens. + * Check whether a topology is in throttle-on status or not: if the backpresure/storm-id dir is + * not empty, this topology has + * throttle-on, otherwise throttle-off. But if the backpresure/storm-id dir is not empty and has + * not been updated for more than + * timeoutMs, we treat it as throttle-off. This will prevent the spouts from getting stuck + * indefinitely if something wrong happens. * * @param stormId The topology Id * @param timeoutMs How long until the backpressure znode is invalid. * @param callback The callback function - * @return True is backpresure/storm-id dir is not empty and at least one of the backpressure znodes has not timed out; false otherwise. + * @return True is backpresure/storm-id dir is not empty and at least one of the backpressure + * znodes has not timed out; false otherwise. */ @Override public boolean topologyBackpressure(String stormId, long timeoutMs, Runnable callback) { @@ -552,7 +588,8 @@ public boolean topologyBackpressure(String stormId, long timeoutMs, Runnable cal if (stateStorage.node_exists(path, false)) { List children = stateStorage.get_children(path, callback != null); mostRecentTimestamp = children.stream() - .map(childPath -> stateStorage.get_data(ClusterUtils.backpressurePath(stormId, childPath), false)) + .map(childPath -> stateStorage.get_data(ClusterUtils + .backpressurePath(stormId, childPath), false)) .filter(data -> data != null) .mapToLong(data -> ByteBuffer.wrap(data).getLong()) .max() @@ -566,7 +603,8 @@ public boolean topologyBackpressure(String stormId, long timeoutMs, Runnable cal @Override public void setupBackpressure(String stormId, Map topoConf) { stateStorage.mkdirs(ClusterUtils.BACKPRESSURE_SUBTREE, defaultAcls); - stateStorage.mkdirs(ClusterUtils.backpressureStormRoot(stormId), ClusterUtils.mkTopoReadWriteAcls(topoConf)); + stateStorage.mkdirs(ClusterUtils.backpressureStormRoot(stormId), ClusterUtils + .mkTopoReadWriteAcls(topoConf)); } @Override @@ -596,7 +634,8 @@ public void removeWorkerBackpressure(String stormId, String node, Long port) { public void activateStorm(String stormId, StormBase stormBase, Map topoConf) { String path = ClusterUtils.stormPath(stormId); stateStorage.mkdirs(ClusterUtils.STORMS_SUBTREE, defaultAcls); - stateStorage.set_data(path, Utils.serialize(stormBase), ClusterUtils.mkTopoReadOnlyAcls(topoConf)); + stateStorage.set_data(path, Utils.serialize(stormBase), ClusterUtils + .mkTopoReadOnlyAcls(topoConf)); this.assignmentsBackend.keepStormId(stormBase.get_name(), stormId); } @@ -614,7 +653,8 @@ public void updateStorm(String stormId, StormBase newElems) { for (Map.Entry entry : componentExecutors.entrySet()) { newComponentExecutors.put(entry.getKey(), entry.getValue()); } - for (Map.Entry entry : stormBase.get_component_executors().entrySet()) { + for (Map.Entry entry : stormBase.get_component_executors() + .entrySet()) { if (!componentExecutors.containsKey(entry.getKey())) { newComponentExecutors.put(entry.getKey(), entry.getValue()); } @@ -628,7 +668,8 @@ public void updateStorm(String stormId, StormBase newElems) { Map oldComponentDebug = stormBase.get_component_debug(); Map newComponentDebug = newElems.get_component_debug(); - /// oldComponentDebug.keySet()/ newComponentDebug.keySet() maybe be APersistentSet, which don't support addAll + // / oldComponentDebug.keySet()/ newComponentDebug.keySet() maybe be APersistentSet, which + // don't support addAll Set debugOptionsKeys = new HashSet<>(); debugOptionsKeys.addAll(oldComponentDebug.keySet()); debugOptionsKeys.addAll(newComponentDebug.keySet()); @@ -656,7 +697,8 @@ public void updateStorm(String stormId, StormBase newElems) { newElems.set_name(stormBase.get_name()); } - if (StringUtils.isBlank(newElems.get_topology_version()) && stormBase.is_set_topology_version()) { + if (StringUtils.isBlank(newElems.get_topology_version()) && stormBase + .is_set_topology_version()) { newElems.set_topology_version(stormBase.get_topology_version()); } @@ -682,7 +724,8 @@ public void updateStorm(String stormId, StormBase newElems) { if (newElems.get_status() == null) { newElems.set_status(stormBase.get_status()); } - stateStorage.set_data(ClusterUtils.stormPath(stormId), Utils.serialize(newElems), defaultAcls); + stateStorage.set_data(ClusterUtils.stormPath(stormId), Utils.serialize(newElems), + defaultAcls); } @Override @@ -694,16 +737,19 @@ public void removeStormBase(String stormId) { public void setAssignment(String stormId, Assignment info, Map topoConf) { byte[] serAssignment = Utils.serialize(info); stateStorage.mkdirs(ClusterUtils.ASSIGNMENTS_SUBTREE, defaultAcls); - stateStorage.set_data(ClusterUtils.assignmentPath(stormId), Utils.serialize(info), ClusterUtils.mkTopoReadOnlyAcls(topoConf)); + stateStorage.set_data(ClusterUtils.assignmentPath(stormId), Utils.serialize(info), + ClusterUtils.mkTopoReadOnlyAcls(topoConf)); this.assignmentsBackend.keepOrUpdateAssignment(stormId, info); } @Override public void setupBlob(String key, NimbusInfo nimbusInfo, Integer versionInfo) { - String path = ClusterUtils.blobstorePath(key) + ClusterUtils.ZK_SEPERATOR + nimbusInfo.toHostPortString() + "-" + versionInfo; + String path = ClusterUtils.blobstorePath(key) + ClusterUtils.ZK_SEPERATOR + nimbusInfo + .toHostPortString() + "-" + versionInfo; LOG.info("set-path: {}", path); stateStorage.mkdirs(ClusterUtils.blobstorePath(key), defaultAcls); - stateStorage.delete_node_blobstore(ClusterUtils.blobstorePath(key), nimbusInfo.toHostPortString()); + stateStorage.delete_node_blobstore(ClusterUtils.blobstorePath(key), nimbusInfo + .toHostPortString()); stateStorage.set_ephemeral_node(path, null, defaultAcls); } @@ -747,18 +793,22 @@ public void removeKeyVersion(String blobKey) { @Override public void setupErrors(String stormId, Map topoConf) { stateStorage.mkdirs(ClusterUtils.ERRORS_SUBTREE, defaultAcls); - stateStorage.mkdirs(ClusterUtils.errorStormRoot(stormId), ClusterUtils.mkTopoReadWriteAcls(topoConf)); + stateStorage.mkdirs(ClusterUtils.errorStormRoot(stormId), ClusterUtils + .mkTopoReadWriteAcls(topoConf)); } @Override - public void reportError(String stormId, String componentId, String node, Long port, Throwable error) { + public void reportError(String stormId, String componentId, String node, Long port, + Throwable error) { String path = ClusterUtils.errorPath(stormId, componentId); - ErrorInfo errorInfo = new ErrorInfo(ClusterUtils.stringifyError(error), Time.currentTimeSecs()); + ErrorInfo errorInfo = new ErrorInfo(ClusterUtils.stringifyError(error), Time + .currentTimeSecs()); errorInfo.set_host(node); errorInfo.set_port(port.intValue()); byte[] serData = Utils.serialize(errorInfo); stateStorage.mkdirs(path, defaultAcls); - stateStorage.create_sequential(path + ClusterUtils.ZK_SEPERATOR + "e", serData, defaultAcls); + stateStorage.create_sequential(path + ClusterUtils.ZK_SEPERATOR + "e", serData, + defaultAcls); String lastErrorPath = ClusterUtils.lastErrorPath(stormId, componentId); stateStorage.set_data(lastErrorPath, serData, defaultAcls); List childrens = stateStorage.get_children(path, false); @@ -766,7 +816,8 @@ public void reportError(String stormId, String componentId, String node, Long po Collections.sort(childrens, new Comparator() { @Override public int compare(String arg0, String arg1) { - return Long.compare(Long.parseLong(arg0.substring(1)), Long.parseLong(arg1.substring(1))); + return Long.compare(Long.parseLong(arg0.substring(1)), Long.parseLong(arg1 + .substring(1))); } }); @@ -793,7 +844,8 @@ public List errors(String stormId, String componentId) { List childrens = stateStorage.get_children(path, false); for (String child : childrens) { String childPath = path + ClusterUtils.ZK_SEPERATOR + child; - ErrorInfo errorInfo = ClusterUtils.maybeDeserialize(stateStorage.get_data(childPath, false), ErrorInfo.class); + ErrorInfo errorInfo = ClusterUtils.maybeDeserialize(stateStorage.get_data(childPath, + false), ErrorInfo.class); if (errorInfo != null) { errorInfos.add(errorInfo); } @@ -813,7 +865,8 @@ public int compare(ErrorInfo arg0, ErrorInfo arg1) { public ErrorInfo lastError(String stormId, String componentId) { String path = ClusterUtils.lastErrorPath(stormId, componentId); if (stateStorage.node_exists(path, false)) { - ErrorInfo errorInfo = ClusterUtils.maybeDeserialize(stateStorage.get_data(path, false), ErrorInfo.class); + ErrorInfo errorInfo = ClusterUtils.maybeDeserialize(stateStorage.get_data(path, false), + ErrorInfo.class); return errorInfo; } @@ -833,7 +886,8 @@ public Credentials credentials(String stormId, Runnable callback) { credentialsCallback.put(stormId, callback); } String path = ClusterUtils.credentialsPath(stormId); - return ClusterUtils.maybeDeserialize(stateStorage.get_data(path, callback != null), Credentials.class); + return ClusterUtils.maybeDeserialize(stateStorage.get_data(path, callback != null), + Credentials.class); } @@ -847,12 +901,13 @@ public void disconnect() { } @Override - public PrivateWorkerKey getPrivateWorkerKey(WorkerTokenServiceType type, String topologyId, long keyVersion) { + public PrivateWorkerKey getPrivateWorkerKey(WorkerTokenServiceType type, String topologyId, + long keyVersion) { String path = ClusterUtils.secretKeysPath(type, topologyId, keyVersion); byte[] data = stateStorage.get_data(path, false); if (data == null) { LOG.debug("Could not find entry at {} will sync to see if that fixes it", path); - //We didn't find it, but there are races, so we want to check again after a sync + // We didn't find it, but there are races, so we want to check again after a sync stateStorage.sync_path(path); data = stateStorage.get_data(path, false); } @@ -860,12 +915,14 @@ public PrivateWorkerKey getPrivateWorkerKey(WorkerTokenServiceType type, String } @Override - public void addPrivateWorkerKey(WorkerTokenServiceType type, String topologyId, long keyVersion, PrivateWorkerKey key) { + public void addPrivateWorkerKey(WorkerTokenServiceType type, String topologyId, long keyVersion, + PrivateWorkerKey key) { assert context.getDaemonType() == DaemonType.NIMBUS; stateStorage.mkdirs(ClusterUtils.SECRET_KEYS_SUBTREE, defaultAcls); List secretAcls = context.getZkSecretAcls(type); String path = ClusterUtils.secretKeysPath(type, topologyId, keyVersion); - LOG.info("Storing private key for {} connecting to a {} at {} with ACL {}", topologyId, type, path, secretAcls); + LOG.info("Storing private key for {} connecting to a {} at {} with ACL {}", topologyId, + type, path, secretAcls); stateStorage.set_data(path, Utils.serialize(key), secretAcls); } @@ -877,7 +934,7 @@ public long getNextPrivateWorkerKeyVersion(WorkerTokenServiceType type, String t return versions.stream().mapToLong(Long::valueOf).max().orElse(0); } catch (RuntimeException e) { if (Utils.exceptionCauseIsInstanceOf(KeeperException.NoNodeException.class, e)) { - //If the node does not exist, then the version must be 0 + // If the node does not exist, then the version must be 0 return 0; } throw e; @@ -893,21 +950,24 @@ public void removeExpiredPrivateWorkerKeys(String topologyId) { String fullPath = basePath + ClusterUtils.ZK_SEPERATOR + version; try { PrivateWorkerKey key = - ClusterUtils.maybeDeserialize(stateStorage.get_data(fullPath, false), PrivateWorkerKey.class); + ClusterUtils.maybeDeserialize(stateStorage.get_data(fullPath, false), + PrivateWorkerKey.class); if (Time.currentTimeMillis() > key.get_expirationTimeMillis()) { LOG.info("Removing expired worker key {}", fullPath); stateStorage.delete_node(fullPath); } } catch (RuntimeException e) { - //This should never happen because only the primary nimbus is active, but just in case + // This should never happen because only the primary nimbus is active, but + // just in case // declare the race safe, even if we lose it. - if (!Utils.exceptionCauseIsInstanceOf(KeeperException.NoNodeException.class, e)) { + if (!Utils.exceptionCauseIsInstanceOf(KeeperException.NoNodeException.class, + e)) { throw e; } } } } catch (RuntimeException e) { - //No node for basePath is OK, nothing to remove + // No node for basePath is OK, nothing to remove if (!Utils.exceptionCauseIsInstanceOf(KeeperException.NoNodeException.class, e)) { throw e; } @@ -923,7 +983,8 @@ public void removeAllPrivateWorkerKeys(String topologyId) { LOG.info("Removing worker keys under {}", path); stateStorage.delete_node(path); } catch (RuntimeException e) { - //This should never happen because only the primary nimbus is active, but just in case + // This should never happen because only the primary nimbus is active, but just in + // case // declare the race safe, even if we lose it. if (!Utils.exceptionCauseIsInstanceOf(KeeperException.NoNodeException.class, e)) { throw e; @@ -940,7 +1001,7 @@ public Set idsOfTopologiesWithPrivateWorkerKeys() { try { ret.addAll(stateStorage.get_children(path, false)); } catch (RuntimeException e) { - //If the node does not exist it is fine/expected... + // If the node does not exist it is fine/expected... if (!Utils.exceptionCauseIsInstanceOf(KeeperException.NoNodeException.class, e)) { throw e; } diff --git a/storm-client/src/jvm/org/apache/storm/cluster/VersionedData.java b/storm-client/src/jvm/org/apache/storm/cluster/VersionedData.java index 4dd0f043b53..dc6d23a8bc9 100644 --- a/storm-client/src/jvm/org/apache/storm/cluster/VersionedData.java +++ b/storm-client/src/jvm/org/apache/storm/cluster/VersionedData.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/cluster/ZKStateStorage.java b/storm-client/src/jvm/org/apache/storm/cluster/ZKStateStorage.java index 64ecf76e29d..29a2e2f40c7 100644 --- a/storm-client/src/jvm/org/apache/storm/cluster/ZKStateStorage.java +++ b/storm-client/src/jvm/org/apache/storm/cluster/ZKStateStorage.java @@ -43,7 +43,8 @@ public class ZKStateStorage implements IStateStorage { private static Logger LOG = LoggerFactory.getLogger(ZKStateStorage.class); - private ConcurrentHashMap callbacks = new ConcurrentHashMap(); + private ConcurrentHashMap callbacks = + new ConcurrentHashMap(); private CuratorFramework zkWriter; private CuratorFramework zkReader; private AtomicBoolean active; @@ -52,7 +53,8 @@ public class ZKStateStorage implements IStateStorage { private Map authConf; private Map conf; - public ZKStateStorage(Map conf, Map authConf, ClusterStateContext context) throws Exception { + public ZKStateStorage(Map conf, Map authConf, + ClusterStateContext context) throws Exception { this.conf = conf; this.authConf = authConf; if (context.getDaemonType().equals(DaemonType.NIMBUS)) { @@ -77,14 +79,18 @@ public ZKStateStorage(Map conf, Map authConf, Cl @SuppressWarnings("unchecked") private CuratorFramework mkZk(DaemonType type) { - return ClientZookeeper.mkClient(conf, (List) conf.get(Config.STORM_ZOOKEEPER_SERVERS), + return ClientZookeeper.mkClient(conf, (List) conf + .get(Config.STORM_ZOOKEEPER_SERVERS), conf.get(Config.STORM_ZOOKEEPER_PORT), "", new DefaultWatcherCallBack(), authConf, type); } @SuppressWarnings("unchecked") - private CuratorFramework mkZk(WatcherCallBack watcher, DaemonType type) throws NumberFormatException { - return ClientZookeeper.mkClient(conf, (List) conf.get(Config.STORM_ZOOKEEPER_SERVERS), - conf.get(Config.STORM_ZOOKEEPER_PORT), String.valueOf(conf.get(Config.STORM_ZOOKEEPER_ROOT)), + private CuratorFramework mkZk(WatcherCallBack watcher, + DaemonType type) throws NumberFormatException { + return ClientZookeeper.mkClient(conf, (List) conf + .get(Config.STORM_ZOOKEEPER_SERVERS), + conf.get(Config.STORM_ZOOKEEPER_PORT), String.valueOf(conf + .get(Config.STORM_ZOOKEEPER_ROOT)), watcher, authConf, type); } @@ -107,7 +113,8 @@ public void unregister(String id) { @Override public String create_sequential(String path, byte[] data, List acls) { - return ClientZookeeper.createNode(zkWriter, path, data, CreateMode.EPHEMERAL_SEQUENTIAL, acls); + return ClientZookeeper.createNode(zkWriter, path, data, CreateMode.EPHEMERAL_SEQUENTIAL, + acls); } @Override @@ -173,7 +180,8 @@ public void set_data(String path, byte[] data, List acls) { try { ClientZookeeper.createNode(zkWriter, path, data, CreateMode.PERSISTENT, acls); } catch (RuntimeException e) { - if (Utils.exceptionCauseIsInstanceOf(KeeperException.NodeExistsException.class, e)) { + if (Utils.exceptionCauseIsInstanceOf(KeeperException.NodeExistsException.class, + e)) { ClientZookeeper.setData(zkWriter, path, data); } else { throw e; @@ -219,7 +227,8 @@ public void delete_worker_hb(String path) { @Override public void add_listener(final ConnectionStateListener listener) { ClientZookeeper.addListener(zkReader, - (curatorFramework, connectionState) -> listener.stateChanged(curatorFramework, connectionState)); + (curatorFramework, connectionState) -> listener.stateChanged(curatorFramework, + connectionState)); } @Override @@ -229,10 +238,12 @@ public void sync_path(String path) { private class ZkWatcherCallBack implements WatcherCallBack { @Override - public void execute(Watcher.Event.KeeperState state, Watcher.Event.EventType type, String path) { + public void execute(Watcher.Event.KeeperState state, Watcher.Event.EventType type, + String path) { if (active.get()) { if (!(state.equals(Watcher.Event.KeeperState.SyncConnected))) { - LOG.debug("Received event {} : {}: {} with disconnected Zookeeper.", state, type, path); + LOG.debug("Received event {} : {}: {} with disconnected Zookeeper.", state, + type, path); } else { LOG.debug("Received event {} : {} : {}", state, type, path); } diff --git a/storm-client/src/jvm/org/apache/storm/cluster/ZKStateStorageFactory.java b/storm-client/src/jvm/org/apache/storm/cluster/ZKStateStorageFactory.java index ff220194965..df5a7907a82 100644 --- a/storm-client/src/jvm/org/apache/storm/cluster/ZKStateStorageFactory.java +++ b/storm-client/src/jvm/org/apache/storm/cluster/ZKStateStorageFactory.java @@ -25,7 +25,8 @@ public class ZKStateStorageFactory implements StateStorageFactory { @Override - public IStateStorage mkStore(Map config, Map authConf, ClusterStateContext context) { + public IStateStorage mkStore(Map config, Map authConf, + ClusterStateContext context) { try { return new ZKStateStorage(config, authConf, context); } catch (Exception e) { diff --git a/storm-client/src/jvm/org/apache/storm/container/cgroup/CgroupCenter.java b/storm-client/src/jvm/org/apache/storm/container/cgroup/CgroupCenter.java index 67c0482dd5b..92d6c659266 100644 --- a/storm-client/src/jvm/org/apache/storm/container/cgroup/CgroupCenter.java +++ b/storm-client/src/jvm/org/apache/storm/container/cgroup/CgroupCenter.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -59,7 +65,7 @@ public List getHierarchies() { String name = strSplit[0]; String type = strSplit[3]; String dir = strSplit[1]; - //Some mount options (i.e. rw and relatime) in type are not cgroups related + // Some mount options (i.e. rw and relatime) in type are not cgroups related Hierarchy h = new Hierarchy(name, CgroupUtils.getSubSystemsFromString(type), dir); hierarchies.put(type, h); } @@ -151,7 +157,8 @@ public void mount(Hierarchy hierarchy) throws IOException { for (SubSystemType type : subSystems) { Hierarchy hierarchyWithSubSystem = this.getHierarchyWithSubSystem(type); if (hierarchyWithSubSystem != null) { - LOG.error("subSystem: {} is already mounted on hierarchy: {}", type.name(), hierarchyWithSubSystem); + LOG.error("subSystem: {} is already mounted on hierarchy: {}", type.name(), + hierarchyWithSubSystem); subSystems.remove(type); } } diff --git a/storm-client/src/jvm/org/apache/storm/container/cgroup/CgroupCommon.java b/storm-client/src/jvm/org/apache/storm/container/cgroup/CgroupCommon.java index 07f1abe315d..24a282b3241 100755 --- a/storm-client/src/jvm/org/apache/storm/container/cgroup/CgroupCommon.java +++ b/storm-client/src/jvm/org/apache/storm/container/cgroup/CgroupCommon.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -70,12 +76,14 @@ public Set getTasks() throws IOException { @Override public void addProcs(int pid) throws IOException { - CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, CGROUP_PROCS), String.valueOf(pid)); + CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, CGROUP_PROCS), String + .valueOf(pid)); } @Override public Set getPids() throws IOException { - List stringPids = CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, CGROUP_PROCS)); + List stringPids = CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, + CGROUP_PROCS)); Set pids = new HashSet<>(); for (String task : stringPids) { pids.add(Long.valueOf(task)); @@ -85,19 +93,22 @@ public Set getPids() throws IOException { @Override public boolean getNotifyOnRelease() throws IOException { - return CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, NOTIFY_ON_RELEASE)).get(0).equals("1") ? true : false; + return CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, NOTIFY_ON_RELEASE)).get(0) + .equals("1") ? true : false; } @Override public void setNotifyOnRelease(boolean flag) throws IOException { - CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, NOTIFY_ON_RELEASE), flag ? "1" : "0"); + CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, NOTIFY_ON_RELEASE), flag + ? "1" : "0"); } @Override public String getReleaseAgent() throws IOException { if (!this.isRoot) { - LOG.warn("Cannot get {} in {} since its not the root group", RELEASE_AGENT, this.isRoot); + LOG.warn("Cannot get {} in {} since its not the root group", RELEASE_AGENT, + this.isRoot); return null; } return CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, RELEASE_AGENT)).get(0); @@ -106,7 +117,8 @@ public String getReleaseAgent() throws IOException { @Override public void setReleaseAgent(String command) throws IOException { if (!this.isRoot) { - LOG.warn("Cannot set {} in {} since its not the root group", RELEASE_AGENT, this.isRoot); + LOG.warn("Cannot set {} in {} since its not the root group", RELEASE_AGENT, + this.isRoot); return; } CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, RELEASE_AGENT), command); @@ -114,7 +126,8 @@ public void setReleaseAgent(String command) throws IOException { @Override public boolean getCgroupCloneChildren() throws IOException { - return CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, CGROUP_CLONE_CHILDREN)).get(0).equals("1") ? true : false; + return CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, CGROUP_CLONE_CHILDREN)) + .get(0).equals("1") ? true : false; } @Override @@ -122,11 +135,13 @@ public void setCgroupCloneChildren(boolean flag) throws IOException { if (!getCores().keySet().contains(SubSystemType.cpuset)) { return; } - CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, CGROUP_CLONE_CHILDREN), flag ? "1" : "0"); + CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, CGROUP_CLONE_CHILDREN), flag + ? "1" : "0"); } @Override - public void setEventControl(String eventFd, String controlFd, String... args) throws IOException { + public void setEventControl(String eventFd, String controlFd, + String... args) throws IOException { StringBuilder sb = new StringBuilder(); sb.append(eventFd); sb.append(' '); @@ -135,7 +150,8 @@ public void setEventControl(String eventFd, String controlFd, String... args) th sb.append(' '); sb.append(arg); } - CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, CGROUP_EVENT_CONTROL), sb.toString()); + CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, CGROUP_EVENT_CONTROL), sb + .toString()); } public Hierarchy getHierarchy() { diff --git a/storm-client/src/jvm/org/apache/storm/container/cgroup/CgroupCommonOperation.java b/storm-client/src/jvm/org/apache/storm/container/cgroup/CgroupCommonOperation.java index 82aa41a6e22..4616ec43c3d 100755 --- a/storm-client/src/jvm/org/apache/storm/container/cgroup/CgroupCommonOperation.java +++ b/storm-client/src/jvm/org/apache/storm/container/cgroup/CgroupCommonOperation.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -18,7 +24,7 @@ public interface CgroupCommonOperation { /** - * add task to cgroup. + * Add task to cgroup. * * @param taskid task id of task to add */ @@ -30,39 +36,39 @@ public interface CgroupCommonOperation { Set getTasks() throws IOException; /** - * add a process to cgroup. + * Add a process to cgroup. * * @param pid the PID of the process to add */ void addProcs(int pid) throws IOException; /** - * get the PIDs of processes running in cgroup. + * Get the PIDs of processes running in cgroup. */ Set getPids() throws IOException; /** - * to get the notify_on_release config. + * To get the notify_on_release config. */ boolean getNotifyOnRelease() throws IOException; /** - * to set notify_on_release config in cgroup. + * To set notify_on_release config in cgroup. */ void setNotifyOnRelease(boolean flag) throws IOException; /** - * get the command for the relase agent to execute. + * Get the command for the relase agent to execute. */ String getReleaseAgent() throws IOException; /** - * set a command for the release agent to execute. + * Set a command for the release agent to execute. */ void setReleaseAgent(String command) throws IOException; /** - * get the cgroup.clone_children config. + * Get the cgroup.clone_children config. */ boolean getCgroupCloneChildren() throws IOException; @@ -72,7 +78,7 @@ public interface CgroupCommonOperation { void setCgroupCloneChildren(boolean flag) throws IOException; /** - * set event control config. + * Set event control config. */ void setEventControl(String eventFd, String controlFd, String... args) throws IOException; } diff --git a/storm-client/src/jvm/org/apache/storm/container/cgroup/CgroupCoreFactory.java b/storm-client/src/jvm/org/apache/storm/container/cgroup/CgroupCoreFactory.java index bda8b0b8153..17eb9d2827f 100755 --- a/storm-client/src/jvm/org/apache/storm/container/cgroup/CgroupCoreFactory.java +++ b/storm-client/src/jvm/org/apache/storm/container/cgroup/CgroupCoreFactory.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/container/cgroup/CgroupOperation.java b/storm-client/src/jvm/org/apache/storm/container/cgroup/CgroupOperation.java index 78970572f54..80e9d1ff1ff 100755 --- a/storm-client/src/jvm/org/apache/storm/container/cgroup/CgroupOperation.java +++ b/storm-client/src/jvm/org/apache/storm/container/cgroup/CgroupOperation.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -17,7 +23,8 @@ import java.util.Set; /** - * An interface to implement the basic functions to manage cgroups such as mount and mounting a hiearchy and creating cgroups. Also + * An interface to implement the basic functions to manage cgroups such as mount and mounting a + * hiearchy and creating cgroups. Also * contains functions to access basic information of cgroups. */ public interface CgroupOperation { @@ -28,7 +35,7 @@ public interface CgroupOperation { List getHierarchies(); /** - * get a list of available subsystems. + * Get a list of available subsystems. */ Set getSubSystems(); @@ -38,37 +45,37 @@ public interface CgroupOperation { boolean isSubSystemEnabled(SubSystemType subsystem); /** - * get the first hierarchy that has a certain subsystem isMounted. + * Get the first hierarchy that has a certain subsystem isMounted. */ Hierarchy getHierarchyWithSubSystem(SubSystemType subsystem); /** - * get the first hierarchy that has a certain list of subsystems isMounted. + * Get the first hierarchy that has a certain list of subsystems isMounted. */ Hierarchy getHierarchyWithSubSystems(List subSystems); /** - * check if a hiearchy is mounted. + * Check if a hiearchy is mounted. */ boolean isMounted(Hierarchy hierarchy); /** - * mount a hierarchy. + * Mount a hierarchy. */ void mount(Hierarchy hierarchy) throws IOException; /** - * umount a heirarchy. + * Umount a heirarchy. */ void umount(Hierarchy hierarchy) throws IOException; /** - * create a cgroup. + * Create a cgroup. */ void createCgroup(CgroupCommon cgroup) throws SecurityException; /** - * delete a cgroup. + * Delete a cgroup. */ void deleteCgroup(CgroupCommon cgroup) throws IOException; } diff --git a/storm-client/src/jvm/org/apache/storm/container/cgroup/CgroupUtils.java b/storm-client/src/jvm/org/apache/storm/container/cgroup/CgroupUtils.java index d55361e2191..02eab34419a 100644 --- a/storm-client/src/jvm/org/apache/storm/container/cgroup/CgroupUtils.java +++ b/storm-client/src/jvm/org/apache/storm/container/cgroup/CgroupUtils.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -54,7 +60,7 @@ public static Set getSubSystemsFromString(String str) { Set result = new HashSet(); String[] subSystems = str.split(","); for (String subSystem : subSystems) { - //return null to mount options in string that is not part of cgroups + // return null to mount options in string that is not part of cgroups SubSystemType type = SubSystemType.getSubSystem(subSystem); if (type != null) { result.add(type); @@ -85,7 +91,8 @@ public static List readFileByLine(String filePath) throws IOException { return Files.readLines(new File(filePath), Charset.defaultCharset()); } - public static void writeFileByLine(String filePath, List linesToWrite) throws IOException { + public static void writeFileByLine(String filePath, + List linesToWrite) throws IOException { LOG.debug("For CGroups - writing {} to {} ", linesToWrite, filePath); File file = new File(filePath); if (!file.exists()) { diff --git a/storm-client/src/jvm/org/apache/storm/container/cgroup/Device.java b/storm-client/src/jvm/org/apache/storm/container/cgroup/Device.java index a949e87bc0e..c8751dd1824 100755 --- a/storm-client/src/jvm/org/apache/storm/container/cgroup/Device.java +++ b/storm-client/src/jvm/org/apache/storm/container/cgroup/Device.java @@ -1,19 +1,25 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.container.cgroup; /** - * a class that represents a device in linux. + * A class that represents a device in linux. */ public class Device { diff --git a/storm-client/src/jvm/org/apache/storm/container/cgroup/Hierarchy.java b/storm-client/src/jvm/org/apache/storm/container/cgroup/Hierarchy.java index 7ef52967b8d..a8f7d16520f 100755 --- a/storm-client/src/jvm/org/apache/storm/container/cgroup/Hierarchy.java +++ b/storm-client/src/jvm/org/apache/storm/container/cgroup/Hierarchy.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -38,14 +44,14 @@ public Hierarchy(String name, Set subSystems, String dir) { } /** - * get subsystems. + * Get subsystems. */ public Set getSubSystems() { return subSystems; } /** - * get all subsystems in hierarchy as a comma delimited list. + * Get all subsystems in hierarchy as a comma delimited list. */ public String getType() { return type; diff --git a/storm-client/src/jvm/org/apache/storm/container/cgroup/SubSystem.java b/storm-client/src/jvm/org/apache/storm/container/cgroup/SubSystem.java index f5aa32fa999..d63ac5c0866 100755 --- a/storm-client/src/jvm/org/apache/storm/container/cgroup/SubSystem.java +++ b/storm-client/src/jvm/org/apache/storm/container/cgroup/SubSystem.java @@ -1,19 +1,25 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.container.cgroup; /** - * a class that implements operations that can be performed on a cgroup subsystem. + * A class that implements operations that can be performed on a cgroup subsystem. */ public class SubSystem { diff --git a/storm-client/src/jvm/org/apache/storm/container/cgroup/SubSystemType.java b/storm-client/src/jvm/org/apache/storm/container/cgroup/SubSystemType.java index f2ea9dd90ce..b44f48170cd 100755 --- a/storm-client/src/jvm/org/apache/storm/container/cgroup/SubSystemType.java +++ b/storm-client/src/jvm/org/apache/storm/container/cgroup/SubSystemType.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -20,7 +26,6 @@ public enum SubSystemType { // net_cls,ns is not supported in ubuntu blkio, cpu, cpuacct, cpuset, devices, freezer, memory, perf_event, net_cls, net_prio; - public static SubSystemType getSubSystem(String str) { try { return SubSystemType.valueOf(str); diff --git a/storm-client/src/jvm/org/apache/storm/container/cgroup/SystemOperation.java b/storm-client/src/jvm/org/apache/storm/container/cgroup/SystemOperation.java index a03fb52ff7d..e026ac0fb13 100644 --- a/storm-client/src/jvm/org/apache/storm/container/cgroup/SystemOperation.java +++ b/storm-client/src/jvm/org/apache/storm/container/cgroup/SystemOperation.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -30,7 +36,8 @@ public static boolean isRoot() throws IOException { return Integer.valueOf(result.substring(0, result.length())).intValue() == 0 ? true : false; } - public static void mount(String name, String target, String type, String options) throws IOException { + public static void mount(String name, String target, String type, + String options) throws IOException { StringBuilder sb = new StringBuilder(); sb.append("mount -t ") .append(type) diff --git a/storm-client/src/jvm/org/apache/storm/container/cgroup/core/BlkioCore.java b/storm-client/src/jvm/org/apache/storm/container/cgroup/core/BlkioCore.java index 7bcf5708b6a..0ef4ee2e073 100755 --- a/storm-client/src/jvm/org/apache/storm/container/cgroup/core/BlkioCore.java +++ b/storm-client/src/jvm/org/apache/storm/container/cgroup/core/BlkioCore.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -29,7 +35,8 @@ public class BlkioCore implements CgroupCore { public static final String BLKIO_THROTTLE_READ_BPS_DEVICE = "/blkio.throttle.read_bps_device"; public static final String BLKIO_THROTTLE_WRITE_BPS_DEVICE = "/blkio.throttle.write_bps_device"; public static final String BLKIO_THROTTLE_READ_IOPS_DEVICE = "/blkio.throttle.read_iops_device"; - public static final String BLKIO_THROTTLE_WRITE_IOPS_DEVICE = "/blkio.throttle.write_iops_device"; + public static final String BLKIO_THROTTLE_WRITE_IOPS_DEVICE = + "/blkio.throttle.write_iops_device"; public static final String BLKIO_THROTTLE_IO_SERVICED = "/blkio.throttle.io_serviced"; public static final String BLKIO_THROTTLE_IO_SERVICE_BYTES = "/blkio.throttle.io_service_bytes"; @@ -55,20 +62,24 @@ public SubSystemType getType() { } public int getBlkioWeight() throws IOException { - return Integer.valueOf(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, BLKIO_WEIGHT)).get(0)).intValue(); + return Integer.valueOf(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, + BLKIO_WEIGHT)).get(0)).intValue(); } /* weight: 100-1000 */ public void setBlkioWeight(int weight) throws IOException { - CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, BLKIO_WEIGHT), String.valueOf(weight)); + CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, BLKIO_WEIGHT), String + .valueOf(weight)); } public void setBlkioWeightDevice(Device device, int weight) throws IOException { - CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, BLKIO_WEIGHT_DEVICE), makeContext(device, weight)); + CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, BLKIO_WEIGHT_DEVICE), + makeContext(device, weight)); } public Map getBlkioWeightDevice() throws IOException { - List strings = CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, BLKIO_WEIGHT_DEVICE)); + List strings = CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, + BLKIO_WEIGHT_DEVICE)); Map result = new HashMap(); for (String string : strings) { String[] strArgs = string.split(" "); @@ -80,7 +91,8 @@ public Map getBlkioWeightDevice() throws IOException { } public void setReadBps(Device device, long bps) throws IOException { - CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, BLKIO_THROTTLE_READ_BPS_DEVICE), makeContext(device, bps)); + CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, BLKIO_THROTTLE_READ_BPS_DEVICE), + makeContext(device, bps)); } public Map getReadBps() throws IOException { @@ -88,7 +100,8 @@ public Map getReadBps() throws IOException { } public void setWriteBps(Device device, long bps) throws IOException { - CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, BLKIO_THROTTLE_WRITE_BPS_DEVICE), makeContext(device, bps)); + CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, BLKIO_THROTTLE_WRITE_BPS_DEVICE), + makeContext(device, bps)); } public Map getWriteBps() throws IOException { @@ -97,7 +110,8 @@ public Map getWriteBps() throws IOException { @SuppressWarnings("checkstyle:AbbreviationAsWordInName") public void setReadIOps(Device device, long iops) throws IOException { - CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, BLKIO_THROTTLE_READ_IOPS_DEVICE), makeContext(device, iops)); + CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, BLKIO_THROTTLE_READ_IOPS_DEVICE), + makeContext(device, iops)); } @SuppressWarnings("checkstyle:AbbreviationAsWordInName") @@ -107,7 +121,8 @@ public Map getReadIOps() throws IOException { @SuppressWarnings("checkstyle:AbbreviationAsWordInName") public void setWriteIOps(Device device, long iops) throws IOException { - CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, BLKIO_THROTTLE_WRITE_IOPS_DEVICE), makeContext(device, iops)); + CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, BLKIO_THROTTLE_WRITE_IOPS_DEVICE), + makeContext(device, iops)); } @SuppressWarnings("checkstyle:AbbreviationAsWordInName") @@ -117,12 +132,14 @@ public Map getWriteIOps() throws IOException { @SuppressWarnings("checkstyle:AbbreviationAsWordInName") public Map> getThrottleIOServiced() throws IOException { - return this.analyseRecord(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, BLKIO_THROTTLE_IO_SERVICED))); + return this.analyseRecord(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, + BLKIO_THROTTLE_IO_SERVICED))); } @SuppressWarnings("checkstyle:AbbreviationAsWordInName") public Map> getThrottleIOServiceByte() throws IOException { - return this.analyseRecord(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, BLKIO_THROTTLE_IO_SERVICE_BYTES))); + return this.analyseRecord(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, + BLKIO_THROTTLE_IO_SERVICE_BYTES))); } public Map getBlkioTime() throws IOException { @@ -135,32 +152,38 @@ public Map getBlkioSectors() throws IOException { @SuppressWarnings("checkstyle:AbbreviationAsWordInName") public Map> getIOServiced() throws IOException { - return this.analyseRecord(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, BLKIO_IO_SERVICED))); + return this.analyseRecord(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, + BLKIO_IO_SERVICED))); } @SuppressWarnings("checkstyle:AbbreviationAsWordInName") public Map> getIOServiceBytes() throws IOException { - return this.analyseRecord(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, BLKIO_IO_SERVICE_BYTES))); + return this.analyseRecord(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, + BLKIO_IO_SERVICE_BYTES))); } @SuppressWarnings("checkstyle:AbbreviationAsWordInName") public Map> getIOServiceTime() throws IOException { - return this.analyseRecord(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, BLKIO_IO_SERVICE_TIME))); + return this.analyseRecord(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, + BLKIO_IO_SERVICE_TIME))); } @SuppressWarnings("checkstyle:AbbreviationAsWordInName") public Map> getIOWaitTime() throws IOException { - return this.analyseRecord(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, BLKIO_IO_WAIT_TIME))); + return this.analyseRecord(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, + BLKIO_IO_WAIT_TIME))); } @SuppressWarnings("checkstyle:AbbreviationAsWordInName") public Map> getIOMerged() throws IOException { - return this.analyseRecord(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, BLKIO_IO_MERGED))); + return this.analyseRecord(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, + BLKIO_IO_MERGED))); } @SuppressWarnings("checkstyle:AbbreviationAsWordInName") public Map> getIOQueued() throws IOException { - return this.analyseRecord(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, BLKIO_IO_QUEUED))); + return this.analyseRecord(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, + BLKIO_IO_QUEUED))); } public void resetStats() throws IOException { diff --git a/storm-client/src/jvm/org/apache/storm/container/cgroup/core/CgroupCore.java b/storm-client/src/jvm/org/apache/storm/container/cgroup/core/CgroupCore.java index 229f0478800..0ca393a1451 100755 --- a/storm-client/src/jvm/org/apache/storm/container/cgroup/core/CgroupCore.java +++ b/storm-client/src/jvm/org/apache/storm/container/cgroup/core/CgroupCore.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/container/cgroup/core/CpuCore.java b/storm-client/src/jvm/org/apache/storm/container/cgroup/core/CpuCore.java index 0367c9de664..5bdc6a269d3 100755 --- a/storm-client/src/jvm/org/apache/storm/container/cgroup/core/CpuCore.java +++ b/storm-client/src/jvm/org/apache/storm/container/cgroup/core/CpuCore.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -38,43 +44,53 @@ public SubSystemType getType() { } public int getCpuShares() throws IOException { - return Integer.parseInt(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, CPU_SHARES)).get(0)); + return Integer.parseInt(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, CPU_SHARES)) + .get(0)); } public void setCpuShares(int weight) throws IOException { - CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, CPU_SHARES), String.valueOf(weight)); + CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, CPU_SHARES), String + .valueOf(weight)); } public long getCpuRtRuntimeUs() throws IOException { - return Long.parseLong(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, CPU_RT_RUNTIME_US)).get(0)); + return Long.parseLong(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, + CPU_RT_RUNTIME_US)).get(0)); } public void setCpuRtRuntimeUs(long us) throws IOException { - CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, CPU_RT_RUNTIME_US), String.valueOf(us)); + CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, CPU_RT_RUNTIME_US), String + .valueOf(us)); } public Long getCpuRtPeriodUs() throws IOException { - return Long.parseLong(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, CPU_RT_PERIOD_US)).get(0)); + return Long.parseLong(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, + CPU_RT_PERIOD_US)).get(0)); } public void setCpuRtPeriodUs(long us) throws IOException { - CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, CPU_RT_PERIOD_US), String.valueOf(us)); + CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, CPU_RT_PERIOD_US), String + .valueOf(us)); } public Long getCpuCfsPeriodUs() throws IOException { - return Long.parseLong(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, CPU_CFS_PERIOD_US)).get(0)); + return Long.parseLong(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, + CPU_CFS_PERIOD_US)).get(0)); } public void setCpuCfsPeriodUs(long us) throws IOException { - CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, CPU_CFS_PERIOD_US), String.valueOf(us)); + CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, CPU_CFS_PERIOD_US), String + .valueOf(us)); } public Long getCpuCfsQuotaUs() throws IOException { - return Long.parseLong(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, CPU_CFS_QUOTA_US)).get(0)); + return Long.parseLong(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, + CPU_CFS_QUOTA_US)).get(0)); } public void setCpuCfsQuotaUs(long us) throws IOException { - CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, CPU_CFS_QUOTA_US), String.valueOf(us)); + CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, CPU_CFS_QUOTA_US), String + .valueOf(us)); } public Stat getCpuStat() throws IOException { diff --git a/storm-client/src/jvm/org/apache/storm/container/cgroup/core/CpuacctCore.java b/storm-client/src/jvm/org/apache/storm/container/cgroup/core/CpuacctCore.java index 09bd0e51080..ad616963578 100755 --- a/storm-client/src/jvm/org/apache/storm/container/cgroup/core/CpuacctCore.java +++ b/storm-client/src/jvm/org/apache/storm/container/cgroup/core/CpuacctCore.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -37,7 +43,8 @@ public SubSystemType getType() { } public Long getCpuUsage() throws IOException { - return Long.parseLong(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, CPUACCT_USAGE)).get(0)); + return Long.parseLong(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, + CPUACCT_USAGE)).get(0)); } public Map getCpuStat() throws IOException { @@ -49,7 +56,8 @@ public Map getCpuStat() throws IOException { } public Long[] getPerCpuUsage() throws IOException { - String str = CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, CPUACCT_USAGE_PERCPU)).get(0); + String str = CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, CPUACCT_USAGE_PERCPU)) + .get(0); String[] strArgs = str.split(" "); Long[] result = new Long[strArgs.length]; for (int i = 0; i < result.length; i++) { diff --git a/storm-client/src/jvm/org/apache/storm/container/cgroup/core/CpusetCore.java b/storm-client/src/jvm/org/apache/storm/container/cgroup/core/CpusetCore.java index 071e2a544f0..be3ef5bed12 100755 --- a/storm-client/src/jvm/org/apache/storm/container/cgroup/core/CpusetCore.java +++ b/storm-client/src/jvm/org/apache/storm/container/cgroup/core/CpusetCore.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -88,7 +94,8 @@ public SubSystemType getType() { } public int[] getCpus() throws IOException { - String output = CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, CPUSET_CPUS)).get(0); + String output = CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, CPUSET_CPUS)) + .get(0); return parseNums(output); } @@ -107,7 +114,8 @@ private void setConfigs(int[] nums, String config) throws IOException { } public int[] getMems() throws IOException { - String output = CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, CPUSET_MEMS)).get(0); + String output = CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, CPUSET_MEMS)) + .get(0); return parseNums(output); } @@ -116,88 +124,107 @@ public void setMems(int[] nums) throws IOException { } public boolean isMemMigrate() throws IOException { - int output = Integer.parseInt(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, CPUSET_MEMORY_MIGRATE)).get(0)); + int output = Integer.parseInt(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, + CPUSET_MEMORY_MIGRATE)).get(0)); return output > 0; } public void setMemMigrate(boolean flag) throws IOException { - CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, CPUSET_MEMORY_MIGRATE), String.valueOf(flag ? 1 : 0)); + CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, CPUSET_MEMORY_MIGRATE), String + .valueOf(flag ? 1 : 0)); } public boolean isCpuExclusive() throws IOException { - int output = Integer.parseInt(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, CPUSET_CPU_EXCLUSIVE)).get(0)); + int output = Integer.parseInt(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, + CPUSET_CPU_EXCLUSIVE)).get(0)); return output > 0; } public void setCpuExclusive(boolean flag) throws IOException { - CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, CPUSET_CPU_EXCLUSIVE), String.valueOf(flag ? 1 : 0)); + CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, CPUSET_CPU_EXCLUSIVE), String + .valueOf(flag ? 1 : 0)); } public boolean isMemExclusive() throws IOException { - int output = Integer.parseInt(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, CPUSET_MEM_EXCLUSIVE)).get(0)); + int output = Integer.parseInt(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, + CPUSET_MEM_EXCLUSIVE)).get(0)); return output > 0; } public void setMemExclusive(boolean flag) throws IOException { - CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, CPUSET_MEM_EXCLUSIVE), String.valueOf(flag ? 1 : 0)); + CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, CPUSET_MEM_EXCLUSIVE), String + .valueOf(flag ? 1 : 0)); } public boolean isMemHardwall() throws IOException { - int output = Integer.parseInt(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, CPUSET_MEM_HARDWALL)).get(0)); + int output = Integer.parseInt(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, + CPUSET_MEM_HARDWALL)).get(0)); return output > 0; } public void setMemHardwall(boolean flag) throws IOException { - CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, CPUSET_MEM_HARDWALL), String.valueOf(flag ? 1 : 0)); + CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, CPUSET_MEM_HARDWALL), String + .valueOf(flag ? 1 : 0)); } public int getMemPressure() throws IOException { - String output = CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, CPUSET_MEMORY_PRESSURE)).get(0); + String output = CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, + CPUSET_MEMORY_PRESSURE)).get(0); return Integer.parseInt(output); } public boolean isMemPressureEnabled() throws IOException { - int output = Integer.parseInt(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, CPUSET_MEMORY_PRESSURE_ENABLED)).get(0)); + int output = Integer.parseInt(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, + CPUSET_MEMORY_PRESSURE_ENABLED)).get(0)); return output > 0; } public void setMemPressureEnabled(boolean flag) throws IOException { - CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, CPUSET_MEMORY_PRESSURE_ENABLED), String.valueOf(flag ? 1 : 0)); + CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, CPUSET_MEMORY_PRESSURE_ENABLED), + String.valueOf(flag ? 1 : 0)); } public boolean isMemSpreadPage() throws IOException { - int output = Integer.parseInt(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, CPUSET_MEMORY_SPREAD_PAGE)).get(0)); + int output = Integer.parseInt(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, + CPUSET_MEMORY_SPREAD_PAGE)).get(0)); return output > 0; } public void setMemSpreadPage(boolean flag) throws IOException { - CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, CPUSET_MEMORY_SPREAD_PAGE), String.valueOf(flag ? 1 : 0)); + CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, CPUSET_MEMORY_SPREAD_PAGE), String + .valueOf(flag ? 1 : 0)); } public boolean isMemSpreadSlab() throws IOException { - int output = Integer.parseInt(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, CPUSET_MEMORY_SPREAD_SLAB)).get(0)); + int output = Integer.parseInt(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, + CPUSET_MEMORY_SPREAD_SLAB)).get(0)); return output > 0; } public void setMemSpreadSlab(boolean flag) throws IOException { - CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, CPUSET_MEMORY_SPREAD_SLAB), String.valueOf(flag ? 1 : 0)); + CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, CPUSET_MEMORY_SPREAD_SLAB), String + .valueOf(flag ? 1 : 0)); } public boolean isSchedLoadBlance() throws IOException { - int output = Integer.parseInt(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, CPUSET_SCHED_LOAD_BALANCE)).get(0)); + int output = Integer.parseInt(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, + CPUSET_SCHED_LOAD_BALANCE)).get(0)); return output > 0; } public void setSchedLoadBlance(boolean flag) throws IOException { - CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, CPUSET_SCHED_LOAD_BALANCE), String.valueOf(flag ? 1 : 0)); + CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, CPUSET_SCHED_LOAD_BALANCE), String + .valueOf(flag ? 1 : 0)); } public int getSchedRelaxDomainLevel() throws IOException { - String output = CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, CPUSET_SCHED_RELAX_DOMAIN_LEVEL)).get(0); + String output = CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, + CPUSET_SCHED_RELAX_DOMAIN_LEVEL)).get(0); return Integer.parseInt(output); } public void setSchedRelaxDomainLevel(int value) throws IOException { - CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, CPUSET_SCHED_RELAX_DOMAIN_LEVEL), String.valueOf(value)); + CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, CPUSET_SCHED_RELAX_DOMAIN_LEVEL), + String.valueOf(value)); } } diff --git a/storm-client/src/jvm/org/apache/storm/container/cgroup/core/DevicesCore.java b/storm-client/src/jvm/org/apache/storm/container/cgroup/core/DevicesCore.java index f7b9d52fdcb..1baf01c8923 100755 --- a/storm-client/src/jvm/org/apache/storm/container/cgroup/core/DevicesCore.java +++ b/storm-client/src/jvm/org/apache/storm/container/cgroup/core/DevicesCore.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -46,7 +52,8 @@ public SubSystemType getType() { return SubSystemType.devices; } - private void setPermission(String prop, char type, Device device, int accesses) throws IOException { + private void setPermission(String prop, char type, Device device, + int accesses) throws IOException { Record record = new Record(type, device, accesses); CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, prop), record.toString()); } @@ -60,7 +67,8 @@ public void setDeny(char type, Device device, int accesses) throws IOException { } public Record[] getList() throws IOException { - List output = CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, DEVICES_LIST)); + List output = CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, + DEVICES_LIST)); return Record.parseRecordList(output); } diff --git a/storm-client/src/jvm/org/apache/storm/container/cgroup/core/FreezerCore.java b/storm-client/src/jvm/org/apache/storm/container/cgroup/core/FreezerCore.java index 225ecb423aa..0e56704c568 100755 --- a/storm-client/src/jvm/org/apache/storm/container/cgroup/core/FreezerCore.java +++ b/storm-client/src/jvm/org/apache/storm/container/cgroup/core/FreezerCore.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -32,11 +38,13 @@ public SubSystemType getType() { } public State getState() throws IOException { - return State.getStateValue(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, FREEZER_STATE)).get(0)); + return State.getStateValue(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, + FREEZER_STATE)).get(0)); } public void setState(State state) throws IOException { - CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, FREEZER_STATE), state.name().toUpperCase()); + CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, FREEZER_STATE), state.name() + .toUpperCase()); } public enum State { diff --git a/storm-client/src/jvm/org/apache/storm/container/cgroup/core/MemoryCore.java b/storm-client/src/jvm/org/apache/storm/container/cgroup/core/MemoryCore.java index 46e03167d0d..fe2145ccccd 100755 --- a/storm-client/src/jvm/org/apache/storm/container/cgroup/core/MemoryCore.java +++ b/storm-client/src/jvm/org/apache/storm/container/cgroup/core/MemoryCore.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -44,81 +50,99 @@ public SubSystemType getType() { } public Stat getStat() throws IOException { - String output = CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, MEMORY_STAT)).get(0); + String output = CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, MEMORY_STAT)) + .get(0); Stat stat = new Stat(output); return stat; } public long getPhysicalUsage() throws IOException { - return Long.parseLong(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, MEMORY_USAGE_IN_BYTES)).get(0)); + return Long.parseLong(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, + MEMORY_USAGE_IN_BYTES)).get(0)); } public long getWithSwapUsage() throws IOException { - return Long.parseLong(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, MEMORY_MEMSW_USAGE_IN_BYTES)).get(0)); + return Long.parseLong(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, + MEMORY_MEMSW_USAGE_IN_BYTES)).get(0)); } public long getMaxPhysicalUsage() throws IOException { - return Long.parseLong(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, MEMORY_MAX_USAGE_IN_BYTES)).get(0)); + return Long.parseLong(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, + MEMORY_MAX_USAGE_IN_BYTES)).get(0)); } public long getMaxWithSwapUsage() throws IOException { - return Long.parseLong(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, MEMORY_MEMSW_MAX_USAGE_IN_BYTES)).get(0)); + return Long.parseLong(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, + MEMORY_MEMSW_MAX_USAGE_IN_BYTES)).get(0)); } public long getPhysicalUsageLimit() throws IOException { - return Long.parseLong(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, MEMORY_LIMIT_IN_BYTES)).get(0)); + return Long.parseLong(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, + MEMORY_LIMIT_IN_BYTES)).get(0)); } public void setPhysicalUsageLimit(long value) throws IOException { - CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, MEMORY_LIMIT_IN_BYTES), String.valueOf(value)); + CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, MEMORY_LIMIT_IN_BYTES), String + .valueOf(value)); } public long getWithSwapUsageLimit() throws IOException { - return Long.parseLong(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, MEMORY_MEMSW_LIMIT_IN_BYTES)).get(0)); + return Long.parseLong(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, + MEMORY_MEMSW_LIMIT_IN_BYTES)).get(0)); } public void setWithSwapUsageLimit(long value) throws IOException { - CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, MEMORY_MEMSW_LIMIT_IN_BYTES), String.valueOf(value)); + CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, MEMORY_MEMSW_LIMIT_IN_BYTES), + String.valueOf(value)); } public int getPhysicalFailCount() throws IOException { - return Integer.parseInt(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, MEMORY_FAILCNT)).get(0)); + return Integer.parseInt(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, + MEMORY_FAILCNT)).get(0)); } public int getWithSwapFailCount() throws IOException { - return Integer.parseInt(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, MEMORY_MEMSW_FAILCNT)).get(0)); + return Integer.parseInt(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, + MEMORY_MEMSW_FAILCNT)).get(0)); } public void clearForceEmpty() throws IOException { - CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, MEMORY_FORCE_EMPTY), String.valueOf(0)); + CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, MEMORY_FORCE_EMPTY), String + .valueOf(0)); } public int getSwappiness() throws IOException { - return Integer.parseInt(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, MEMORY_SWAPPINESS)).get(0)); + return Integer.parseInt(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, + MEMORY_SWAPPINESS)).get(0)); } public void setSwappiness(int value) throws IOException { - CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, MEMORY_SWAPPINESS), String.valueOf(value)); + CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, MEMORY_SWAPPINESS), String + .valueOf(value)); } public boolean isUseHierarchy() throws IOException { - int output = Integer.parseInt(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, MEMORY_USE_HIERARCHY)).get(0)); + int output = Integer.parseInt(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, + MEMORY_USE_HIERARCHY)).get(0)); return output > 0; } public void setUseHierarchy(boolean flag) throws IOException { - CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, MEMORY_USE_HIERARCHY), String.valueOf(flag ? 1 : 0)); + CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, MEMORY_USE_HIERARCHY), String + .valueOf(flag ? 1 : 0)); } public boolean isOomControl() throws IOException { - String output = CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, MEMORY_OOM_CONTROL)).get(0); + String output = CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, MEMORY_OOM_CONTROL)) + .get(0); output = output.split("\n")[0].split("[\\s]")[1]; int value = Integer.parseInt(output); return value > 0; } public void setOomControl(boolean flag) throws IOException { - CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, MEMORY_OOM_CONTROL), String.valueOf(flag ? 1 : 0)); + CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, MEMORY_OOM_CONTROL), String + .valueOf(flag ? 1 : 0)); } public static class Stat { diff --git a/storm-client/src/jvm/org/apache/storm/container/cgroup/core/NetClsCore.java b/storm-client/src/jvm/org/apache/storm/container/cgroup/core/NetClsCore.java index c239da3adb0..d80b6c2b2cf 100755 --- a/storm-client/src/jvm/org/apache/storm/container/cgroup/core/NetClsCore.java +++ b/storm-client/src/jvm/org/apache/storm/container/cgroup/core/NetClsCore.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -54,7 +60,8 @@ public void setClassId(int major, int minor) throws IOException { } public Device getClassId() throws IOException { - String output = CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, NET_CLS_CLASSID)).get(0); + String output = CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, NET_CLS_CLASSID)) + .get(0); output = Integer.toHexString(Integer.parseInt(output)); int major = Integer.parseInt(output.substring(0, output.length() - 4)); int minor = Integer.parseInt(output.substring(output.length() - 4)); diff --git a/storm-client/src/jvm/org/apache/storm/container/cgroup/core/NetPrioCore.java b/storm-client/src/jvm/org/apache/storm/container/cgroup/core/NetPrioCore.java index 29d2e95c3c1..27494cf4366 100755 --- a/storm-client/src/jvm/org/apache/storm/container/cgroup/core/NetPrioCore.java +++ b/storm-client/src/jvm/org/apache/storm/container/cgroup/core/NetPrioCore.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -36,7 +42,8 @@ public SubSystemType getType() { } public int getPrioId() throws IOException { - return Integer.parseInt(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, NET_PRIO_PRIOIDX)).get(0)); + return Integer.parseInt(CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, + NET_PRIO_PRIOIDX)).get(0)); } public void setIfPrioMap(String iface, int priority) throws IOException { @@ -44,12 +51,14 @@ public void setIfPrioMap(String iface, int priority) throws IOException { sb.append(iface); sb.append(' '); sb.append(priority); - CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, NET_PRIO_IFPRIOMAP), sb.toString()); + CgroupUtils.writeFileByLine(CgroupUtils.getDir(this.dir, NET_PRIO_IFPRIOMAP), sb + .toString()); } public Map getIfPrioMap() throws IOException { Map result = new HashMap(); - List strs = CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, NET_PRIO_IFPRIOMAP)); + List strs = CgroupUtils.readFileByLine(CgroupUtils.getDir(this.dir, + NET_PRIO_IFPRIOMAP)); for (String str : strs) { String[] strArgs = str.split(" "); result.put(strArgs[0], Integer.valueOf(strArgs[1])); diff --git a/storm-client/src/jvm/org/apache/storm/coordination/BatchBoltExecutor.java b/storm-client/src/jvm/org/apache/storm/coordination/BatchBoltExecutor.java index 4b7fcf4bd13..b78248a8701 100644 --- a/storm-client/src/jvm/org/apache/storm/coordination/BatchBoltExecutor.java +++ b/storm-client/src/jvm/org/apache/storm/coordination/BatchBoltExecutor.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -40,7 +46,8 @@ public BatchBoltExecutor(IBatchBolt bolt) { } @Override - public void prepare(Map conf, TopologyContext context, OutputCollector collector) { + public void prepare(Map conf, TopologyContext context, + OutputCollector collector) { this.conf = conf; this.context = context; this.collector = new BatchOutputCollectorImpl(collector); @@ -76,7 +83,6 @@ public void timeoutId(Object attempt) { openTransactions.remove(attempt); } - @Override public void declareOutputFields(OutputFieldsDeclarer declarer) { newTransactionalBolt().declareOutputFields(declarer); diff --git a/storm-client/src/jvm/org/apache/storm/coordination/BatchOutputCollector.java b/storm-client/src/jvm/org/apache/storm/coordination/BatchOutputCollector.java index 2baff31aaa5..a2d69ce9360 100644 --- a/storm-client/src/jvm/org/apache/storm/coordination/BatchOutputCollector.java +++ b/storm-client/src/jvm/org/apache/storm/coordination/BatchOutputCollector.java @@ -33,7 +33,8 @@ public List emit(List tuple) { public abstract List emit(String streamId, List tuple); /** - * Emits a tuple to the specified task on the default output stream. This output stream must have been declared as a direct stream, and + * Emits a tuple to the specified task on the default output stream. This output stream must + * have been declared as a direct stream, and * the specified task must use a direct grouping on this stream to receive the message. */ public void emitDirect(int taskId, List tuple) { diff --git a/storm-client/src/jvm/org/apache/storm/coordination/BatchOutputCollectorImpl.java b/storm-client/src/jvm/org/apache/storm/coordination/BatchOutputCollectorImpl.java index bea8a45812c..9e5cad7490d 100644 --- a/storm-client/src/jvm/org/apache/storm/coordination/BatchOutputCollectorImpl.java +++ b/storm-client/src/jvm/org/apache/storm/coordination/BatchOutputCollectorImpl.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/coordination/BatchSubtopologyBuilder.java b/storm-client/src/jvm/org/apache/storm/coordination/BatchSubtopologyBuilder.java index 734be3619ba..186d56dd406 100644 --- a/storm-client/src/jvm/org/apache/storm/coordination/BatchSubtopologyBuilder.java +++ b/storm-client/src/jvm/org/apache/storm/coordination/BatchSubtopologyBuilder.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -34,13 +40,13 @@ import org.apache.storm.topology.TopologyBuilder; import org.apache.storm.tuple.Fields; - public class BatchSubtopologyBuilder { Map bolts = new HashMap<>(); Component masterBolt; String masterId; - public BatchSubtopologyBuilder(String masterBoltId, IBasicBolt masterBolt, Number boltParallelism) { + public BatchSubtopologyBuilder(String masterBoltId, IBasicBolt masterBolt, + Number boltParallelism) { Integer p = boltParallelism == null ? null : boltParallelism.intValue(); this.masterBolt = new Component(new BasicBoltExecutor(masterBolt), p); masterId = masterBoltId; @@ -81,7 +87,8 @@ private BoltDeclarer setBolt(String id, IRichBolt bolt, Number parallelism) { } public void extendTopology(TopologyBuilder builder) { - BoltDeclarer declarer = builder.setBolt(masterId, new CoordinatedBolt(masterBolt.bolt), masterBolt.parallelism); + BoltDeclarer declarer = builder.setBolt(masterId, new CoordinatedBolt(masterBolt.bolt), + masterBolt.parallelism); for (InputDeclaration decl : masterBolt.declarations) { decl.declare(declarer); } @@ -171,7 +178,8 @@ public String getComponent() { } @Override - public BoltDeclarer fieldsGrouping(final String component, final String streamId, final Fields fields) { + public BoltDeclarer fieldsGrouping(final String component, final String streamId, + final Fields fields) { addDeclaration(new InputDeclaration() { @Override public void declare(InputDeclarer declarer) { @@ -389,7 +397,8 @@ public BoltDeclarer partialKeyGrouping(String componentId, String streamId, Fiel } @Override - public BoltDeclarer customGrouping(final String component, final CustomStreamGrouping grouping) { + public BoltDeclarer customGrouping(final String component, + final CustomStreamGrouping grouping) { addDeclaration(new InputDeclaration() { @Override public void declare(InputDeclarer declarer) { @@ -405,7 +414,8 @@ public String getComponent() { } @Override - public BoltDeclarer customGrouping(final String component, final String streamId, final CustomStreamGrouping grouping) { + public BoltDeclarer customGrouping(final String component, final String streamId, + final CustomStreamGrouping grouping) { addDeclaration(new InputDeclaration() { @Override public void declare(InputDeclarer declarer) { diff --git a/storm-client/src/jvm/org/apache/storm/coordination/CoordinatedBolt.java b/storm-client/src/jvm/org/apache/storm/coordination/CoordinatedBolt.java index 9e763f33bd8..794fb0664e2 100644 --- a/storm-client/src/jvm/org/apache/storm/coordination/CoordinatedBolt.java +++ b/storm-client/src/jvm/org/apache/storm/coordination/CoordinatedBolt.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -19,8 +25,8 @@ import java.util.HashMap; import java.util.Iterator; import java.util.List; -import java.util.Map; import java.util.Map.Entry; +import java.util.Map; import org.apache.storm.Constants; import org.apache.storm.generated.GlobalStreamId; import org.apache.storm.generated.Grouping; @@ -39,7 +45,8 @@ import org.slf4j.LoggerFactory; /** - * Coordination requires the request ids to be globally unique for awhile. This is so it doesn't get confused in the case of retries. + * Coordination requires the request ids to be globally unique for awhile. This is so it doesn't get + * confused in the case of retries. */ public class CoordinatedBolt implements IRichBolt { public static final Logger LOG = LoggerFactory.getLogger(CoordinatedBolt.class); @@ -55,11 +62,13 @@ public CoordinatedBolt(IRichBolt delegate) { this(delegate, null, null); } - public CoordinatedBolt(IRichBolt delegate, String sourceComponent, SourceArgs sourceArgs, IdStreamSpec idStreamSpec) { + public CoordinatedBolt(IRichBolt delegate, String sourceComponent, SourceArgs sourceArgs, + IdStreamSpec idStreamSpec) { this(delegate, singleSourceArgs(sourceComponent, sourceArgs), idStreamSpec); } - public CoordinatedBolt(IRichBolt delegate, Map sourceArgs, IdStreamSpec idStreamSpec) { + public CoordinatedBolt(IRichBolt delegate, Map sourceArgs, + IdStreamSpec idStreamSpec) { this.sourceArgs = sourceArgs; if (this.sourceArgs == null) { this.sourceArgs = new HashMap<>(); @@ -68,21 +77,24 @@ public CoordinatedBolt(IRichBolt delegate, Map sourceArgs, I this.idStreamSpec = idStreamSpec; } - private static Map singleSourceArgs(String sourceComponent, SourceArgs sourceArgs) { + private static Map singleSourceArgs(String sourceComponent, + SourceArgs sourceArgs) { Map ret = new HashMap<>(); ret.put(sourceComponent, sourceArgs); return ret; } @Override - public void prepare(Map config, TopologyContext context, OutputCollector collector) { + public void prepare(Map config, TopologyContext context, + OutputCollector collector) { TimeCacheMap.ExpiredCallback callback = null; if (delegate instanceof TimeoutCallback) { callback = new TimeoutItems(); } tracked = new TimeCacheMap<>(context.maxTopologyMessageTimeout(), callback); this.collector = collector; - delegate.prepare(config, context, new OutputCollector(new CoordinatedOutputCollector(collector))); + delegate.prepare(config, context, + new OutputCollector(new CoordinatedOutputCollector(collector))); for (String component : Utils.get(context.getThisTargets(), Constants.COORDINATED_STREAM_ID, new HashMap()) @@ -112,7 +124,8 @@ private boolean checkFinishId(Tuple tup, TupleType type) { try { if (track != null) { boolean delayed = false; - if (idStreamSpec == null && type == TupleType.COORD || idStreamSpec != null && type == TupleType.ID) { + if (idStreamSpec == null && type == TupleType.COORD || idStreamSpec != null + && type == TupleType.ID) { track.ackTuples.add(tup); delayed = true; } @@ -123,18 +136,21 @@ private boolean checkFinishId(Tuple tup, TupleType type) { } tracked.remove(id); } else if (track.receivedId && (sourceArgs.isEmpty() - || track.reportCount == numSourceReports && track.expectedTupleCount == track.receivedTuples)) { + || track.reportCount == numSourceReports + && track.expectedTupleCount == track.receivedTuples)) { if (delegate instanceof FinishedCallback) { ((FinishedCallback) delegate).finishedId(id); } if (!(sourceArgs.isEmpty() || type != TupleType.REGULAR)) { - throw new IllegalStateException("Coordination condition met on a non-coordinating tuple. Should be impossible"); + throw new IllegalStateException("Coordination condition met on a " + + "non-coordinating tuple. Should be impossible"); } Iterator outTasks = countOutTasks.iterator(); while (outTasks.hasNext()) { int task = outTasks.next(); int numTuples = Utils.get(track.taskEmittedTuples, task, 0); - collector.emitDirect(task, Constants.COORDINATED_STREAM_ID, tup, new Values(id, numTuples)); + collector.emitDirect(task, Constants.COORDINATED_STREAM_ID, tup, + new Values(id, numTuples)); } for (Tuple t : track.ackTuples) { collector.ack(t); @@ -316,7 +332,8 @@ public List emit(String stream, Collection anchors, List } @Override - public void emitDirect(int task, String stream, Collection anchors, List tuple) { + public void emitDirect(int task, String stream, Collection anchors, + List tuple) { updateTaskCounts(tuple.get(0), Arrays.asList(task)); delegate.emitDirect(task, stream, anchors, tuple); } @@ -366,7 +383,6 @@ public void reportError(Throwable error) { delegate.reportError(error); } - private void updateTaskCounts(Object id, List tasks) { synchronized (tracked) { TrackingInfo track = tracked.get(id); diff --git a/storm-client/src/jvm/org/apache/storm/coordination/IBatchBolt.java b/storm-client/src/jvm/org/apache/storm/coordination/IBatchBolt.java index 8cb7be378a9..5ced565c17e 100644 --- a/storm-client/src/jvm/org/apache/storm/coordination/IBatchBolt.java +++ b/storm-client/src/jvm/org/apache/storm/coordination/IBatchBolt.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -19,7 +25,8 @@ import org.apache.storm.tuple.Tuple; public interface IBatchBolt extends Serializable, IComponent { - void prepare(Map conf, TopologyContext context, BatchOutputCollector collector, T id); + void prepare(Map conf, TopologyContext context, BatchOutputCollector collector, + T id); void execute(Tuple tuple); diff --git a/storm-client/src/jvm/org/apache/storm/daemon/Acker.java b/storm-client/src/jvm/org/apache/storm/daemon/Acker.java index 336957af2bb..64d47629990 100644 --- a/storm-client/src/jvm/org/apache/storm/daemon/Acker.java +++ b/storm-client/src/jvm/org/apache/storm/daemon/Acker.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -39,7 +45,8 @@ public class Acker implements IBolt { private RotatingMap pending; @Override - public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) { + public void prepare(Map topoConf, TopologyContext context, + OutputCollector collector) { this.collector = collector; this.pending = new RotatingMap<>(TIMEOUT_BUCKET_NUM); } @@ -102,7 +109,8 @@ public void execute(Tuple input) { } else if (resetTimeout) { collector.emitDirect(task, ACKER_RESET_TIMEOUT_STREAM_ID, tuple); } else { - throw new IllegalStateException("The checks are inconsistent we reach what should be unreachable code."); + throw new IllegalStateException("The checks are inconsistent we reach what should " + + "be unreachable code."); } } diff --git a/storm-client/src/jvm/org/apache/storm/daemon/DaemonCommon.java b/storm-client/src/jvm/org/apache/storm/daemon/DaemonCommon.java index d4f9376d203..71ef4edb8ad 100644 --- a/storm-client/src/jvm/org/apache/storm/daemon/DaemonCommon.java +++ b/storm-client/src/jvm/org/apache/storm/daemon/DaemonCommon.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/daemon/GrouperFactory.java b/storm-client/src/jvm/org/apache/storm/daemon/GrouperFactory.java index 0f8359783c2..955ed675dfd 100644 --- a/storm-client/src/jvm/org/apache/storm/daemon/GrouperFactory.java +++ b/storm-client/src/jvm/org/apache/storm/daemon/GrouperFactory.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -44,7 +50,8 @@ public void refreshLoad(LoadMapping loadMapping) { } @Override - public void prepare(WorkerTopologyContext context, GlobalStreamId stream, List targetTasks) { + public void prepare(WorkerTopologyContext context, GlobalStreamId stream, + List targetTasks) { } @@ -55,13 +62,15 @@ public List chooseTasks(int taskId, List values) { }; - public static LoadAwareCustomStreamGrouping mkGrouper(WorkerTopologyContext context, String componentId, String streamId, + public static LoadAwareCustomStreamGrouping mkGrouper(WorkerTopologyContext context, + String componentId, String streamId, Fields outFields, Grouping thriftGrouping, List unsortedTargetTasks, Map topoConf) { List targetTasks = Ordering.natural().sortedCopy(unsortedTargetTasks); - final boolean isNotLoadAware = (null != topoConf.get(Config.TOPOLOGY_DISABLE_LOADAWARE_MESSAGING) && (boolean) topoConf + final boolean isNotLoadAware = (null != topoConf + .get(Config.TOPOLOGY_DISABLE_LOADAWARE_MESSAGING) && (boolean) topoConf .get(Config.TOPOLOGY_DISABLE_LOADAWARE_MESSAGING)); CustomStreamGrouping result = null; switch (Thrift.groupingType(thriftGrouping)) { @@ -84,7 +93,8 @@ public static LoadAwareCustomStreamGrouping mkGrouper(WorkerTopologyContext cont break; case LOCAL_OR_SHUFFLE: // Prefer local tasks as target tasks if possible - Set sameTasks = Sets.intersection(Sets.newHashSet(targetTasks), Sets.newHashSet(context.getThisWorkerTasks())); + Set sameTasks = Sets.intersection(Sets.newHashSet(targetTasks), Sets + .newHashSet(context.getThisWorkerTasks())); targetTasks = (sameTasks.isEmpty()) ? targetTasks : new ArrayList<>(sameTasks); if (isNotLoadAware) { result = new ShuffleGrouping(); @@ -96,10 +106,12 @@ public static LoadAwareCustomStreamGrouping mkGrouper(WorkerTopologyContext cont result = new NoneGrouper(); break; case CUSTOM_OBJECT: - result = (CustomStreamGrouping) Thrift.instantiateJavaObject(thriftGrouping.get_custom_object()); + result = (CustomStreamGrouping) Thrift.instantiateJavaObject(thriftGrouping + .get_custom_object()); break; case CUSTOM_SERIALIZED: - result = Utils.javaDeserialize(thriftGrouping.get_custom_serialized(), CustomStreamGrouping.class); + result = Utils.javaDeserialize(thriftGrouping.get_custom_serialized(), + CustomStreamGrouping.class); break; case DIRECT: result = DIRECT; @@ -137,7 +149,8 @@ public void refreshLoad(LoadMapping loadMapping) { } @Override - public void prepare(WorkerTopologyContext context, GlobalStreamId stream, List targetTasks) { + public void prepare(WorkerTopologyContext context, GlobalStreamId stream, + List targetTasks) { customStreamGrouping.prepare(context, stream, targetTasks); } @@ -161,7 +174,8 @@ public FieldsGrouper(Fields outFields, Grouping thriftGrouping) { } @Override - public void prepare(WorkerTopologyContext context, GlobalStreamId stream, List targetTasks) { + public void prepare(WorkerTopologyContext context, GlobalStreamId stream, + List targetTasks) { this.targetTasks = new ArrayList>(); for (Integer targetTask : targetTasks) { this.targetTasks.add(Collections.singletonList(targetTask)); @@ -171,7 +185,8 @@ public void prepare(WorkerTopologyContext context, GlobalStreamId stream, List chooseTasks(int taskId, List values) { - int targetTaskIndex = TupleUtils.chooseTaskIndex(outFields.select(groupFields, values), numTasks); + int targetTaskIndex = TupleUtils.chooseTaskIndex(outFields.select(groupFields, values), + numTasks); return targetTasks.get(targetTaskIndex); } @@ -185,7 +200,8 @@ public GlobalGrouper() { } @Override - public void prepare(WorkerTopologyContext context, GlobalStreamId stream, List targetTasks) { + public void prepare(WorkerTopologyContext context, GlobalStreamId stream, + List targetTasks) { this.targetTasks = targetTasks; } @@ -210,7 +226,8 @@ public NoneGrouper() { } @Override - public void prepare(WorkerTopologyContext context, GlobalStreamId stream, List targetTasks) { + public void prepare(WorkerTopologyContext context, GlobalStreamId stream, + List targetTasks) { this.targetTasks = targetTasks; this.numTasks = targetTasks.size(); } @@ -227,7 +244,8 @@ public static class AllGrouper implements CustomStreamGrouping { private List targetTasks; @Override - public void prepare(WorkerTopologyContext context, GlobalStreamId stream, List targetTasks) { + public void prepare(WorkerTopologyContext context, GlobalStreamId stream, + List targetTasks) { this.targetTasks = targetTasks; } diff --git a/storm-client/src/jvm/org/apache/storm/daemon/Shutdownable.java b/storm-client/src/jvm/org/apache/storm/daemon/Shutdownable.java index 4d9517be88f..0101e69b980 100644 --- a/storm-client/src/jvm/org/apache/storm/daemon/Shutdownable.java +++ b/storm-client/src/jvm/org/apache/storm/daemon/Shutdownable.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/daemon/StormCommon.java b/storm-client/src/jvm/org/apache/storm/daemon/StormCommon.java index 0776e1eea47..1c6b5054041 100644 --- a/storm-client/src/jvm/org/apache/storm/daemon/StormCommon.java +++ b/storm-client/src/jvm/org/apache/storm/daemon/StormCommon.java @@ -1,12 +1,17 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -21,7 +26,6 @@ import java.util.Map; import java.util.Set; import java.util.TreeMap; - import org.apache.storm.Config; import org.apache.storm.Constants; import org.apache.storm.Thrift; @@ -60,12 +64,14 @@ public class StormCommon { public static final String EVENTLOGGER_STREAM_ID = "__eventlog"; public static final String TOPOLOGY_METRICS_CONSUMER_CLASS = "class"; public static final String TOPOLOGY_METRICS_CONSUMER_ARGUMENT = "argument"; - public static final String TOPOLOGY_METRICS_CONSUMER_MAX_RETAIN_METRIC_TUPLES = "max.retain.metric.tuples"; + public static final String TOPOLOGY_METRICS_CONSUMER_MAX_RETAIN_METRIC_TUPLES = + "max.retain.metric.tuples"; public static final String TOPOLOGY_METRICS_CONSUMER_PARALLELISM_HINT = "parallelism.hint"; public static final String TOPOLOGY_METRICS_CONSUMER_WHITELIST = "whitelist"; public static final String TOPOLOGY_METRICS_CONSUMER_BLACKLIST = "blacklist"; public static final String TOPOLOGY_METRICS_CONSUMER_EXPAND_MAP_TYPE = "expandMapType"; - public static final String TOPOLOGY_METRICS_CONSUMER_METRIC_NAME_SEPARATOR = "metricNameSeparator"; + public static final String TOPOLOGY_METRICS_CONSUMER_METRIC_NAME_SEPARATOR = + "metricNameSeparator"; public static final String TOPOLOGY_EVENT_LOGGER_CLASS = "class"; public static final String TOPOLOGY_EVENT_LOGGER_ARGUMENTS = "arguments"; private static final Logger LOG = LoggerFactory.getLogger(StormCommon.class); @@ -74,7 +80,8 @@ public class StormCommon { private static StormCommon _instance = new StormCommon(); /** - * Provide an instance of this class for delegates to use. To mock out delegated methods, provide an instance of a subclass that + * Provide an instance of this class for delegates to use. To mock out delegated methods, + * provide an instance of a subclass that * overrides the implementation of the delegated method. * * @param common a StormCommon instance @@ -92,7 +99,8 @@ public static void validateDistributedMode(Map conf) { } } - private static Set validateIds(Map componentMap) throws InvalidTopologyException { + private static Set validateIds(Map componentMap) throws InvalidTopologyException { Set keys = componentMap.keySet(); for (String id : keys) { if (Utils.isSystemId(id)) { @@ -158,12 +166,14 @@ public static void validateBasic(StormTopology topology) throws InvalidTopologyE validateIds(topology); for (StormTopology._Fields field : Thrift.getSpoutFields()) { - Map spoutComponents = (Map) topology.getFieldValue(field); + Map spoutComponents = (Map) topology + .getFieldValue(field); if (spoutComponents != null) { for (Object obj : spoutComponents.values()) { ComponentCommon common = getComponentCommon(obj); if (!isEmptyInputs(common)) { - throw new WrappedInvalidTopologyException("May not declare inputs for a spout"); + throw new WrappedInvalidTopologyException("May not declare inputs for a " + + "spout"); } } } @@ -177,7 +187,8 @@ public static void validateBasic(StormTopology topology) throws InvalidTopologyE Integer taskNum = ObjectReader.getInt(conf.get(Config.TOPOLOGY_TASKS), 0); if (taskNum > 0 && parallelismHintNum <= 0) { throw new WrappedInvalidTopologyException( - "Number of executors must be greater than 0 when number of tasks is greater than 0"); + "Number of executors must be greater than 0 when number of tasks is " + + "greater than 0"); } } } @@ -201,14 +212,17 @@ public static void validateStructure(StormTopology topology) throws InvalidTopol String sourceComponentId = input.getKey().get_componentId(); if (!componentMap.keySet().contains(sourceComponentId)) { throw new WrappedInvalidTopologyException("Component: [" + componentId - + "] subscribes from non-existent component [" + sourceComponentId + "]"); + + "] subscribes from non-existent " + + "component [" + sourceComponentId + "]"); } - ComponentCommon sourceComponent = getComponentCommon(componentMap.get(sourceComponentId)); + ComponentCommon sourceComponent = getComponentCommon(componentMap + .get(sourceComponentId)); if (!sourceComponent.get_streams().containsKey(sourceStreamId)) { throw new WrappedInvalidTopologyException("Component: [" + componentId + "] subscribes from non-existent stream: " - + "[" + sourceStreamId + "] of component [" + sourceComponentId + "]"); + + "[" + sourceStreamId + "] of component [" + + sourceComponentId + "]"); } Grouping grouping = input.getValue(); @@ -219,8 +233,11 @@ public static void validateStructure(StormTopology topology) throws InvalidTopol fields.removeAll(sourceOutputFields); if (fields.size() != 0) { throw new WrappedInvalidTopologyException("Component: [" + componentId - + "] subscribes from stream: [" + sourceStreamId + "] of component " - + "[" + sourceComponentId + "] + with non-existent fields: " + fields); + + "] subscribes from stream: [" + + sourceStreamId + "] of component " + + "[" + sourceComponentId + + "] + with non-existent fields: " + + fields); } } } @@ -256,24 +273,33 @@ public static IBolt makeAckerBolt() { public static void addAcker(Map conf, StormTopology topology) { Map outputStreams = new HashMap(); - outputStreams.put(Acker.ACKER_ACK_STREAM_ID, Thrift.directOutputFields(Arrays.asList("id", "time-delta-ms"))); - outputStreams.put(Acker.ACKER_FAIL_STREAM_ID, Thrift.directOutputFields(Arrays.asList("id", "time-delta-ms"))); - outputStreams.put(Acker.ACKER_RESET_TIMEOUT_STREAM_ID, Thrift.directOutputFields(Arrays.asList("id", "time-delta-ms"))); + outputStreams.put(Acker.ACKER_ACK_STREAM_ID, Thrift.directOutputFields(Arrays.asList("id", + "time-delta-ms"))); + outputStreams.put(Acker.ACKER_FAIL_STREAM_ID, Thrift.directOutputFields(Arrays.asList("id", + "time-delta-ms"))); + outputStreams.put(Acker.ACKER_RESET_TIMEOUT_STREAM_ID, Thrift.directOutputFields(Arrays + .asList("id", "time-delta-ms"))); Map ackerConf = new HashMap<>(); int ackerNum = - ObjectReader.getInt(conf.get(Config.TOPOLOGY_ACKER_EXECUTORS), ObjectReader.getInt(conf.get(Config.TOPOLOGY_WORKERS))); + ObjectReader.getInt(conf.get(Config.TOPOLOGY_ACKER_EXECUTORS), ObjectReader + .getInt(conf.get(Config.TOPOLOGY_WORKERS))); ackerConf.put(Config.TOPOLOGY_TASKS, ackerNum); - ackerConf.put(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, ObjectReader.getInt(conf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS))); + ackerConf.put(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, ObjectReader.getInt(conf + .get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS))); Map inputs = ackerInputs(topology); - Bolt acker = Thrift.prepareSerializedBoltDetails(inputs, makeAckerBolt(), outputStreams, ackerNum, ackerConf); + Bolt acker = Thrift.prepareSerializedBoltDetails(inputs, makeAckerBolt(), outputStreams, + ackerNum, ackerConf); for (Bolt bolt : topology.get_bolts().values()) { ComponentCommon common = bolt.get_common(); - common.put_to_streams(Acker.ACKER_ACK_STREAM_ID, Thrift.outputFields(Arrays.asList("id", "ack-val"))); - common.put_to_streams(Acker.ACKER_FAIL_STREAM_ID, Thrift.outputFields(Arrays.asList("id"))); - common.put_to_streams(Acker.ACKER_RESET_TIMEOUT_STREAM_ID, Thrift.outputFields(Arrays.asList("id"))); + common.put_to_streams(Acker.ACKER_ACK_STREAM_ID, Thrift.outputFields(Arrays.asList("id", + "ack-val"))); + common.put_to_streams(Acker.ACKER_FAIL_STREAM_ID, Thrift.outputFields(Arrays + .asList("id"))); + common.put_to_streams(Acker.ACKER_RESET_TIMEOUT_STREAM_ID, Thrift.outputFields(Arrays + .asList("id"))); } for (SpoutSpec spout : topology.get_spouts().values()) { @@ -283,12 +309,16 @@ public static void addAcker(Map conf, StormTopology topology) { ObjectReader.getInt(conf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS))); common.set_json_conf(JSONValue.toJSONString(spoutConf)); common.put_to_streams(Acker.ACKER_INIT_STREAM_ID, - Thrift.outputFields(Arrays.asList("id", "init-val", "spout-task"))); - common.put_to_inputs(Utils.getGlobalStreamId(Acker.ACKER_COMPONENT_ID, Acker.ACKER_ACK_STREAM_ID), + Thrift.outputFields(Arrays.asList("id", "init-val", + "spout-task"))); + common.put_to_inputs(Utils.getGlobalStreamId(Acker.ACKER_COMPONENT_ID, + Acker.ACKER_ACK_STREAM_ID), Thrift.prepareDirectGrouping()); - common.put_to_inputs(Utils.getGlobalStreamId(Acker.ACKER_COMPONENT_ID, Acker.ACKER_FAIL_STREAM_ID), + common.put_to_inputs(Utils.getGlobalStreamId(Acker.ACKER_COMPONENT_ID, + Acker.ACKER_FAIL_STREAM_ID), Thrift.prepareDirectGrouping()); - common.put_to_inputs(Utils.getGlobalStreamId(Acker.ACKER_COMPONENT_ID, Acker.ACKER_RESET_TIMEOUT_STREAM_ID), + common.put_to_inputs(Utils.getGlobalStreamId(Acker.ACKER_COMPONENT_ID, + Acker.ACKER_RESET_TIMEOUT_STREAM_ID), Thrift.prepareDirectGrouping()); } @@ -343,30 +373,35 @@ public static Map eventLoggerInputs(StormTopology topo public static void addEventLogger(Map conf, StormTopology topology) { Integer numExecutors = ObjectReader.getInt(conf.get(Config.TOPOLOGY_EVENTLOGGER_EXECUTORS), - ObjectReader.getInt(conf.get(Config.TOPOLOGY_WORKERS))); + ObjectReader.getInt(conf + .get(Config.TOPOLOGY_WORKERS))); if (numExecutors == null || numExecutors == 0) { return; } HashMap componentConf = new HashMap<>(); componentConf.put(Config.TOPOLOGY_TASKS, numExecutors); - componentConf.put(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, ObjectReader.getInt(conf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS))); + componentConf.put(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, ObjectReader.getInt(conf + .get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS))); Bolt eventLoggerBolt = Thrift.prepareSerializedBoltDetails( eventLoggerInputs(topology), new EventLoggerBolt(), null, numExecutors, componentConf); for (Object component : allComponents(topology).values()) { ComponentCommon common = getComponentCommon(component); - common.put_to_streams(EVENTLOGGER_STREAM_ID, Thrift.outputFields(eventLoggerBoltFields())); + common.put_to_streams(EVENTLOGGER_STREAM_ID, Thrift + .outputFields(eventLoggerBoltFields())); } topology.put_to_bolts(EVENTLOGGER_COMPONENT_ID, eventLoggerBolt); } public static void addUpstreamFeedback(Map conf, StormTopology topology) { - // Only invoked when hasUpstreamFeedback(conf) is true, so declare the feedback stream on every + // Only invoked when hasUpstreamFeedback(conf) is true, so declare the feedback stream on + // every // component unconditionally. The schema must match the tuple emitted by // Executor.buildUpstreamFeedbackTuple: [TaskInfo, EwmaFeedbackRecord]. for (Object component : allComponents(topology).values()) { ComponentCommon common = getComponentCommon(component); - common.put_to_streams(Constants.FEEDBACK_STREAM_ID, Thrift.outputFields(upstreamFeedbackFields())); + common.put_to_streams(Constants.FEEDBACK_STREAM_ID, Thrift + .outputFields(upstreamFeedbackFields())); } } @@ -375,7 +410,8 @@ public static List upstreamFeedbackFields() { } @SuppressWarnings("unchecked") - public static Map metricsConsumerBoltSpecs(Map conf, StormTopology topology) { + public static Map metricsConsumerBoltSpecs(Map conf, + StormTopology topology) { Map metricsConsumerBolts = new HashMap<>(); Set componentIdsEmitMetrics = new HashSet<>(); @@ -384,10 +420,12 @@ public static Map metricsConsumerBoltSpecs(Map con Map inputs = new HashMap<>(); for (String componentId : componentIdsEmitMetrics) { - inputs.put(Utils.getGlobalStreamId(componentId, Constants.METRICS_STREAM_ID), Thrift.prepareShuffleGrouping()); + inputs.put(Utils.getGlobalStreamId(componentId, Constants.METRICS_STREAM_ID), Thrift + .prepareShuffleGrouping()); } - List> registerInfo = (List>) conf.get(Config.TOPOLOGY_METRICS_CONSUMER_REGISTER); + List> registerInfo = (List>) conf + .get(Config.TOPOLOGY_METRICS_CONSUMER_REGISTER); if (registerInfo != null) { Map classOccurrencesMap = new HashMap<>(); for (Map info : registerInfo) { @@ -395,7 +433,8 @@ public static Map metricsConsumerBoltSpecs(Map con Object argument = info.get(TOPOLOGY_METRICS_CONSUMER_ARGUMENT); Integer maxRetainMetricTuples = ObjectReader.getInt(info.get( TOPOLOGY_METRICS_CONSUMER_MAX_RETAIN_METRIC_TUPLES), 100); - Integer phintNum = ObjectReader.getInt(info.get(TOPOLOGY_METRICS_CONSUMER_PARALLELISM_HINT), 1); + Integer phintNum = ObjectReader.getInt(info + .get(TOPOLOGY_METRICS_CONSUMER_PARALLELISM_HINT), 1); Map metricsConsumerConf = new HashMap<>(); metricsConsumerConf.put(Config.TOPOLOGY_TASKS, phintNum); List whitelist = (List) info.get( @@ -407,7 +446,8 @@ public static Map metricsConsumerBoltSpecs(Map con TOPOLOGY_METRICS_CONSUMER_EXPAND_MAP_TYPE), false); String metricNameSeparator = ObjectReader.getString(info.get( TOPOLOGY_METRICS_CONSUMER_METRIC_NAME_SEPARATOR), "."); - DataPointExpander expander = new DataPointExpander(expandMapType, metricNameSeparator); + DataPointExpander expander = new DataPointExpander(expandMapType, + metricNameSeparator); MetricsConsumerBolt boltInstance = new MetricsConsumerBolt(className, argument, maxRetainMetricTuples, filterPredicate, expander); Bolt metricsConsumerBolt = Thrift.prepareSerializedBoltDetails(inputs, @@ -440,34 +480,43 @@ public static void addMetricComponents(Map conf, StormTopology t @SuppressWarnings("unused") public static void addSystemComponents(Map conf, StormTopology topology) { Map outputStreams = new HashMap<>(); - outputStreams.put(Constants.SYSTEM_TICK_STREAM_ID, Thrift.outputFields(Arrays.asList("rate_secs"))); + outputStreams.put(Constants.SYSTEM_TICK_STREAM_ID, Thrift.outputFields(Arrays + .asList("rate_secs"))); outputStreams.put(Constants.SYSTEM_FLUSH_STREAM_ID, Thrift.outputFields(Arrays.asList())); - outputStreams.put(Constants.METRICS_TICK_STREAM_ID, Thrift.outputFields(Arrays.asList("interval"))); + outputStreams.put(Constants.METRICS_TICK_STREAM_ID, Thrift.outputFields(Arrays + .asList("interval"))); if (ConfigUtils.upstreamFeedbackEnable(conf)) { - outputStreams.put(Constants.FEEDBACK_TICK_STREAM_ID, Thrift.outputFields(Arrays.asList("interval"))); + outputStreams.put(Constants.FEEDBACK_TICK_STREAM_ID, Thrift.outputFields(Arrays + .asList("interval"))); } Map boltConf = new HashMap<>(); boltConf.put(Config.TOPOLOGY_TASKS, 0); - Bolt systemBoltSpec = Thrift.prepareSerializedBoltDetails(null, new SystemBolt(), outputStreams, 0, boltConf); + Bolt systemBoltSpec = Thrift.prepareSerializedBoltDetails(null, new SystemBolt(), + outputStreams, 0, boltConf); topology.put_to_bolts(Constants.SYSTEM_COMPONENT_ID, systemBoltSpec); } /** * Construct a new topology structure after adding system components and streams. - * WARNING: while changing the existing code to add or remove streams for a component is allowed, please be aware that + * WARNING: while changing the existing code to add or remove streams for a component is + * allowed, please be aware that * it might cause issues during cluster rolling upgrade - * because {@link SerializationFactory.IdDictionary} depends on having a consistent map of component to streams + * because {@link SerializationFactory.IdDictionary} depends on having a consistent map of + * component to streams * to work properly (see STORM-3687 for an example). - * It will not impact a cluster running on a single version or running an older topology on a newer cluster. + * It will not impact a cluster running on a single version or running an older topology on a + * newer cluster. * But a mixed cluster (with different versions of daemons running) is not guaranteed to work. + * * @param topoConf the topology configuration * @param topology the original topology structure * @return the newly constructed topology * @throws InvalidTopologyException if the topology is invalid */ - public static StormTopology systemTopology(Map topoConf, StormTopology topology) throws InvalidTopologyException { + public static StormTopology systemTopology(Map topoConf, + StormTopology topology) throws InvalidTopologyException { return _instance.systemTopologyImpl(topoConf, topology); } @@ -490,7 +539,8 @@ public static int numStartExecutors(Object component) throws InvalidTopologyExce return Thrift.getParallelismHint(common); } - public static Map stormTaskInfo(StormTopology userTopology, Map topoConf) throws + public static Map stormTaskInfo(StormTopology userTopology, Map topoConf) throws InvalidTopologyException { return _instance.stormTaskInfoImpl(userTopology, topoConf); } @@ -505,7 +555,8 @@ public static List executorIdToTasks(List executorId) { return taskIds; } - public static Map taskToNodeport(Map, NodeInfo> executorToNodePort) { + public static Map taskToNodeport(Map, + NodeInfo> executorToNodePort) { Map tasksToNodePort = new HashMap<>(); for (Map.Entry, NodeInfo> entry : executorToNodePort.entrySet()) { List taskIds = executorIdToTasks(entry.getKey()); @@ -526,20 +577,26 @@ public static WorkerTopologyContext makeWorkerContext(Map worker try { StormTopology stormTopology = (StormTopology) workerData.get(Constants.SYSTEM_TOPOLOGY); Map topoConf = (Map) workerData.get(Constants.STORM_CONF); - Map taskToComponent = (Map) workerData.get(Constants.TASK_TO_COMPONENT); + Map taskToComponent = (Map) workerData + .get(Constants.TASK_TO_COMPONENT); Map> componentToSortedTasks = (Map>) workerData.get(Constants.COMPONENT_TO_SORTED_TASKS); Map> componentToStreamToFields = - (Map>) workerData.get(Constants.COMPONENT_TO_STREAM_TO_FIELDS); + (Map>) workerData + .get(Constants.COMPONENT_TO_STREAM_TO_FIELDS); String stormId = (String) workerData.get(Constants.STORM_ID); Map conf = (Map) workerData.get(Constants.CONF); Integer port = (Integer) workerData.get(Constants.PORT); - String codeDir = ConfigUtils.supervisorStormResourcesPath(ConfigUtils.supervisorStormDistRoot(conf, stormId)); + String codeDir = ConfigUtils.supervisorStormResourcesPath(ConfigUtils + .supervisorStormDistRoot(conf, stormId)); String pidDir = ConfigUtils.workerPidsRoot(conf, stormId); List workerTasks = (List) workerData.get(Constants.TASK_IDS); - Map defaultResources = (Map) workerData.get(Constants.DEFAULT_SHARED_RESOURCES); - Map userResources = (Map) workerData.get(Constants.USER_SHARED_RESOURCES); - return new WorkerTopologyContext(stormTopology, topoConf, taskToComponent, componentToSortedTasks, + Map defaultResources = (Map) workerData + .get(Constants.DEFAULT_SHARED_RESOURCES); + Map userResources = (Map) workerData + .get(Constants.USER_SHARED_RESOURCES); + return new WorkerTopologyContext(stormTopology, topoConf, taskToComponent, + componentToSortedTasks, componentToStreamToFields, stormId, codeDir, pidDir, port, workerTasks, defaultResources, userResources); } catch (IOException e) { @@ -551,7 +608,8 @@ public IBolt makeAckerBoltImpl() { return new Acker(); } - protected StormTopology systemTopologyImpl(Map topoConf, StormTopology topology) throws InvalidTopologyException { + protected StormTopology systemTopologyImpl(Map topoConf, + StormTopology topology) throws InvalidTopologyException { validateBasic(topology); StormTopology ret = topology.deepCopy(); @@ -575,7 +633,8 @@ protected StormTopology systemTopologyImpl(Map topoConf, StormTo /* * Returns map from task -> componentId */ - protected Map stormTaskInfoImpl(StormTopology userTopology, Map topoConf) throws + protected Map stormTaskInfoImpl(StormTopology userTopology, Map topoConf) throws InvalidTopologyException { Map taskIdToComponentId = new HashMap<>(); @@ -611,7 +670,8 @@ protected IAuthorizer mkAuthorizationHandlerImpl(String klassName, Maphttp://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -17,8 +23,8 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Map.Entry; +import java.util.Map; import java.util.Queue; import java.util.Random; import java.util.function.BooleanSupplier; @@ -86,17 +92,21 @@ public Task(Executor executor, Integer taskId) throws IOException { this.systemTopologyContext = mkTopologyContext(workerData.getSystemTopology()); this.userTopologyContext = mkTopologyContext(workerData.getTopology()); this.taskObject = mkTaskObject(); - this.debug = topoConf.containsKey(Config.TOPOLOGY_DEBUG) && (Boolean) topoConf.get(Config.TOPOLOGY_DEBUG); + this.debug = topoConf.containsKey(Config.TOPOLOGY_DEBUG) && (Boolean) topoConf + .get(Config.TOPOLOGY_DEBUG); this.addTaskHooks(); - this.taskMetrics = new TaskMetrics(this.workerTopologyContext, this.componentId, this.taskId, + this.taskMetrics = new TaskMetrics(this.workerTopologyContext, this.componentId, + this.taskId, workerData.getMetricRegistry(), topoConf); } private static HashMap> getGroupersPerStream( Map> streamComponentToGrouper) { - HashMap> result = new HashMap<>(streamComponentToGrouper.size()); + HashMap> result = + new HashMap<>(streamComponentToGrouper.size()); - for (Entry> entry : streamComponentToGrouper.entrySet()) { + for (Entry> entry : streamComponentToGrouper.entrySet()) { String stream = entry.getKey(); Map groupers = entry.getValue(); ArrayList perStreamGroupers = new ArrayList<>(); @@ -115,16 +125,19 @@ public List getOutgoingTasks(Integer outTaskId, String stream, List componentGrouping = streamComponentToGrouper.get(stream); + Map componentGrouping = streamComponentToGrouper + .get(stream); LoadAwareCustomStreamGrouping grouping = componentGrouping.get(targetComponent); if (null == grouping) { outTaskId = null; } if (grouping != null && grouping != GrouperFactory.DIRECT) { - throw new IllegalArgumentException("Cannot emitDirect to a task expecting a regular grouping"); + throw new IllegalArgumentException("Cannot emitDirect to a task expecting a regular " + + "grouping"); } if (!userTopologyContext.getHooks().isEmpty()) { - new EmitInfo(values, stream, taskId, Collections.singletonList(outTaskId)).applyOn(userTopologyContext); + new EmitInfo(values, stream, taskId, Collections.singletonList(outTaskId)) + .applyOn(userTopologyContext); } try { @@ -147,7 +160,8 @@ public List getOutgoingTasks(Integer outTaskId, String stream, List getOutgoingTasks(String stream, List values) { if (debug) { - LOG.info("Emitting Tuple: taskId={} componentId={} stream={} values={}", taskId, componentId, stream, values); + LOG.info("Emitting Tuple: taskId={} componentId={} stream={} values={}", taskId, + componentId, stream, values); } ArrayList outTasks = new ArrayList<>(); @@ -183,7 +197,8 @@ public List getOutgoingTasks(String stream, List values) { } public Tuple getTuple(String stream, List values) { - return new TupleImpl(systemTopologyContext, values, executor.getComponentId(), systemTopologyContext.getThisTaskId(), stream); + return new TupleImpl(systemTopologyContext, values, executor.getComponentId(), + systemTopologyContext.getThisTaskId(), stream); } public Integer getTaskId() { @@ -206,8 +221,10 @@ public TaskMetrics getTaskMetrics() { return taskMetrics; } - // Non Blocking call. If cannot emit to destination immediately, such tuples will be added to `pendingEmits` argument - public void sendUnanchored(String stream, List values, ExecutorTransfer transfer, Queue pendingEmits) { + // Non Blocking call. If cannot emit to destination immediately, such tuples will be added to + // `pendingEmits` argument + public void sendUnanchored(String stream, List values, ExecutorTransfer transfer, + Queue pendingEmits) { Tuple tuple = getTuple(stream, values); List tasks = getOutgoingTasks(stream, values); for (int i = 0; i < tasks.size(); i++) { @@ -218,8 +235,8 @@ public void sendUnanchored(String stream, List values, ExecutorTransfer /** * Sends an unanchored feedback tuple directly to a specific task ID (typically upstream). - *

    - * This method bypasses standard stream grouping logic and routes the tuple + * + *

    This method bypasses standard stream grouping logic and routes the tuple * exclusively to the provided {@code targetTaskId}. It is a non-blocking call: * if the destination buffer is full, the tuple is added to the {@code pendingEmits} * queue for later retry, preventing executor stalls. @@ -227,18 +244,22 @@ public void sendUnanchored(String stream, List values, ExecutorTransfer * * @param stream The ID of the stream to emit on (must be declared in the topology). * @param values The data payload to be sent. - * @param targetTaskId The unique ID of the destination task (e.g., the sourceTaskId of an incoming tuple). + * @param targetTaskId The unique ID of the destination task (e.g., the sourceTaskId of an + * incoming tuple). * @param transfer The {@link ExecutorTransfer} instance handling the physical data transfer. - * @param pendingEmits A queue used to store tuples that cannot be transferred immediately due to backpressure. + * @param pendingEmits A queue used to store tuples that cannot be transferred immediately due + * to backpressure. */ - public void sendUnanchoredFeedback(String stream, List values, int targetTaskId, ExecutorTransfer transfer, Queue pendingEmits) { + public void sendUnanchoredFeedback(String stream, List values, int targetTaskId, + ExecutorTransfer transfer, Queue pendingEmits) { Tuple tuple = getTuple(stream, values); AddressedTuple addressedTuple = new AddressedTuple(targetTaskId, tuple); transfer.tryTransfer(addressedTuple, pendingEmits); } /** - * Send sampled data to the eventlogger if the global or component level debug flag is set (via nimbus api). + * Send sampled data to the eventlogger if the global or component level debug flag is set (via + * nimbus api). */ public void sendToEventLogger(Executor executor, List values, String componentId, Object messageId, Random random, Queue overflow) { @@ -247,7 +268,8 @@ public void sendToEventLogger(Executor executor, List values, if (debugOptions == null) { debugOptions = componentDebug.get(executor.getStormId()); } - double spct = ((debugOptions != null) && (debugOptions.is_enable())) ? debugOptions.get_samplingpct() : 0; + double spct = ((debugOptions != null) && (debugOptions.is_enable())) ? debugOptions + .get_samplingpct() : 0; if (spct > 0 && (random.nextDouble() * 100) < spct) { sendUnanchored(StormCommon.EVENTLOGGER_STREAM_ID, new Values(componentId, messageId, System.currentTimeMillis(), values), @@ -317,7 +339,8 @@ private void addTaskHooks() { if (null != hooksClassList) { for (String hookClass : hooksClassList) { try { - userTopologyContext.addTaskHook(((ITaskHook) Class.forName(hookClass).newInstance())); + userTopologyContext.addTaskHook(((ITaskHook) Class.forName(hookClass) + .newInstance())); } catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) { throw new RuntimeException("Failed to add hook: " + hookClass, e); } diff --git a/storm-client/src/jvm/org/apache/storm/daemon/common/FileWatcher.java b/storm-client/src/jvm/org/apache/storm/daemon/common/FileWatcher.java index b9926f3800c..420f6faf479 100644 --- a/storm-client/src/jvm/org/apache/storm/daemon/common/FileWatcher.java +++ b/storm-client/src/jvm/org/apache/storm/daemon/common/FileWatcher.java @@ -44,7 +44,8 @@ public FileWatcher(final Path watchedFile, Callback callback) throws IOException this(watchedFile, callback, Collections.singletonList(ENTRY_MODIFY)); } - public FileWatcher(final Path watchedFile, Callback callback, List> kinds) throws IOException { + public FileWatcher(final Path watchedFile, Callback callback, + List> kinds) throws IOException { this.watchedFile = watchedFile; this.callback = callback; Path parent = watchedFile.getParent(); @@ -77,7 +78,8 @@ public void run() { return; } for (WatchEvent event : watchKey.pollEvents()) { - if (this.kinds.contains(event.kind()) && event.context().equals(watchedFile.getFileName())) { + if (this.kinds.contains(event.kind()) && event.context().equals(watchedFile + .getFileName())) { try { LOG.info("Event {} on {}; invoking callback", event.kind(), watchedFile); callback.run(); diff --git a/storm-client/src/jvm/org/apache/storm/daemon/metrics/BuiltinMetricsUtil.java b/storm-client/src/jvm/org/apache/storm/daemon/metrics/BuiltinMetricsUtil.java index 828d7ea7fed..71efd94b365 100644 --- a/storm-client/src/jvm/org/apache/storm/daemon/metrics/BuiltinMetricsUtil.java +++ b/storm-client/src/jvm/org/apache/storm/daemon/metrics/BuiltinMetricsUtil.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -20,14 +26,18 @@ import org.apache.storm.task.TopologyContext; public class BuiltinMetricsUtil { - public static void registerIconnectionServerMetric(Object server, Map topoConf, TopologyContext context) { + public static void registerIconnectionServerMetric(Object server, Map topoConf, + TopologyContext context) { if (server instanceof IStatefulObject) { - registerMetric("__recv-iconnection", new StateMetric((IStatefulObject) server), topoConf, context); + registerMetric("__recv-iconnection", new StateMetric((IStatefulObject) server), + topoConf, context); } } - public static void registerMetric(String name, IMetric metric, Map topoConf, TopologyContext context) { - int bucketSize = ((Number) topoConf.get(Config.TOPOLOGY_BUILTIN_METRICS_BUCKET_SIZE_SECS)).intValue(); + public static void registerMetric(String name, IMetric metric, Map topoConf, + TopologyContext context) { + int bucketSize = ((Number) topoConf.get(Config.TOPOLOGY_BUILTIN_METRICS_BUCKET_SIZE_SECS)) + .intValue(); context.registerMetric(name, metric, bucketSize); } } diff --git a/storm-client/src/jvm/org/apache/storm/daemon/metrics/ClientMetricsUtils.java b/storm-client/src/jvm/org/apache/storm/daemon/metrics/ClientMetricsUtils.java index 68e5bc77298..211f0221118 100644 --- a/storm-client/src/jvm/org/apache/storm/daemon/metrics/ClientMetricsUtils.java +++ b/storm-client/src/jvm/org/apache/storm/daemon/metrics/ClientMetricsUtils.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -41,7 +47,8 @@ public static Locale getMetricsReporterLocale(Map reporterConf) return null; } - public static TimeUnit getTimeUnitForConfig(Map reporterConf, String configName) { + public static TimeUnit getTimeUnitForConfig(Map reporterConf, + String configName) { String timeUnitString = ObjectReader.getString(reporterConf.get(configName), null); if (timeUnitString != null) { return TimeUnit.valueOf(timeUnitString); diff --git a/storm-client/src/jvm/org/apache/storm/daemon/supervisor/AdvancedFSOps.java b/storm-client/src/jvm/org/apache/storm/daemon/supervisor/AdvancedFSOps.java index d2a9ced1b49..d85c6e20d2b 100644 --- a/storm-client/src/jvm/org/apache/storm/daemon/supervisor/AdvancedFSOps.java +++ b/storm-client/src/jvm/org/apache/storm/daemon/supervisor/AdvancedFSOps.java @@ -69,7 +69,8 @@ public static AdvancedFSOps make(Map conf) { } /** - * Set directory permissions to (OWNER)RWX (GROUP)R-X (OTHER)--- On some systems that do not support this, it may become a noop. + * Set directory permissions to (OWNER)RWX (GROUP)R-X (OTHER)--- On some systems that do not + * support this, it may become a noop. * * @param dir the directory to change permissions on * @throws IOException on any error @@ -110,6 +111,7 @@ public void moveFile(File fromFile, File toFile) throws IOException { /** * Check whether supports atomic directory move. + * * @return true if an atomic directory move works, else false */ @Override @@ -138,7 +140,7 @@ public void copyDirectory(File fromDir, File toDir) throws IOException { */ @Override public void setupBlobPermissions(File path, String user) throws IOException { - //Normally this is a NOOP + // Normally this is a NOOP } /** @@ -146,12 +148,13 @@ public void setupBlobPermissions(File path, String user) throws IOException { * * @param path what to delete * @param user who to delete it as if doing it as someone else is supported - * @param logPrefix if an external process needs to be launched to delete the object what prefix to include in the logs + * @param logPrefix if an external process needs to be launched to delete the object what prefix + * to include in the logs * @throws IOException on any error. */ @Override public void deleteIfExists(File path, String user, String logPrefix) throws IOException { - //by default no need to do this as a different user + // by default no need to do this as a different user deleteIfExists(path); } @@ -169,7 +172,7 @@ public void deleteIfExists(File path) throws IOException { try { FileUtils.forceDelete(path); } catch (FileNotFoundException ignored) { - //ignore + // ignore } } } @@ -183,7 +186,7 @@ public void deleteIfExists(File path) throws IOException { */ @Override public void setupStormCodeDir(String user, File path) throws IOException { - //By default this is a NOOP + // By default this is a NOOP } /** @@ -195,7 +198,7 @@ public void setupStormCodeDir(String user, File path) throws IOException { */ @Override public void setupWorkerArtifactsDir(String user, File path) throws IOException { - //By default this is a NOOP + // By default this is a NOOP } /** @@ -208,7 +211,8 @@ public void setupWorkerArtifactsDir(String user, File path) throws IOException { * @throws IOException on any error */ @Override - public boolean doRequiredTopoFilesExist(Map conf, String topologyId) throws IOException { + public boolean doRequiredTopoFilesExist(Map conf, + String topologyId) throws IOException { return ClientSupervisorUtils.doRequiredTopoFilesExist(conf, topologyId); } @@ -235,7 +239,8 @@ public void forceMkdir(Path path) throws IOException { } @Override - public DirectoryStream newDirectoryStream(Path dir, DirectoryStream.Filter filter) throws IOException { + public DirectoryStream newDirectoryStream(Path dir, + DirectoryStream.Filter filter) throws IOException { return Files.newDirectoryStream(dir, filter); } @@ -357,7 +362,7 @@ public void createSymlink(File link, File target) throws IOException { LOG.debug("Creating symlink [{}] to [{}]", plink, ptarget); if (Files.exists(plink)) { if (Files.isSameFile(plink, ptarget)) { - //It already points where we want it to + // It already points where we want it to return; } FileUtils.forceDelete(link); @@ -372,7 +377,8 @@ private static class AdvancedRunAsUserFSOps extends AdvancedFSOps { AdvancedRunAsUserFSOps(Map conf) { super(conf); if (Utils.isOnWindows()) { - throw new UnsupportedOperationException("ERROR: Windows doesn't support running workers as different users yet"); + throw new UnsupportedOperationException("ERROR: Windows doesn't support running " + + "workers as different users yet"); } this.conf = conf; } @@ -380,7 +386,8 @@ private static class AdvancedRunAsUserFSOps extends AdvancedFSOps { @Override public void setupBlobPermissions(File path, String user) throws IOException { String logPrefix = "setup blob permissions for " + path; - ClientSupervisorUtils.processLauncherAndWait(conf, user, Arrays.asList("blob", path.toString()), null, logPrefix); + ClientSupervisorUtils.processLauncherAndWait(conf, user, Arrays.asList("blob", path + .toString()), null, logPrefix); } @Override @@ -433,13 +440,15 @@ private static class AdvancedWindowsFSOps extends AdvancedFSOps { AdvancedWindowsFSOps(Map conf) { super(conf); if (ObjectReader.getBoolean(conf.get(Config.SUPERVISOR_RUN_WORKER_AS_USER), false)) { - throw new RuntimeException("ERROR: Windows doesn't support running workers as different users yet"); + throw new RuntimeException("ERROR: Windows doesn't support running workers as " + + "different users yet"); } } @Override public void restrictDirectoryPermissions(File dir) throws IOException { - //NOOP, if windows gets support for run as user we will need to find a way to support this + // NOOP, if windows gets support for run as user we will need to find a way to support + // this } @Override diff --git a/storm-client/src/jvm/org/apache/storm/daemon/supervisor/ClientSupervisorUtils.java b/storm-client/src/jvm/org/apache/storm/daemon/supervisor/ClientSupervisorUtils.java index 9dee5e03dbf..d67391025bd 100644 --- a/storm-client/src/jvm/org/apache/storm/daemon/supervisor/ClientSupervisorUtils.java +++ b/storm-client/src/jvm/org/apache/storm/daemon/supervisor/ClientSupervisorUtils.java @@ -35,12 +35,14 @@ import org.slf4j.LoggerFactory; public class ClientSupervisorUtils { - //Worker launched through external commands, hence we count their exceptions toward shell exceptions + // Worker launched through external commands, hence we count their exceptions toward shell + // exceptions public static final Meter numWorkerLaunchExceptions = ShellUtils.numShellExceptions; private static final Logger LOG = LoggerFactory.getLogger(ClientSupervisorUtils.class); - static boolean doRequiredTopoFilesExist(Map conf, String stormId) throws IOException { + static boolean doRequiredTopoFilesExist(Map conf, + String stormId) throws IOException { String stormroot = ConfigUtils.supervisorStormDistRoot(conf, stormId); String stormcodepath = ConfigUtils.supervisorStormCodePath(stormroot); String stormconfpath = ConfigUtils.supervisorStormConfPath(stormroot); @@ -60,12 +62,14 @@ static boolean doRequiredTopoFilesExist(Map conf, String stormId return false; } - public static int processLauncherAndWait(Map conf, String user, List args, + public static int processLauncherAndWait(Map conf, String user, + List args, final Map environment, final String logPreFix) throws IOException { return processLauncherAndWait(conf, user, args, environment, logPreFix, null); } - public static int processLauncherAndWait(Map conf, String user, List args, + public static int processLauncherAndWait(Map conf, String user, + List args, final Map environment, final String logPreFix, File dir) throws IOException { int ret = 0; @@ -85,7 +89,8 @@ public static int processLauncherAndWait(Map conf, String user, return ret; } - public static Process processLauncher(Map conf, String user, List commandPrefix, List args, + public static Process processLauncher(Map conf, String user, + List commandPrefix, List args, Map environment, final String logPreFix, final ExitCodeCallback exitCodeCallback, File dir) throws IOException { if (StringUtils.isBlank(user)) { @@ -116,7 +121,8 @@ public static Process processLauncher(Map conf, String user, Lis * @param command the command to be executed in the new process * @param environment the environment to be applied to the process. Can be null. * @param logPrefix a prefix for log entries from the output of the process. Can be null. - * @param exitCodeCallback code to be called passing the exit code value when the process completes + * @param exitCodeCallback code to be called passing the exit code value when the process + * completes * @param dir the working directory of the new process * @return the new process */ @@ -166,7 +172,8 @@ public Long call() { return process; } - public static void setupStormCodeDir(Map conf, String user, String dir) throws IOException { + public static void setupStormCodeDir(Map conf, String user, + String dir) throws IOException { if (ObjectReader.getBoolean(conf.get(Config.SUPERVISOR_RUN_WORKER_AS_USER), false)) { String logPrefix = "Storm Code Dir Setup for " + dir; List commands = new ArrayList<>(); @@ -176,7 +183,8 @@ public static void setupStormCodeDir(Map conf, String user, Stri } } - public static void setupWorkerArtifactsDir(Map conf, String user, String dir) throws IOException { + public static void setupWorkerArtifactsDir(Map conf, String user, + String dir) throws IOException { if (ObjectReader.getBoolean(conf.get(Config.SUPERVISOR_RUN_WORKER_AS_USER), false)) { String logPrefix = "Worker Artifacts Setup for " + dir; List commands = new ArrayList<>(); diff --git a/storm-client/src/jvm/org/apache/storm/daemon/supervisor/ExitCodeCallback.java b/storm-client/src/jvm/org/apache/storm/daemon/supervisor/ExitCodeCallback.java index 54bf7fabcb0..b3793b52d6f 100644 --- a/storm-client/src/jvm/org/apache/storm/daemon/supervisor/ExitCodeCallback.java +++ b/storm-client/src/jvm/org/apache/storm/daemon/supervisor/ExitCodeCallback.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/daemon/supervisor/IAdvancedFSOps.java b/storm-client/src/jvm/org/apache/storm/daemon/supervisor/IAdvancedFSOps.java index ccb4a1b21d5..a5901ff51b6 100644 --- a/storm-client/src/jvm/org/apache/storm/daemon/supervisor/IAdvancedFSOps.java +++ b/storm-client/src/jvm/org/apache/storm/daemon/supervisor/IAdvancedFSOps.java @@ -30,7 +30,8 @@ public interface IAdvancedFSOps { /** - * Set directory permissions to (OWNER)RWX (GROUP)R-X (OTHER)--- On some systems that do not support this, it may become a noop. + * Set directory permissions to (OWNER)RWX (GROUP)R-X (OTHER)--- On some systems that do not + * support this, it may become a noop. * * @param dir the directory to change permissions on * @throws IOException on any error @@ -46,7 +47,6 @@ public interface IAdvancedFSOps { */ void moveDirectoryPreferAtomic(File fromDir, File toDir) throws IOException; - /** * Moves a file to a given destination. * @@ -58,6 +58,7 @@ public interface IAdvancedFSOps { /** * Check whether supports atomic directory move. + * * @return true if an atomic directory move works, else false */ boolean supportsAtomicDirectoryMove(); @@ -85,7 +86,8 @@ public interface IAdvancedFSOps { * * @param path what to delete * @param user who to delete it as if doing it as someone else is supported - * @param logPrefix if an external process needs to be launched to delete the object what prefix to include in the logs + * @param logPrefix if an external process needs to be launched to delete the object what prefix + * to include in the logs * @throws IOException on any error. */ void deleteIfExists(File path, String user, String logPrefix) throws IOException; @@ -125,7 +127,8 @@ public interface IAdvancedFSOps { * * @throws IOException on any error */ - boolean doRequiredTopoFilesExist(Map conf, String topologyId) throws IOException; + boolean doRequiredTopoFilesExist(Map conf, + String topologyId) throws IOException; /** * Makes a directory, including any necessary but nonexistent parent directories. @@ -152,7 +155,8 @@ public interface IAdvancedFSOps { * * @throws IOException on any error */ - DirectoryStream newDirectoryStream(Path dir, DirectoryStream.Filter filter) throws IOException; + DirectoryStream newDirectoryStream(Path dir, + DirectoryStream.Filter filter) throws IOException; /** * List the contents of a directory. diff --git a/storm-client/src/jvm/org/apache/storm/daemon/worker/BackPressureTracker.java b/storm-client/src/jvm/org/apache/storm/daemon/worker/BackPressureTracker.java index da3548bef75..39e2b2a2244 100644 --- a/storm-client/src/jvm/org/apache/storm/daemon/worker/BackPressureTracker.java +++ b/storm-client/src/jvm/org/apache/storm/daemon/worker/BackPressureTracker.java @@ -20,8 +20,8 @@ import com.codahale.metrics.Gauge; import java.util.ArrayList; -import java.util.Map; import java.util.Map.Entry; +import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; import org.apache.storm.messaging.netty.BackPressureStatus; @@ -88,7 +88,7 @@ public BackPressureStatus getCurrStatus() { ArrayList nonBpTasks = new ArrayList<>(tasks.size()); for (Entry entry : tasks.entrySet()) { - //System bolt is not a part of backpressure. + // System bolt is not a part of backpressure. if (entry.getKey() >= 0) { boolean backpressure = entry.getValue().backpressure.get(); if (backpressure) { @@ -109,16 +109,15 @@ public void setLastOverflowCount(BackpressureState state, int value) { state.lastOverflowCount = value; } - public static class BackpressureState { private final JCQueue queue; - //No task is under backpressure initially + // No task is under backpressure initially private final AtomicBoolean backpressure = new AtomicBoolean(false); - //The overflow count last time BP status was sent + // The overflow count last time BP status was sent private int lastOverflowCount = 0; - - BackpressureState(JCQueue queue, Integer taskId, String componentId, StormMetricRegistry metricRegistry) { + BackpressureState(JCQueue queue, Integer taskId, String componentId, + StormMetricRegistry metricRegistry) { this.queue = queue; // System bolt is not a part of backpressure. @@ -136,7 +135,8 @@ public Integer getValue() { return 0; } }; - metricRegistry.gauge("__backpressure-last-overflow-count", bpOverflowCount, componentId, taskId); + metricRegistry.gauge("__backpressure-last-overflow-count", bpOverflowCount, + componentId, taskId); } } diff --git a/storm-client/src/jvm/org/apache/storm/daemon/worker/LogConfigManager.java b/storm-client/src/jvm/org/apache/storm/daemon/worker/LogConfigManager.java index af169cf1efa..7edf3b416f8 100644 --- a/storm-client/src/jvm/org/apache/storm/daemon/worker/LogConfigManager.java +++ b/storm-client/src/jvm/org/apache/storm/daemon/worker/LogConfigManager.java @@ -60,7 +60,8 @@ public void processLogConfigChange(LogConfig logConfig) { Map newLogConfigs = new HashMap<>(); for (Map.Entry entry : loggers.entrySet()) { String msgLoggerName = entry.getKey(); - msgLoggerName = ("ROOT".equalsIgnoreCase(msgLoggerName)) ? LogManager.ROOT_LOGGER_NAME : msgLoggerName; + msgLoggerName = ("ROOT".equalsIgnoreCase(msgLoggerName)) + ? LogManager.ROOT_LOGGER_NAME : msgLoggerName; LogLevel loggerLevel = entry.getValue(); // the new-timeouts map now contains logger => timeout if (loggerLevel.is_set_reset_log_level_timeout_epoch()) { @@ -82,7 +83,8 @@ public void processLogConfigChange(LogConfig logConfig) { for (String loggerName : latestConf.descendingKeySet()) { if (!newLogConfigs.containsKey(loggerName)) { // if we had a timeout, but the timeout is no longer active - setLoggerLevel(logContext, loggerName, latestConf.get(loggerName).get_reset_log_level()); + setLoggerLevel(logContext, loggerName, latestConf.get(loggerName) + .get_reset_log_level()); } } @@ -92,7 +94,8 @@ public void processLogConfigChange(LogConfig logConfig) { // the merged configs are only for the reset logic for (String loggerName : new TreeSet<>(logConfig.get_named_logger_level().keySet())) { LogLevel logLevel = logConfig.get_named_logger_level().get(loggerName); - loggerName = ("ROOT".equalsIgnoreCase(loggerName)) ? LogManager.ROOT_LOGGER_NAME : loggerName; + loggerName = ("ROOT".equalsIgnoreCase(loggerName)) + ? LogManager.ROOT_LOGGER_NAME : loggerName; LogLevelAction action = logLevel.get_action(); if (action == LogLevelAction.UPDATE) { setLoggerLevel(logContext, loggerName, logLevel.get_target_log_level()); @@ -133,7 +136,8 @@ public void resetLogLevels() { } public Map getLoggerLevels() { - Configuration loggerConfig = ((LoggerContext) LogManager.getContext(false)).getConfiguration(); + Configuration loggerConfig = ((LoggerContext) LogManager.getContext(false)) + .getConfiguration(); Map logLevelMap = new HashMap<>(); for (Map.Entry entry : loggerConfig.getLoggers().entrySet()) { logLevelMap.put(entry.getKey(), entry.getValue().getLevel()); diff --git a/storm-client/src/jvm/org/apache/storm/daemon/worker/Worker.java b/storm-client/src/jvm/org/apache/storm/daemon/worker/Worker.java index aab416491e7..bfca922de0f 100644 --- a/storm-client/src/jvm/org/apache/storm/daemon/worker/Worker.java +++ b/storm-client/src/jvm/org/apache/storm/daemon/worker/Worker.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

    - * http://www.apache.org/licenses/LICENSE-2.0 - *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

    http://www.apache.org/licenses/LICENSE-2.0 + * + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -31,7 +37,6 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; import javax.security.auth.Subject; - import org.apache.storm.Config; import org.apache.storm.Constants; import org.apache.storm.cluster.ClusterStateContext; @@ -99,8 +104,10 @@ public class Worker implements Shutdownable, DaemonCommon { private final Supplier supervisorIfaceSupplier; /** - * TODO: should worker even take the topologyId as input? this should be deducible from cluster state (by searching through assignments) - * what about if there's inconsistency in assignments? -> but nimbus should guarantee this consistency. + * TODO: should worker even take the topologyId as input? this should be deducible from cluster + * state (by searching through assignments) + * what about if there's inconsistency in assignments? -> but nimbus should guarantee this + * consistency. * * @param conf - Storm configuration * @param context - @@ -110,7 +117,8 @@ public class Worker implements Shutdownable, DaemonCommon { * @param port - port on which the worker runs * @param workerId - worker id */ - public Worker(Map conf, IContext context, String topologyId, String assignmentId, + public Worker(Map conf, IContext context, String topologyId, + String assignmentId, int supervisorPort, int port, String workerId, Supplier supervisorIfaceSupplier) throws IOException { this.conf = conf; @@ -123,7 +131,8 @@ public Worker(Map conf, IContext context, String topologyId, Str this.logConfigManager = new LogConfigManager(); this.metricRegistry = new StormMetricRegistry(); - this.topologyConf = ConfigUtils.overrideLoginConfigWithSystemProperty(ConfigUtils.readSupervisorStormConf(conf, topologyId)); + this.topologyConf = ConfigUtils.overrideLoginConfigWithSystemProperty(ConfigUtils + .readSupervisorStormConf(conf, topologyId)); // See STORM-3728. // Writes to Pacemaker are currently always allowed. @@ -135,7 +144,8 @@ public Worker(Map conf, IContext context, String topologyId, Str this.supervisorIfaceSupplier = () -> { try { return SupervisorClient.Builder.withConf(topologyConf) - .withHostName(Utils.hostname()).withPort(supervisorPort).createSupervisorClient(); + .withHostName(Utils.hostname()).withPort(supervisorPort) + .createSupervisorClient(); } catch (UnknownHostException e) { throw Utils.wrapInRuntime(e); } @@ -145,13 +155,15 @@ public Worker(Map conf, IContext context, String topologyId, Str } } - public Worker(Map conf, IContext context, String topologyId, String assignmentId, + public Worker(Map conf, IContext context, String topologyId, + String assignmentId, int supervisorPort, int port, String workerId) throws IOException { this(conf, context, topologyId, assignmentId, supervisorPort, port, workerId, null); } public static void main(String[] args) throws Exception { - Preconditions.checkArgument(args.length == 5, "Illegal number of arguments. Expected: 5, Actual: " + args.length); + Preconditions.checkArgument(args.length == 5, + "Illegal number of arguments. Expected: 5, Actual: " + args.length); String stormId = args[0]; String assignmentId = args[1]; String supervisorPort = args[2]; @@ -161,11 +173,13 @@ public static void main(String[] args) throws Exception { Utils.setupWorkerUncaughtExceptionHandler(); StormCommon.validateDistributedMode(conf); int supervisorPortInt = Integer.parseInt(supervisorPort); - Worker worker = new Worker(conf, null, stormId, assignmentId, supervisorPortInt, Integer.parseInt(portStr), workerId); + Worker worker = new Worker(conf, null, stormId, assignmentId, supervisorPortInt, Integer + .parseInt(portStr), workerId); - //Add shutdown hooks before starting any other threads to avoid possible race condition - //between invoking shutdown hooks and registering shutdown hooks. See STORM-3658. - int workerShutdownSleepSecs = ObjectReader.getInt(conf.get(Config.SUPERVISOR_WORKER_SHUTDOWN_SLEEP_SECS)); + // Add shutdown hooks before starting any other threads to avoid possible race condition + // between invoking shutdown hooks and registering shutdown hooks. See STORM-3658. + int workerShutdownSleepSecs = ObjectReader.getInt(conf + .get(Config.SUPERVISOR_WORKER_SHUTDOWN_SLEEP_SECS)); LOG.info("Adding shutdown hook with kill in {} secs", workerShutdownSleepSecs); Utils.addShutdownHookWithDelayedForceKill(worker::shutdown, workerShutdownSleepSecs); @@ -173,7 +187,8 @@ public static void main(String[] args) throws Exception { } public void start() throws Exception { - LOG.info("Launching worker for {} on {}:{} with id {} and conf {}", topologyId, assignmentId, port, workerId, + LOG.info("Launching worker for {} on {}:{} with id {} and conf {}", topologyId, + assignmentId, port, workerId, ConfigUtils.maskPasswords(conf)); // because in local mode, its not a separate // process. supervisor will register it in this case @@ -183,13 +198,15 @@ public void start() throws Exception { SysOutOverSLF4J.sendSystemOutAndErrToSLF4J(); String pid = Utils.processPid(); FileUtils.touch(new File(ConfigUtils.workerPidPath(conf, workerId, pid))); - FileUtils.writeStringToFile(new File(ConfigUtils.workerArtifactsPidPath(conf, topologyId, port)), pid, + FileUtils.writeStringToFile(new File(ConfigUtils.workerArtifactsPidPath(conf, + topologyId, port)), pid, Charset.forName("UTF-8")); } ClusterStateContext csContext = new ClusterStateContext(DaemonType.WORKER, topologyConf); IStateStorage stateStorage = ClusterUtils.mkStateStorage(conf, topologyConf, csContext); - IStormClusterState stormClusterState = ClusterUtils.mkStormClusterState(stateStorage, null, csContext); + IStormClusterState stormClusterState = ClusterUtils.mkStormClusterState(stateStorage, null, + csContext); metricRegistry.start(topologyConf, port); SharedMetricRegistries.add(WORKER_METRICS_REGISTRY, metricRegistry.getRegistry()); @@ -212,10 +229,12 @@ private Object loadWorker(IStateStorage stateStorage, IStormClusterState stormCl Map initCreds, Credentials initialCredentials) throws Exception { workerState = - new WorkerState(conf, context, topologyId, assignmentId, supervisorIfaceSupplier, port, workerId, + new WorkerState(conf, context, topologyId, assignmentId, supervisorIfaceSupplier, port, + workerId, topologyConf, stateStorage, stormClusterState, autoCreds, metricRegistry, initialCredentials); - this.heatbeatMeter = metricRegistry.meter("doHeartbeat-calls", workerState.getWorkerTopologyContext(), + this.heatbeatMeter = metricRegistry.meter("doHeartbeat-calls", workerState + .getWorkerTopologyContext(), Constants.SYSTEM_COMPONENT_ID, (int) Constants.SYSTEM_TASK_ID); // Heartbeat here so that worker process dies if this fails @@ -225,16 +244,18 @@ private Object loadWorker(IStateStorage stateStorage, IStormClusterState stormCl executorsAtom = new AtomicReference<>(null); - // launch heartbeat threads immediately so that slow-loading tasks don't cause the worker to timeout + // launch heartbeat threads immediately so that slow-loading tasks don't cause the worker to + // timeout // to the supervisor workerState.heartbeatTimer - .scheduleRecurring(0, (Integer) conf.get(Config.WORKER_HEARTBEAT_FREQUENCY_SECS), () -> { - try { - doHeartBeat(); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); + .scheduleRecurring(0, (Integer) conf.get(Config.WORKER_HEARTBEAT_FREQUENCY_SECS), + () -> { + try { + doHeartBeat(); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); Integer execHeartBeatFreqSecs = workerState.stormClusterState.isPacemakerStateStore() ? (Integer) conf.get(Config.TASK_HEARTBEAT_FREQUENCY_SECS) @@ -257,12 +278,14 @@ private Object loadWorker(IStateStorage stateStorage, IStormClusterState stormCl Executor executor = LocalExecutor.mkExecutor(workerState, e, initCreds); execs.add(executor); for (int i = 0; i < executor.getTaskIds().size(); ++i) { - workerState.localReceiveQueues.put(executor.getTaskIds().get(i), executor.getReceiveQueue()); + workerState.localReceiveQueues.put(executor.getTaskIds().get(i), executor + .getReceiveQueue()); } } else { Executor executor = Executor.mkExecutor(workerState, e, initCreds); for (int i = 0; i < executor.getTaskIds().size(); ++i) { - workerState.localReceiveQueues.put(executor.getTaskIds().get(i), executor.getReceiveQueue()); + workerState.localReceiveQueues.put(executor.getTaskIds().get(i), executor + .getReceiveQueue()); } execs.add(executor); } @@ -287,17 +310,20 @@ private Object loadWorker(IStateStorage stateStorage, IStormClusterState stormCl final int[] credCheckErrCnt = new int[1]; // consecutive-error-count workerState.refreshCredentialsTimer.scheduleRecurring(0, - (Integer) conf.get(Config.TASK_CREDENTIALS_POLL_SECS), () -> { + (Integer) conf + .get(Config.TASK_CREDENTIALS_POLL_SECS), () -> { try { checkCredentialsChanged(); credCheckErrCnt[0] = 0; } catch (Exception ex) { credCheckErrCnt[0]++; if (credCheckErrCnt[0] <= credCheckMaxAllowed) { - LOG.warn("Ignoring {} of {} consecutive exceptions when checking for credential change", + LOG.warn("Ignoring {} of {} consecutive exceptions when checking for " + + "credential change", credCheckErrCnt[0], credCheckMaxAllowed, ex); } else { - LOG.error("Received {} consecutive exceptions, {} tolerated, when checking for credential change", + LOG.error("Received {} consecutive exceptions, {} tolerated, when " + + "checking for credential change", credCheckErrCnt[0], credCheckMaxAllowed, ex); throw ex; } @@ -317,36 +343,47 @@ private Object loadWorker(IStateStorage stateStorage, IStormClusterState stormCl } ); - // The jitter allows the clients to get the data at different times, and avoids thundering herd + // The jitter allows the clients to get the data at different times, and avoids thundering + // herd if (!(Boolean) topologyConf.get(Config.TOPOLOGY_DISABLE_LOADAWARE_MESSAGING)) { - workerState.refreshLoadTimer.scheduleRecurringWithJitter(0, 1, 500, Worker.this::doRefreshLoad); + workerState.refreshLoadTimer.scheduleRecurringWithJitter(0, 1, 500, + Worker.this::doRefreshLoad); } workerState.refreshConnectionsTimer.scheduleRecurring(0, - (Integer) conf.get(Config.TASK_REFRESH_POLL_SECS), + (Integer) conf + .get(Config.TASK_REFRESH_POLL_SECS), workerState::refreshConnections); workerState.resetLogLevelsTimer.scheduleRecurring(0, - (Integer) conf.get(Config.WORKER_LOG_LEVEL_RESET_POLL_SECS), + (Integer) conf + .get(Config.WORKER_LOG_LEVEL_RESET_POLL_SECS), logConfigManager::resetLogLevels); - workerState.refreshActiveTimer.scheduleRecurring(0, (Integer) conf.get(Config.TASK_REFRESH_POLL_SECS), + workerState.refreshActiveTimer.scheduleRecurring(0, (Integer) conf + .get(Config.TASK_REFRESH_POLL_SECS), workerState::refreshStormActive); setupFlushTupleTimer(topologyConf, newExecutors); setupBackPressureCheckTimer(topologyConf); LOG.info("Worker has topology config {}", ConfigUtils.maskPasswords(topologyConf)); - LOG.info("Worker {} for storm {} on {}:{} has finished loading", workerId, topologyId, assignmentId, port); + LOG.info("Worker {} for storm {} on {}:{} has finished loading", workerId, topologyId, + assignmentId, port); return this; } - private void setupFlushTupleTimer(final Map topologyConf, final List executors) { - final Integer producerBatchSize = ObjectReader.getInt(topologyConf.get(Config.TOPOLOGY_PRODUCER_BATCH_SIZE)); - final Integer xferBatchSize = ObjectReader.getInt(topologyConf.get(Config.TOPOLOGY_TRANSFER_BATCH_SIZE)); - final Long flushIntervalMillis = ObjectReader.getLong(topologyConf.get(Config.TOPOLOGY_BATCH_FLUSH_INTERVAL_MILLIS)); + private void setupFlushTupleTimer(final Map topologyConf, + final List executors) { + final Integer producerBatchSize = ObjectReader.getInt(topologyConf + .get(Config.TOPOLOGY_PRODUCER_BATCH_SIZE)); + final Integer xferBatchSize = ObjectReader.getInt(topologyConf + .get(Config.TOPOLOGY_TRANSFER_BATCH_SIZE)); + final Long flushIntervalMillis = ObjectReader.getLong(topologyConf + .get(Config.TOPOLOGY_BATCH_FLUSH_INTERVAL_MILLIS)); if ((producerBatchSize == 1 && xferBatchSize == 1) || flushIntervalMillis == 0) { - LOG.info("Flush Tuple generation disabled. producerBatchSize={}, xferBatchSize={}, flushIntervalMillis={}", + LOG.info("Flush Tuple generation disabled. producerBatchSize={}, xferBatchSize={}, " + + "flushIntervalMillis={}", producerBatchSize, xferBatchSize, flushIntervalMillis); return; } @@ -370,10 +407,13 @@ private void setupBackPressureCheckTimer(final Map topologyConf) LOG.info("BackPressure change checking is disabled as there is only one worker"); return; } - final Long bpCheckIntervalMs = ObjectReader.getLong(topologyConf.get(Config.TOPOLOGY_BACKPRESSURE_CHECK_MILLIS)); + final Long bpCheckIntervalMs = ObjectReader.getLong(topologyConf + .get(Config.TOPOLOGY_BACKPRESSURE_CHECK_MILLIS)); workerState.backPressureCheckTimer.scheduleRecurringMs(bpCheckIntervalMs, - bpCheckIntervalMs, () -> workerState.refreshBackPressureStatus()); - LOG.info("BackPressure status change checking will be performed every {} millis", bpCheckIntervalMs); + bpCheckIntervalMs, () -> workerState + .refreshBackPressureStatus()); + LOG.info("BackPressure status change checking will be performed every {} millis", + bpCheckIntervalMs); } public void doRefreshLoad() { @@ -387,8 +427,10 @@ public void doRefreshLoad() { public void doHeartBeat() throws IOException { LocalState state = ConfigUtils.workerState(workerState.conf, workerState.workerId); - LSWorkerHeartbeat lsWorkerHeartbeat = new LSWorkerHeartbeat(Time.currentTimeSecsLong(), workerState.topologyId, - workerState.localExecutors.stream() + LSWorkerHeartbeat lsWorkerHeartbeat = new LSWorkerHeartbeat(Time.currentTimeSecsLong(), + workerState.topologyId, + workerState.localExecutors + .stream() .map(executor -> new ExecutorInfo( executor.get(0).intValue(), executor.get(1).intValue())) @@ -414,10 +456,12 @@ public void doExecutorHeartbeats() { .toMap(IRunningExecutor::getExecutorId, IRunningExecutor::renderStats))); } - Map zkHb = ClientStatsUtil.mkZkWorkerHb(workerState.topologyId, stats, workerState.uptime.upTime()); + Map zkHb = ClientStatsUtil.mkZkWorkerHb(workerState.topologyId, stats, + workerState.uptime.upTime()); try { workerState.stormClusterState - .workerHeartbeat(workerState.topologyId, workerState.assignmentId, (long) workerState.port, + .workerHeartbeat(workerState.topologyId, workerState.assignmentId, + (long) workerState.port, ClientStatsUtil.thriftifyZkWorkerHb(zkHb)); } catch (Exception ex) { LOG.error("Worker failed to write heartbeats to ZK or Pacemaker...will retry", ex); @@ -427,9 +471,11 @@ public void doExecutorHeartbeats() { public Map getCurrentBlobVersions() throws IOException { Map results = new HashMap<>(); Map> blobstoreMap = - (Map>) workerState.getTopologyConf().get(Config.TOPOLOGY_BLOBSTORE_MAP); + (Map>) workerState.getTopologyConf() + .get(Config.TOPOLOGY_BLOBSTORE_MAP); if (blobstoreMap != null) { - String stormRoot = ConfigUtils.supervisorStormDistRoot(workerState.getTopologyConf(), workerState.getTopologyId()); + String stormRoot = ConfigUtils.supervisorStormDistRoot(workerState.getTopologyConf(), + workerState.getTopologyId()); for (Map.Entry> entry : blobstoreMap.entrySet()) { String localFileName = entry.getKey(); Map blobInfo = entry.getValue(); @@ -437,7 +483,8 @@ public Map getCurrentBlobVersions() throws IOException { localFileName = (String) blobInfo.get("localname"); } - String blobWithVersion = new File(stormRoot, localFileName).getCanonicalFile().getName(); + String blobWithVersion = new File(stormRoot, localFileName).getCanonicalFile() + .getName(); Matcher m = BLOB_VERSION_EXTRACTION.matcher(blobWithVersion); if (m.matches()) { results.put(localFileName, Long.valueOf(m.group(1))); @@ -457,7 +504,8 @@ public void checkCredentialsChanged() { Credentials newCreds = workerState.stormClusterState.credentials(topologyId, null); if (!ObjectUtils.equals(newCreds, this.workerState.getCredentials())) { // This does not have to be atomic, worst case we update when one is not needed - ClientAuthUtils.updateSubject(subject, autoCreds, (null == newCreds) ? null : newCreds.get_creds()); + ClientAuthUtils.updateSubject(subject, autoCreds, (null == newCreds) ? null : newCreds + .get_creds()); this.workerState.setCredentials(newCreds); for (IRunningExecutor executor : executorsAtom.get()) { executor.credentialsChanged(newCreds); @@ -483,19 +531,22 @@ private void heartbeatToMasterIfLocalbeatFail(LSWorkerHeartbeat lsWorkerHeartbea return; } - //In distributed mode, send heartbeat directly to master if local supervisor goes down. - SupervisorWorkerHeartbeat workerHeartbeat = new SupervisorWorkerHeartbeat(lsWorkerHeartbeat.get_topology_id(), - lsWorkerHeartbeat.get_executors(), - lsWorkerHeartbeat.get_time_secs()); + // In distributed mode, send heartbeat directly to master if local supervisor goes down. + SupervisorWorkerHeartbeat workerHeartbeat = new SupervisorWorkerHeartbeat(lsWorkerHeartbeat + .get_topology_id(), + lsWorkerHeartbeat + .get_executors(), + lsWorkerHeartbeat + .get_time_secs()); try (SupervisorIfaceFactory fac = supervisorIfaceSupplier.get()) { fac.getIface().sendSupervisorWorkerHeartbeat(workerHeartbeat); } catch (Exception tr1) { - //If any error/exception thrown, report directly to nimbus. + // If any error/exception thrown, report directly to nimbus. LOG.warn("Exception when send heartbeat to local supervisor", tr1.getMessage()); try (NimbusClient nimbusClient = NimbusClient.Builder.withConf(topologyConf).build()) { nimbusClient.getClient().sendSupervisorWorkerHeartbeat(workerHeartbeat); } catch (Exception tr2) { - //if any error/exception thrown, just ignore. + // if any error/exception thrown, just ignore. LOG.error("Exception when send heartbeat to master", tr2.getMessage()); } } @@ -508,7 +559,7 @@ public void shutdown() { if (workerState != null) { for (IConnection socket : workerState.cachedNodeToPortSocket.get().values()) { - //this will do best effort flushing since the linger period + // this will do best effort flushing since the linger period // was set on creation socket.close(); } @@ -549,7 +600,8 @@ public void shutdown() { LOG.info("Trigger any worker shutdown hooks"); workerState.runWorkerShutdownHooks(); - workerState.stormClusterState.removeWorkerHeartbeat(topologyId, assignmentId, (long) port); + workerState.stormClusterState.removeWorkerHeartbeat(topologyId, assignmentId, + (long) port); LOG.info("Disconnecting from storm cluster state context"); workerState.stormClusterState.disconnect(); workerState.stateStorage.close(); diff --git a/storm-client/src/jvm/org/apache/storm/daemon/worker/WorkerState.java b/storm-client/src/jvm/org/apache/storm/daemon/worker/WorkerState.java index 59aceb8d65e..3a677446e17 100644 --- a/storm-client/src/jvm/org/apache/storm/daemon/worker/WorkerState.java +++ b/storm-client/src/jvm/org/apache/storm/daemon/worker/WorkerState.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

    - * http://www.apache.org/licenses/LICENSE-2.0 - *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

    http://www.apache.org/licenses/LICENSE-2.0 + * + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -79,8 +85,8 @@ import org.apache.storm.utils.ObjectReader; import org.apache.storm.utils.SupervisorIfaceFactory; import org.apache.storm.utils.ThriftTopologyUtils; -import org.apache.storm.utils.Utils; import org.apache.storm.utils.Utils.SmartThread; +import org.apache.storm.utils.Utils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -184,13 +190,16 @@ public WorkerState(Map conf, this.stateStorage = stateStorage; this.stormClusterState = stormClusterState; this.localExecutors = - new HashSet<>(readWorkerExecutors(assignmentId, port, getLocalAssignment(this.stormClusterState, topologyId))); + new HashSet<>(readWorkerExecutors(assignmentId, port, + getLocalAssignment(this.stormClusterState, topologyId))); this.isWorkerActive = new CountDownLatch(1); this.isTopologyActive = new AtomicBoolean(false); this.stormComponentToDebug = new AtomicReference<>(); - this.topology = ConfigUtils.readSupervisorTopology(conf, topologyId, AdvancedFSOps.make(conf)); + this.topology = ConfigUtils.readSupervisorTopology(conf, topologyId, AdvancedFSOps + .make(conf)); this.taskToComponent = StormCommon.stormTaskInfo(topology, topologyConf); - this.executorReceiveQueueMap = mkReceiveQueueMap(topologyConf, localExecutors, taskToComponent); + this.executorReceiveQueueMap = mkReceiveQueueMap(topologyConf, localExecutors, + taskToComponent); this.localTaskIds = new ArrayList<>(); this.taskToExecutorQueue = new HashMap<>(); this.blobToLastKnownVersion = new ConcurrentHashMap<>(); @@ -208,8 +217,10 @@ public WorkerState(Map conf, for (String c : ThriftTopologyUtils.getComponentIds(systemTopology)) { Map streamToFields = new HashMap<>(); for (Map.Entry stream : - ThriftTopologyUtils.getComponentCommon(systemTopology, c).get_streams().entrySet()) { - streamToFields.put(stream.getKey(), new Fields(stream.getValue().get_output_fields())); + ThriftTopologyUtils.getComponentCommon(systemTopology, c).get_streams() + .entrySet()) { + streamToFields.put(stream.getKey(), new Fields(stream.getValue() + .get_output_fields())); } componentToStreamToFields.put(c, streamToFields); } @@ -226,15 +237,19 @@ public WorkerState(Map conf, this.loadMapping = new LoadMapping(); this.assignmentVersions = new AtomicReference<>(new HashMap<>()); this.outboundTasks = workerOutboundTasks(); - this.trySerializeLocal = topologyConf.containsKey(Config.TOPOLOGY_TESTING_ALWAYS_TRY_SERIALIZE) - && (Boolean) topologyConf.get(Config.TOPOLOGY_TESTING_ALWAYS_TRY_SERIALIZE); + this.trySerializeLocal = topologyConf + .containsKey(Config.TOPOLOGY_TESTING_ALWAYS_TRY_SERIALIZE) + && (Boolean) topologyConf + .get(Config.TOPOLOGY_TESTING_ALWAYS_TRY_SERIALIZE); if (trySerializeLocal) { - LOG.warn("WILL TRY TO SERIALIZE ALL TUPLES (Turn off {} for production", Config.TOPOLOGY_TESTING_ALWAYS_TRY_SERIALIZE); + LOG.warn("WILL TRY TO SERIALIZE ALL TUPLES (Turn off {} for production", + Config.TOPOLOGY_TESTING_ALWAYS_TRY_SERIALIZE); } int maxTaskId = getMaxTaskId(componentToSortedTasks); this.workerTransfer = new WorkerTransfer(this, topologyConf, maxTaskId); - this.bpTracker = new BackPressureTracker(workerId, taskToExecutorQueue, metricRegistry, taskToComponent); + this.bpTracker = new BackPressureTracker(workerId, taskToExecutorQueue, metricRegistry, + taskToComponent); this.deserializedWorkerHooks = deserializeWorkerHooks(); LOG.info("Registering IConnectionCallbacks for {}:{}", assignmentId, port); IConnectionCallback cb = new DeserializingConnectionCallback(topologyConf, @@ -389,9 +404,11 @@ public SmartThread makeTransferThread() { public void suicideIfLocalAssignmentsChanged(Assignment assignment) { boolean shouldHalt = false; if (assignment != null) { - Set> assignedExecutors = new HashSet<>(readWorkerExecutors(assignmentId, port, assignment)); + Set> assignedExecutors = new HashSet<>(readWorkerExecutors(assignmentId, + port, assignment)); if (!localExecutors.equals(assignedExecutors)) { - LOG.info("Found conflicting assignments. We shouldn't be alive!" + " Assigned: " + assignedExecutors + LOG.info("Found conflicting assignments. We shouldn't be alive!" + " Assigned: " + + assignedExecutors + ", Current: " + localExecutors); shouldHalt = true; } @@ -413,14 +430,16 @@ public void refreshConnections() { try { assignment = getLocalAssignment(stormClusterState, topologyId); } catch (Exception e) { - LOG.warn("Failed to read assignment. This should only happen when topology is shutting down.", e); + LOG.warn("Failed to read assignment. This should only happen when topology is " + + "shutting down.", e); } suicideIfLocalAssignmentsChanged(assignment); Set neededConnections = new HashSet<>(); Map newTaskToNodePort = new HashMap<>(); if (null != assignment) { - Map taskToNodePort = StormCommon.taskToNodeport(assignment.get_executor_node_port()); + Map taskToNodePort = StormCommon.taskToNodeport(assignment + .get_executor_node_port()); for (Map.Entry taskToNodePortEntry : taskToNodePort.entrySet()) { Integer task = taskToNodePortEntry.getKey(); if (outboundTasks.contains(task)) { @@ -434,7 +453,8 @@ public void refreshConnections() { final Set currentConnections = cachedNodeToPortSocket.get().keySet(); final Set newConnections = Sets.difference(neededConnections, currentConnections); - final Set removeConnections = Sets.difference(currentConnections, neededConnections); + final Set removeConnections = Sets.difference(currentConnections, + neededConnections); Map nodeHost = assignment != null ? assignment.get_node_host() : null; // Add new connections atomically @@ -444,7 +464,8 @@ public void refreshConnections() { next.put(nodeInfo, mqContext.connect( topologyId, - //nodeHost is not null here, as newConnections is only non-empty if assignment was not null above. + // nodeHost is not null here, as newConnections is only non-empty if + // assignment was not null above. nodeHost.get(nodeInfo.get_node()), // Host nodeInfo.get_port().iterator().next().intValue(), // Port workerTransfer.getRemoteBackPressureStatus())); @@ -504,7 +525,8 @@ public void refreshStormActive(Runnable callback) { } public void refreshLoad(List execs) { - Set remoteTasks = Sets.difference(new HashSet<>(outboundTasks), new HashSet<>(localTaskIds)); + Set remoteTasks = Sets.difference(new HashSet<>(outboundTasks), + new HashSet<>(localTaskIds)); Map localLoad = new HashMap<>(); for (IRunningExecutor exec : execs) { double receiveLoad = exec.getReceiveQueue().getQueueLoad(); @@ -512,7 +534,8 @@ public void refreshLoad(List execs) { } Map remoteLoad = new HashMap<>(); - cachedNodeToPortSocket.get().values().stream().forEach(conn -> remoteLoad.putAll(conn.getLoad(remoteTasks))); + cachedNodeToPortSocket.get().values().stream().forEach(conn -> remoteLoad.putAll(conn + .getLoad(remoteTasks))); loadMapping.setLocal(localLoad); loadMapping.setRemote(remoteLoad); @@ -523,7 +546,8 @@ public void refreshLoad(List execs) { } } - // checks if the tasks which had back pressure are now free again. if so, sends an update to other workers + // checks if the tasks which had back pressure are now free again. if so, sends an update to + // other workers public void refreshBackPressureStatus() { LOG.debug("Checking for change in Backpressure status on worker's tasks"); boolean bpSituationChanged = bpTracker.refreshBpTaskList(); @@ -534,7 +558,8 @@ public void refreshBackPressureStatus() { } /** - * we will wait all connections to be ready and then activate the spout/bolt when the worker bootup. + * We will wait all connections to be ready and then activate the spout/bolt when the worker + * bootup. */ public void activateWorkerWhenAllConnectionsReady() { int delaySecs = 0; @@ -542,17 +567,20 @@ public void activateWorkerWhenAllConnectionsReady() { refreshActiveTimer.schedule(delaySecs, () -> { if (areAllConnectionsReady()) { - LOG.info("All connections are ready for worker {}:{} with id {}", assignmentId, port, workerId); + LOG.info("All connections are ready for worker {}:{} with id {}", assignmentId, + port, workerId); isWorkerActive.countDown(); } else { - refreshActiveTimer.schedule(recurSecs, () -> activateWorkerWhenAllConnectionsReady(), false, 0); + refreshActiveTimer.schedule(recurSecs, + () -> activateWorkerWhenAllConnectionsReady(), false, 0); } } ); } /* Not a Blocking call. If cannot emit, will add 'tuple' to pendingEmits and return 'false'. 'pendingEmits' can be null */ - public boolean tryTransferRemote(AddressedTuple tuple, Queue pendingEmits, ITupleSerializer serializer) { + public boolean tryTransferRemote(AddressedTuple tuple, Queue pendingEmits, + ITupleSerializer serializer) { return workerTransfer.tryTransferRemote(tuple, pendingEmits, serializer); } @@ -564,7 +592,8 @@ public boolean tryFlushRemotes() { return workerTransfer.tryFlushRemotes(); } - // Receives msgs from remote workers and feeds them to local executors. If any receiving local executor is under Back Pressure, + // Receives msgs from remote workers and feeds them to local executors. If any receiving local + // executor is under Back Pressure, // informs other workers about back pressure situation. Runs in the NettyWorker thread. // Package-private for testing. void transferLocalBatch(ArrayList tupleBatch) { @@ -578,10 +607,13 @@ void transferLocalBatch(ArrayList tupleBatch) { continue; } - // 0- route control tuples to the control lane: un-batched, ahead of any data backlog, exempt from - // backpressure and overflow. If the lane is full the tuple is dropped and counted; control signals + // 0- route control tuples to the control lane: un-batched, ahead of any data backlog, + // exempt from + // backpressure and overflow. If the lane is full the tuple is dropped and counted; + // control signals // are periodic, so the next one arrives within its period. - if (queue.isControlLaneEnabled() && Constants.isControlStreamId(tuple.getTuple().getSourceStreamId())) { + if (queue.isControlLaneEnabled() && Constants.isControlStreamId(tuple.getTuple() + .getSourceStreamId())) { queue.tryPublishControl(tuple); continue; } @@ -601,12 +633,14 @@ void transferLocalBatch(ArrayList tupleBatch) { receiver.sendBackPressureStatus(bpTracker.getCurrStatus()); bpTracker.setLastOverflowCount(bpState, currOverflowCount); } else { - if (currOverflowCount - bpTracker.getLastOverflowCount(bpState) > RESEND_BACKPRESSURE_SIZE) { + if (currOverflowCount - bpTracker + .getLastOverflowCount(bpState) > RESEND_BACKPRESSURE_SIZE) { // resend BP status, in case prev notification was missed or reordered BackPressureStatus bpStatus = bpTracker.getCurrStatus(); receiver.sendBackPressureStatus(bpStatus); bpTracker.setLastOverflowCount(bpState, currOverflowCount); - LOG.debug("Re-sent BackPressure Status. OverflowCount = {}, BP Status ID = {}. ", currOverflowCount, bpStatus.id); + LOG.debug("Re-sent BackPressure Status. OverflowCount = {}, BP Status ID = " + + "{}. ", currOverflowCount, bpStatus.id); } } @@ -620,13 +654,15 @@ private void dropMessage(AddressedTuple tuple, JCQueue queue) { ++dropCount; queue.recordMsgDrop(); LOG.warn( - "Dropping message as overflow threshold has reached for Q = {}. OverflowCount = {}. Total Drop Count= {}, Dropped Message : {}", + "Dropping message as overflow threshold has reached for Q = {}. OverflowCount = {}. " + + "Total Drop Count= {}, Dropped Message : {}", queue.getQueueName(), queue.getOverflowCount(), dropCount, tuple); } private void dropMessage(AddressedTuple tuple) { ++dropCount; - LOG.warn("Dropping message for unknown task {}. Total Drop Count = {}, Dropped Message : {}", + LOG.warn("Dropping message for unknown task {}. Total Drop Count = {}, Dropped Message : " + + "{}", tuple.dest, dropCount, tuple); } @@ -638,9 +674,11 @@ public void checkSerialize(KryoTupleSerializer serializer, AddressedTuple tuple) public final WorkerTopologyContext getWorkerTopologyContext() { try { - String codeDir = ConfigUtils.supervisorStormResourcesPath(ConfigUtils.supervisorStormDistRoot(conf, topologyId)); + String codeDir = ConfigUtils.supervisorStormResourcesPath(ConfigUtils + .supervisorStormDistRoot(conf, topologyId)); String pidDir = ConfigUtils.workerPidsRoot(conf, topologyId); - return new WorkerTopologyContext(systemTopology, topologyConf, taskToComponent, componentToSortedTasks, + return new WorkerTopologyContext(systemTopology, topologyConf, taskToComponent, + componentToSortedTasks, componentToStreamToFields, topologyId, codeDir, pidDir, port, localTaskIds, defaultSharedResources, userSharedResources, cachedTaskToNodePort, assignmentId, cachedNodeToHost); @@ -651,9 +689,11 @@ public final WorkerTopologyContext getWorkerTopologyContext() { public final WorkerUserContext getWorkerUserContext() { try { - String codeDir = ConfigUtils.supervisorStormResourcesPath(ConfigUtils.supervisorStormDistRoot(conf, topologyId)); + String codeDir = ConfigUtils.supervisorStormResourcesPath(ConfigUtils + .supervisorStormDistRoot(conf, topologyId)); String pidDir = ConfigUtils.workerPidsRoot(conf, topologyId); - return new WorkerUserContext(systemTopology, topologyConf, taskToComponent, componentToSortedTasks, + return new WorkerUserContext(systemTopology, topologyConf, taskToComponent, + componentToSortedTasks, componentToStreamToFields, topologyId, codeDir, pidDir, port, localTaskIds, defaultSharedResources, userSharedResources, cachedTaskToNodePort, assignmentId, cachedNodeToHost); @@ -689,7 +729,8 @@ public void runWorkerShutdownHooks() { public void closeResources() { LOG.info("Shutting down default resources"); - ((ExecutorService) defaultSharedResources.get(WorkerTopologyContext.SHARED_EXECUTOR)).shutdownNow(); + ((ExecutorService) defaultSharedResources.get(WorkerTopologyContext.SHARED_EXECUTOR)) + .shutdownNow(); LOG.info("Shut down default resources"); } @@ -713,14 +754,16 @@ public void setCredentials(Credentials credentials) { this.credentialsAtom.set(credentials); } - private List> readWorkerExecutors(String assignmentId, int port, Assignment assignment) { + private List> readWorkerExecutors(String assignmentId, int port, + Assignment assignment) { List> executorsAssignedToThisWorker = new ArrayList<>(); executorsAssignedToThisWorker.add(Constants.SYSTEM_EXECUTOR_ID); Map, NodeInfo> executorToNodePort = assignment.get_executor_node_port(); for (Map.Entry, NodeInfo> entry : executorToNodePort.entrySet()) { NodeInfo nodeInfo = entry.getValue(); - if (nodeInfo.get_node().equals(assignmentId) && nodeInfo.get_port().iterator().next() == port) { + if (nodeInfo.get_node().equals(assignmentId) && nodeInfo.get_port().iterator() + .next() == port) { executorsAssignedToThisWorker.add(entry.getKey()); } } @@ -731,7 +774,7 @@ private Assignment getLocalAssignment(IStormClusterState stormClusterState, Stri try (SupervisorIfaceFactory fac = supervisorIfaceSupplier.get()) { return fac.getIface().getLocalAssignmentForStorm(topologyId); } catch (Throwable e) { - //if any error/exception thrown, fetch it from zookeeper + // if any error/exception thrown, fetch it from zookeeper Assignment assignment = stormClusterState.remoteAssignmentInfo(topologyId, null); if (assignment == null) { throw new RuntimeException("Failed to read worker assignment." @@ -743,23 +786,32 @@ private Assignment getLocalAssignment(IStormClusterState stormClusterState, Stri private Map, JCQueue> mkReceiveQueueMap(Map topologyConf, Set> executors, Map taskToComponent) { - Integer recvQueueSize = ObjectReader.getInt(topologyConf.get(Config.TOPOLOGY_EXECUTOR_RECEIVE_BUFFER_SIZE)); - Integer recvBatchSize = ObjectReader.getInt(topologyConf.get(Config.TOPOLOGY_PRODUCER_BATCH_SIZE)); - boolean dynamicBatch = ObjectReader.getBoolean(topologyConf.get(Config.TOPOLOGY_PRODUCER_BATCH_DYNAMIC), false); - Integer overflowLimit = ObjectReader.getInt(topologyConf.get(Config.TOPOLOGY_EXECUTOR_OVERFLOW_LIMIT)); + Integer recvQueueSize = ObjectReader.getInt(topologyConf + .get(Config.TOPOLOGY_EXECUTOR_RECEIVE_BUFFER_SIZE)); + Integer recvBatchSize = ObjectReader.getInt(topologyConf + .get(Config.TOPOLOGY_PRODUCER_BATCH_SIZE)); + boolean dynamicBatch = ObjectReader.getBoolean(topologyConf + .get(Config.TOPOLOGY_PRODUCER_BATCH_DYNAMIC), false); + Integer overflowLimit = ObjectReader.getInt(topologyConf + .get(Config.TOPOLOGY_EXECUTOR_OVERFLOW_LIMIT)); boolean controlQueueEnable = - ObjectReader.getBoolean(topologyConf.get(Config.TOPOLOGY_EXECUTOR_RECEIVE_CONTROL_QUEUE_ENABLE), false); + ObjectReader.getBoolean(topologyConf + .get(Config.TOPOLOGY_EXECUTOR_RECEIVE_CONTROL_QUEUE_ENABLE), false); int controlQueueSize = controlQueueEnable - ? ObjectReader.getInt(topologyConf.get(Config.TOPOLOGY_EXECUTOR_RECEIVE_CONTROL_BUFFER_SIZE), 1024) + ? ObjectReader.getInt(topologyConf + .get(Config.TOPOLOGY_EXECUTOR_RECEIVE_CONTROL_BUFFER_SIZE), 1024) : 0; if (recvBatchSize > recvQueueSize / 2) { - throw new IllegalArgumentException(Config.TOPOLOGY_PRODUCER_BATCH_SIZE + ":" + recvBatchSize - + " is greater than half of " + Config.TOPOLOGY_EXECUTOR_RECEIVE_BUFFER_SIZE + ":" + throw new IllegalArgumentException(Config.TOPOLOGY_PRODUCER_BATCH_SIZE + ":" + + recvBatchSize + + " is greater than half of " + Config.TOPOLOGY_EXECUTOR_RECEIVE_BUFFER_SIZE + + ":" + recvQueueSize); } - IWaitStrategy backPressureWaitStrategy = IWaitStrategy.createBackPressureWaitStrategy(topologyConf); + IWaitStrategy backPressureWaitStrategy = IWaitStrategy + .createBackPressureWaitStrategy(topologyConf); Map, JCQueue> receiveQueueMap = new HashMap<>(); for (List executor : executors) { @@ -771,17 +823,21 @@ private Map, JCQueue> mkReceiveQueueMap(Map topologyC } else { compId = taskToComponent.get(taskId); } - receiveQueueMap.put(executor, new JCQueue("receive-queue" + executor.toString(), "receive-queue", + receiveQueueMap.put(executor, new JCQueue("receive-queue" + executor.toString(), + "receive-queue", recvQueueSize, overflowLimit, recvBatchSize, backPressureWaitStrategy, - this.getTopologyId(), compId, taskIds, this.getPort(), metricRegistry, dynamicBatch, controlQueueSize)); + this.getTopologyId(), compId, taskIds, this + .getPort(), metricRegistry, dynamicBatch, controlQueueSize)); } return receiveQueueMap; } private Map makeDefaultResources() { - int threadPoolSize = ObjectReader.getInt(conf.get(Config.TOPOLOGY_WORKER_SHARED_THREAD_POOL_SIZE)); - return ImmutableMap.of(WorkerTopologyContext.SHARED_EXECUTOR, Executors.newFixedThreadPool(threadPoolSize)); + int threadPoolSize = ObjectReader.getInt(conf + .get(Config.TOPOLOGY_WORKER_SHARED_THREAD_POOL_SIZE)); + return ImmutableMap.of(WorkerTopologyContext.SHARED_EXECUTOR, Executors + .newFixedThreadPool(threadPoolSize)); } private Map makeUserResources() { @@ -797,20 +853,23 @@ private StormTimer mkHaltingTimer(String name) { /** * Get worker outbound tasks. + * * @return seq of task ids that receive messages from this worker */ private Set workerOutboundTasks() { WorkerTopologyContext context = getWorkerTopologyContext(); Set components = new HashSet<>(); for (Integer taskId : localTaskIds) { - for (Map value : context.getTargets(context.getComponentId(taskId)).values()) { + for (Map value : context.getTargets(context.getComponentId(taskId)) + .values()) { components.addAll(value.keySet()); } } Set outboundTasks = new HashSet<>(); - for (Map.Entry> entry : Utils.reverseMap(taskToComponent).entrySet()) { + for (Map.Entry> entry : Utils.reverseMap(taskToComponent) + .entrySet()) { if (components.contains(entry.getKey())) { outboundTasks.addAll(entry.getValue()); } @@ -824,15 +883,18 @@ public Set getOutboundTasks() { /** * Check if this worker has remote outbound tasks. + * * @return true if this worker has remote outbound tasks; false otherwise. */ public boolean hasRemoteOutboundTasks() { - Set remoteTasks = Sets.difference(new HashSet<>(outboundTasks), new HashSet<>(localTaskIds)); + Set remoteTasks = Sets.difference(new HashSet<>(outboundTasks), + new HashSet<>(localTaskIds)); return !remoteTasks.isEmpty(); } /** * If all the tasks are local tasks, the topology has only one worker. + * * @return true if this worker is the single worker; false otherwise. */ public boolean isSingleWorker() { diff --git a/storm-client/src/jvm/org/apache/storm/daemon/worker/WorkerTransfer.java b/storm-client/src/jvm/org/apache/storm/daemon/worker/WorkerTransfer.java index 40e75aba0f4..5897106a268 100644 --- a/storm-client/src/jvm/org/apache/storm/daemon/worker/WorkerTransfer.java +++ b/storm-client/src/jvm/org/apache/storm/daemon/worker/WorkerTransfer.java @@ -32,8 +32,8 @@ import org.apache.storm.utils.JCQueue; import org.apache.storm.utils.ObjectReader; import org.apache.storm.utils.TransferDrainer; -import org.apache.storm.utils.Utils; import org.apache.storm.utils.Utils.SmartThread; +import org.apache.storm.utils.Utils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -50,7 +50,8 @@ public class WorkerTransfer implements JCQueue.Consumer { private final AtomicBoolean[] remoteBackPressureStatus; // [[remoteTaskId] -> true/false : indicates if remote task is under BP. - public WorkerTransfer(WorkerState workerState, Map topologyConf, int maxTaskIdInTopo) { + public WorkerTransfer(WorkerState workerState, Map topologyConf, + int maxTaskIdInTopo) { this.workerState = workerState; this.backPressureWaitStrategy = IWaitStrategy.createBackPressureWaitStrategy(topologyConf); this.drainer = new TransferDrainer(); @@ -59,16 +60,21 @@ public WorkerTransfer(WorkerState workerState, Map topologyConf, remoteBackPressureStatus[i] = new AtomicBoolean(false); } - Integer xferQueueSz = ObjectReader.getInt(topologyConf.get(Config.TOPOLOGY_TRANSFER_BUFFER_SIZE)); - Integer xferBatchSz = ObjectReader.getInt(topologyConf.get(Config.TOPOLOGY_TRANSFER_BATCH_SIZE)); + Integer xferQueueSz = ObjectReader.getInt(topologyConf + .get(Config.TOPOLOGY_TRANSFER_BUFFER_SIZE)); + Integer xferBatchSz = ObjectReader.getInt(topologyConf + .get(Config.TOPOLOGY_TRANSFER_BATCH_SIZE)); if (xferBatchSz > xferQueueSz / 2) { - throw new IllegalArgumentException(Config.TOPOLOGY_TRANSFER_BATCH_SIZE + ":" + xferBatchSz + " must be no more than half of " - + Config.TOPOLOGY_TRANSFER_BUFFER_SIZE + ":" + xferQueueSz); + throw new IllegalArgumentException(Config.TOPOLOGY_TRANSFER_BATCH_SIZE + ":" + + xferBatchSz + " must be no more than half of " + + Config.TOPOLOGY_TRANSFER_BUFFER_SIZE + ":" + + xferQueueSz); } this.transferQueue = new JCQueue("worker-transfer-queue", "worker-transfer-queue", xferQueueSz, 0, xferBatchSz, backPressureWaitStrategy, - workerState.getTopologyId(), Constants.SYSTEM_COMPONENT_ID, Collections.singletonList(-1), workerState.getPort(), + workerState.getTopologyId(), Constants.SYSTEM_COMPONENT_ID, Collections + .singletonList(-1), workerState.getPort(), workerState.getMetricRegistry()); } @@ -100,7 +106,8 @@ public void flush() throws InterruptedException { ReentrantReadWriteLock.ReadLock readLock = workerState.endpointSocketLock.readLock(); try { readLock.lock(); - drainer.send(workerState.cachedTaskToNodePort.get(), workerState.cachedNodeToPortSocket.get()); + drainer.send(workerState.cachedTaskToNodePort.get(), workerState.cachedNodeToPortSocket + .get()); } finally { readLock.unlock(); } @@ -108,14 +115,16 @@ public void flush() throws InterruptedException { } /* Not a Blocking call. If cannot emit, will add 'tuple' to 'pendingEmits' and return 'false'. 'pendingEmits' can be null */ - public boolean tryTransferRemote(AddressedTuple addressedTuple, Queue pendingEmits, ITupleSerializer serializer) { + public boolean tryTransferRemote(AddressedTuple addressedTuple, + Queue pendingEmits, ITupleSerializer serializer) { if (pendingEmits != null && !pendingEmits.isEmpty()) { pendingEmits.add(addressedTuple); return false; } if (!remoteBackPressureStatus[addressedTuple.dest].get()) { - TaskMessage tm = new TaskMessage(addressedTuple.getDest(), serializer.serialize(addressedTuple.getTuple())); + TaskMessage tm = new TaskMessage(addressedTuple.getDest(), serializer + .serialize(addressedTuple.getTuple())); if (transferQueue.tryPublish(tm)) { return true; } @@ -136,7 +145,6 @@ public boolean tryFlushRemotes() { return transferQueue.tryFlush(); } - public void haltTransferThd() { transferQueue.close(); } diff --git a/storm-client/src/jvm/org/apache/storm/dependency/DependencyBlobStoreUtils.java b/storm-client/src/jvm/org/apache/storm/dependency/DependencyBlobStoreUtils.java index f692323b9ad..b3774de5877 100644 --- a/storm-client/src/jvm/org/apache/storm/dependency/DependencyBlobStoreUtils.java +++ b/storm-client/src/jvm/org/apache/storm/dependency/DependencyBlobStoreUtils.java @@ -34,9 +34,12 @@ public static String generateDependencyBlobKey(String key) { } /** - * Tell whether a blob key names a topology dependency, i.e. whether it could have been produced by - * {@link #generateDependencyBlobKey(String)}. Keys that a topology only refers to, rather than owns, must be - * checked with this before they are acted upon, because the dependency lists of a submitted topology are filled + * Tell whether a blob key names a topology dependency, i.e. whether it could have been produced + * by + * {@link #generateDependencyBlobKey(String)}. Keys that a topology only refers to, rather than + * owns, must be + * checked with this before they are acted upon, because the dependency lists of a submitted + * topology are filled * in by the client and can name any blob at all. * * @param key the blob key to check, may be null diff --git a/storm-client/src/jvm/org/apache/storm/dependency/DependencyPropertiesParser.java b/storm-client/src/jvm/org/apache/storm/dependency/DependencyPropertiesParser.java index 971a3c124c8..f199834a24d 100644 --- a/storm-client/src/jvm/org/apache/storm/dependency/DependencyPropertiesParser.java +++ b/storm-client/src/jvm/org/apache/storm/dependency/DependencyPropertiesParser.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/dependency/DependencyUploader.java b/storm-client/src/jvm/org/apache/storm/dependency/DependencyUploader.java index cf89990c113..67bb74581a8 100644 --- a/storm-client/src/jvm/org/apache/storm/dependency/DependencyUploader.java +++ b/storm-client/src/jvm/org/apache/storm/dependency/DependencyUploader.java @@ -52,11 +52,12 @@ public class DependencyUploader { public DependencyUploader() { conf = Utils.readStormConfig(); - this.uploadChunkSize = ObjectReader.getInt(conf.get(Config.STORM_BLOBSTORE_DEPENDENCY_JAR_UPLOAD_CHUNK_SIZE_BYTES), 1024 * 1024); + this.uploadChunkSize = ObjectReader.getInt(conf + .get(Config.STORM_BLOBSTORE_DEPENDENCY_JAR_UPLOAD_CHUNK_SIZE_BYTES), 1024 * 1024); } public void init() { - //NOOP + // NOOP } public void shutdown() { @@ -77,14 +78,17 @@ void setBlobStore(ClientBlobStore blobStore) { this.blobStore = blobStore; } - public List uploadFiles(List dependencies, boolean cleanupIfFails) throws IOException, AuthorizationException { + public List uploadFiles(List dependencies, + boolean cleanupIfFails) throws IOException, AuthorizationException { checkFilesExist(dependencies); List keys = new ArrayList<>(dependencies.size()); try { for (File dependency : dependencies) { String fileName = dependency.getName(); - String key = DependencyBlobStoreUtils.generateDependencyBlobKey(DependencyBlobStoreUtils.applyUUIDToFileName(fileName)); + String key = DependencyBlobStoreUtils + .generateDependencyBlobKey(DependencyBlobStoreUtils + .applyUUIDToFileName(fileName)); try { uploadDependencyToBlobStore(key, dependency); @@ -131,7 +135,8 @@ public List uploadArtifacts(Map artifacts) { keys.add(key); } } catch (Throwable e) { - // the keys are unique to this upload and no topology refers to them, so the ones that made it to + // the keys are unique to this upload and no topology refers to them, so the ones that + // made it to // the blob store are only reachable from here if (getBlobStore() != null) { deleteBlobs(keys); @@ -161,11 +166,13 @@ private boolean uploadDependencyToBlobStore(String key, File dependency) boolean uploadNew = false; try { - // FIXME: we can filter by listKeys() with local blobstore when STORM-1986 is going to be resolved + // FIXME: we can filter by listKeys() with local blobstore when STORM-1986 is going to + // be resolved // as a workaround, we call getBlobMeta() for all keys getBlobStore().getBlobMeta(key); } catch (KeyNotFoundException e) { - // set acl to below so that it can be shared by other users as well, but allows only read + // set acl to below so that it can be shared by other users as well, but allows only + // read List acls = new ArrayList<>(); acls.add(new AccessControl(AccessControlType.USER, BlobStoreAclHandler.READ | BlobStoreAclHandler.WRITE | BlobStoreAclHandler.ADMIN)); diff --git a/storm-client/src/jvm/org/apache/storm/dependency/FileNotAvailableException.java b/storm-client/src/jvm/org/apache/storm/dependency/FileNotAvailableException.java index 86959b31bc8..22e41fd8d61 100644 --- a/storm-client/src/jvm/org/apache/storm/dependency/FileNotAvailableException.java +++ b/storm-client/src/jvm/org/apache/storm/dependency/FileNotAvailableException.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

    - * http://www.apache.org/licenses/LICENSE-2.0 - *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

    http://www.apache.org/licenses/LICENSE-2.0 + * + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/drpc/DRPCInvocationsClient.java b/storm-client/src/jvm/org/apache/storm/drpc/DRPCInvocationsClient.java index c4ca44aa7d6..21774b7464d 100644 --- a/storm-client/src/jvm/org/apache/storm/drpc/DRPCInvocationsClient.java +++ b/storm-client/src/jvm/org/apache/storm/drpc/DRPCInvocationsClient.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -28,11 +34,13 @@ @SuppressWarnings("checkstyle:AbbreviationAsWordInName") public class DRPCInvocationsClient extends ThriftClient implements DistributedRPCInvocations.Iface { public static final Logger LOG = LoggerFactory.getLogger(DRPCInvocationsClient.class); - private final AtomicReference client = new AtomicReference<>(); + private final AtomicReference client = + new AtomicReference<>(); private String host; private int port; - public DRPCInvocationsClient(Map conf, String host, int port) throws TTransportException { + public DRPCInvocationsClient(Map conf, String host, + int port) throws TTransportException { super(conf, ThriftConnectionType.DRPC_INVOCATIONS, host, port, null); this.host = host; this.port = port; @@ -111,7 +119,8 @@ public DistributedRPCInvocations.Client getClient() { } @Override - public void failRequestV2(String id, DRPCExecutionException ex) throws AuthorizationException, TException { + public void failRequestV2(String id, + DRPCExecutionException ex) throws AuthorizationException, TException { DistributedRPCInvocations.Client c = client.get(); try { if (c == null) { diff --git a/storm-client/src/jvm/org/apache/storm/drpc/DRPCSpout.java b/storm-client/src/jvm/org/apache/storm/drpc/DRPCSpout.java index ccd52c34937..7faff8973cd 100644 --- a/storm-client/src/jvm/org/apache/storm/drpc/DRPCSpout.java +++ b/storm-client/src/jvm/org/apache/storm/drpc/DRPCSpout.java @@ -50,7 +50,7 @@ @SuppressWarnings("checkstyle:AbbreviationAsWordInName") public class DRPCSpout extends BaseRichSpout { public static final Logger LOG = LoggerFactory.getLogger(DRPCSpout.class); - //ANY CHANGE TO THIS CODE MUST BE SERIALIZABLE COMPATIBLE OR THERE WILL BE PROBLEMS + // ANY CHANGE TO THIS CODE MUST BE SERIALIZABLE COMPATIBLE OR THERE WILL BE PROBLEMS private static final long serialVersionUID = 2387848310969237877L; private static final int clientConstructionRetryIntervalSec = 120; private final String function; @@ -69,7 +69,6 @@ public DRPCSpout(String function) { } } - public DRPCSpout(String function, ILocalDRPC drpc) { this.function = function; localDrpcId = drpc.getServiceId(); @@ -83,7 +82,8 @@ private void reconnectAsync(final DRPCInvocationsClient client) { String remote = client.getHost(); CompletableFuture future = futuresMap.get(remote); if (future.isDone()) { - LOG.warn("DRPCInvocationsClient [{}:{}] connection failed, no pending reconnection. Try reconnecting...", + LOG.warn("DRPCInvocationsClient [{}:{}] connection failed, no pending reconnection. " + + "Try reconnecting...", client.getHost(), client.getPort()); CompletableFuture newFuture = CompletableFuture.runAsync(() -> { @@ -104,14 +104,15 @@ private void reconnectAsync(final DRPCInvocationsClient client) { private void reconnectSync(DRPCInvocationsClient client) { try { LOG.info("reconnecting... "); - client.reconnectClient(); //Blocking call + client.reconnectClient(); // Blocking call } catch (TException e2) { LOG.error("Failed to connect to DRPC server", e2); } } @Override - public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { + public void open(Map conf, TopologyContext context, + SpoutOutputCollector collector) { this.collector = collector; if (localDrpcId == null) { background = new ExtendedThreadPoolExecutor(0, Integer.MAX_VALUE, @@ -164,7 +165,7 @@ public void close() { @Override public void nextTuple() { if (localDrpcId == null) { - //This will only ever grow and at least one client has been up + // This will only ever grow and at least one client has been up for (int i = 0; i < clients.size(); i++) { DRPCInvocationsClient client = clients.get(i); if (!client.isConnected()) { @@ -178,7 +179,8 @@ public void nextTuple() { returnInfo.put("id", req.get_request_id()); returnInfo.put("host", client.getHost()); returnInfo.put("port", client.getPort()); - collector.emit(new Values(req.get_func_args(), JSONValue.toJSONString(returnInfo)), + collector.emit(new Values(req.get_func_args(), JSONValue + .toJSONString(returnInfo)), new DRPCMessageId(req.get_request_id(), i)); break; } @@ -193,7 +195,8 @@ public void nextTuple() { } } } else { - DistributedRPCInvocations.Iface drpc = (DistributedRPCInvocations.Iface) ServiceRegistry.getService(localDrpcId); + DistributedRPCInvocations.Iface drpc = (DistributedRPCInvocations.Iface) ServiceRegistry + .getService(localDrpcId); if (drpc != null) { // can happen during shutdown of drpc while topology is still up try { DRPCRequest req = drpc.fetchRequest(function); @@ -202,7 +205,8 @@ public void nextTuple() { returnInfo.put("id", req.get_request_id()); returnInfo.put("host", localDrpcId); returnInfo.put("port", 0); - collector.emit(new Values(req.get_func_args(), JSONValue.toJSONString(returnInfo)), + collector.emit(new Values(req.get_func_args(), JSONValue + .toJSONString(returnInfo)), new DRPCMessageId(req.get_request_id(), 0)); } } catch (AuthorizationException aze) { @@ -286,7 +290,8 @@ public void run() { c = new DRPCInvocationsClient(conf, server, port); } catch (Exception e) { collector.reportError(e); - LOG.error("Failed to create DRPCInvocationsClient for remote {}:{}. Retrying after {} secs.", + LOG.error("Failed to create DRPCInvocationsClient for remote {}:{}. Retrying " + + "after {} secs.", server, port, clientConstructionRetryIntervalSec, e); try { Thread.sleep(clientConstructionRetryIntervalSec * 1000); @@ -297,7 +302,8 @@ public void run() { } } if (c != null) { - LOG.info("Successfully created DRPCInvocationsClient for remote {}:{}.", server, port); + LOG.info("Successfully created DRPCInvocationsClient for remote {}:{}.", server, + port); clients.add(c); } else { LOG.warn("DRPCInvocationsClient creation retry for remote {}:{} interrupted.", diff --git a/storm-client/src/jvm/org/apache/storm/drpc/JoinResult.java b/storm-client/src/jvm/org/apache/storm/drpc/JoinResult.java index d4b7214a024..f191644edc1 100644 --- a/storm-client/src/jvm/org/apache/storm/drpc/JoinResult.java +++ b/storm-client/src/jvm/org/apache/storm/drpc/JoinResult.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -26,7 +32,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; - public class JoinResult extends BaseRichBolt { public static final Logger LOG = LoggerFactory.getLogger(JoinResult.class); @@ -40,7 +45,8 @@ public JoinResult(String returnComponent) { } @Override - public void prepare(Map map, TopologyContext context, OutputCollector collector) { + public void prepare(Map map, TopologyContext context, + OutputCollector collector) { this.collector = collector; } diff --git a/storm-client/src/jvm/org/apache/storm/drpc/KeyedFairBolt.java b/storm-client/src/jvm/org/apache/storm/drpc/KeyedFairBolt.java index 7a358bf2e37..1a9f0da1ddd 100644 --- a/storm-client/src/jvm/org/apache/storm/drpc/KeyedFairBolt.java +++ b/storm-client/src/jvm/org/apache/storm/drpc/KeyedFairBolt.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -24,7 +30,6 @@ import org.apache.storm.tuple.Tuple; import org.apache.storm.utils.KeyedRoundRobinQueue; - public class KeyedFairBolt implements IRichBolt, FinishedCallback { IRichBolt delegate; KeyedRoundRobinQueue rrQueue; @@ -40,7 +45,8 @@ public KeyedFairBolt(IBasicBolt delegate) { } @Override - public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) { + public void prepare(Map topoConf, TopologyContext context, + OutputCollector collector) { if (delegate instanceof FinishedCallback) { callback = (FinishedCallback) delegate; } @@ -54,7 +60,7 @@ public void run() { delegate.execute(rrQueue.take()); } } catch (InterruptedException e) { - //ignore + // ignore } } }); diff --git a/storm-client/src/jvm/org/apache/storm/drpc/LinearDRPCInputDeclarer.java b/storm-client/src/jvm/org/apache/storm/drpc/LinearDRPCInputDeclarer.java index af82d04a42d..08f5aed1458 100644 --- a/storm-client/src/jvm/org/apache/storm/drpc/LinearDRPCInputDeclarer.java +++ b/storm-client/src/jvm/org/apache/storm/drpc/LinearDRPCInputDeclarer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/drpc/LinearDRPCTopologyBuilder.java b/storm-client/src/jvm/org/apache/storm/drpc/LinearDRPCTopologyBuilder.java index 3092f9a9beb..30882fabb93 100644 --- a/storm-client/src/jvm/org/apache/storm/drpc/LinearDRPCTopologyBuilder.java +++ b/storm-client/src/jvm/org/apache/storm/drpc/LinearDRPCTopologyBuilder.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -21,10 +27,10 @@ import org.apache.storm.Constants; import org.apache.storm.ILocalDRPC; import org.apache.storm.coordination.BatchBoltExecutor; -import org.apache.storm.coordination.CoordinatedBolt; import org.apache.storm.coordination.CoordinatedBolt.FinishedCallback; import org.apache.storm.coordination.CoordinatedBolt.IdStreamSpec; import org.apache.storm.coordination.CoordinatedBolt.SourceArgs; +import org.apache.storm.coordination.CoordinatedBolt; import org.apache.storm.coordination.IBatchBolt; import org.apache.storm.generated.SharedMemory; import org.apache.storm.generated.StormTopology; @@ -46,7 +52,6 @@ public class LinearDRPCTopologyBuilder { String function; List components = new ArrayList<>(); - public LinearDRPCTopologyBuilder(String function) { this.function = function; } @@ -130,7 +135,8 @@ private StormTopology createTopology(DRPCSpout spout) { } if (idSpec != null) { - declarer.fieldsGrouping(idSpec.getGlobalStreamId().get_componentId(), PrepareRequest.ID_STREAM, new Fields("request")); + declarer.fieldsGrouping(idSpec.getGlobalStreamId().get_componentId(), + PrepareRequest.ID_STREAM, new Fields("request")); } if (i == 0 && component.declarations.isEmpty()) { declarer.noneGrouping(PREPARE_ID, PrepareRequest.ARGS_STREAM); @@ -155,13 +161,15 @@ private StormTopology createTopology(DRPCSpout spout) { lastBolt.declareOutputFields(getter); Map streams = getter.getFieldsDeclaration(); if (streams.size() != 1) { - throw new RuntimeException("Must declare exactly one stream from last bolt in LinearDRPCTopology"); + throw new RuntimeException("Must declare exactly one stream from last bolt in " + + "LinearDRPCTopology"); } String outputStream = streams.keySet().iterator().next(); List fields = streams.get(outputStream).get_output_fields(); if (fields.size() != 2) { throw new RuntimeException( - "Output stream of last component in LinearDRPCTopology must contain exactly two fields. " + "Output stream of last component in LinearDRPCTopology must contain exactly two " + + "fields. " + "The first should be the request id, and the second should be the result."); } @@ -374,7 +382,8 @@ public void declare(String prevComponent, InputDeclarer declarer) { } @Override - public LinearDRPCInputDeclarer customGrouping(final String streamId, final CustomStreamGrouping grouping) { + public LinearDRPCInputDeclarer customGrouping(final String streamId, + final CustomStreamGrouping grouping) { addDeclaration(new InputDeclaration() { @Override public void declare(String prevComponent, InputDeclarer declarer) { @@ -397,7 +406,7 @@ public LinearDRPCInputDeclarer addConfigurations(Map conf) { } /** - * return the current component configuration. + * Return the current component configuration. * * @return the current configuration. */ diff --git a/storm-client/src/jvm/org/apache/storm/drpc/PrepareRequest.java b/storm-client/src/jvm/org/apache/storm/drpc/PrepareRequest.java index a0a7cfbc457..61f6248a161 100644 --- a/storm-client/src/jvm/org/apache/storm/drpc/PrepareRequest.java +++ b/storm-client/src/jvm/org/apache/storm/drpc/PrepareRequest.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -23,7 +29,6 @@ import org.apache.storm.tuple.Values; import org.apache.storm.utils.Utils; - public class PrepareRequest extends BaseBasicBolt { public static final String ARGS_STREAM = Utils.DEFAULT_STREAM_ID; public static final String RETURN_STREAM = "ret"; diff --git a/storm-client/src/jvm/org/apache/storm/drpc/ReturnResults.java b/storm-client/src/jvm/org/apache/storm/drpc/ReturnResults.java index abca51b0e96..598fb1469e7 100644 --- a/storm-client/src/jvm/org/apache/storm/drpc/ReturnResults.java +++ b/storm-client/src/jvm/org/apache/storm/drpc/ReturnResults.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -35,7 +41,7 @@ public class ReturnResults extends BaseRichBolt { public static final Logger LOG = LoggerFactory.getLogger(ReturnResults.class); - //ANY CHANGE TO THIS CODE MUST BE SERIALIZABLE COMPATIBLE OR THERE WILL BE PROBLEMS + // ANY CHANGE TO THIS CODE MUST BE SERIALIZABLE COMPATIBLE OR THERE WILL BE PROBLEMS static final long serialVersionUID = -774882142710631591L; OutputCollector collector; boolean local; @@ -43,7 +49,8 @@ public class ReturnResults extends BaseRichBolt { Map clients = new HashMap(); @Override - public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) { + public void prepare(Map topoConf, TopologyContext context, + OutputCollector collector) { conf = topoConf; this.collector = collector; local = topoConf.get(Config.STORM_CLUSTER_MODE).equals("local"); @@ -114,7 +121,7 @@ private void reconnectClient(DRPCInvocationsClient client) { if (client instanceof DRPCInvocationsClient) { try { LOG.info("reconnecting... "); - client.reconnectClient(); //Blocking call + client.reconnectClient(); // Blocking call } catch (TException e2) { LOG.error("Failed to connect to DRPC server", e2); } diff --git a/storm-client/src/jvm/org/apache/storm/executor/ChildEwmaStats.java b/storm-client/src/jvm/org/apache/storm/executor/ChildEwmaStats.java index 0eb5fac88d5..bfbc0982bc3 100644 --- a/storm-client/src/jvm/org/apache/storm/executor/ChildEwmaStats.java +++ b/storm-client/src/jvm/org/apache/storm/executor/ChildEwmaStats.java @@ -1,12 +1,17 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -18,9 +23,11 @@ import org.apache.storm.metrics2.TaskMetrics; /** - * Thread-safe store of EWMA jitter statistics reported by downstream (child) tasks back to a parent task. + * Thread-safe store of EWMA jitter statistics reported by downstream (child) tasks back to a parent + * task. * The data is indexed by parent {@code taskId} so a lookup touches only that task's children: - * {@link #getStats} is an O(1) map lookup and {@link #update} is O(metrics), neither scanning the whole + * {@link #getStats} is an O(1) map lookup and {@link #update} is O(metrics), neither scanning the + * whole * store. This keeps cost bound to a single task's child fan-out, independent of how many tasks the * executor hosts. */ @@ -42,7 +49,8 @@ public ChildEwmaStats(boolean enabled) { /** * Records the jitter metrics reported by {@code childTaskId} for the given parent {@code taskId}. - * Runs in O(metrics) by writing straight into the task's bucket; no rescanning of existing data. + * Runs in O(metrics) by writing straight into the task's bucket; no rescanning of existing + * data. */ public void update(int taskId, int childTaskId, EwmaFeedbackRecord feedback) { if (!enabled) { @@ -50,7 +58,8 @@ public void update(int taskId, int childTaskId, EwmaFeedbackRecord feedback) { } ConcurrentHashMap> children = byTask.computeIfAbsent(taskId, k -> new ConcurrentHashMap<>()); - Map metrics = children.computeIfAbsent(childTaskId, k -> new ConcurrentHashMap<>()); + Map metrics = children.computeIfAbsent(childTaskId, + k -> new ConcurrentHashMap<>()); feedback.forEachMetric(metrics::put); } diff --git a/storm-client/src/jvm/org/apache/storm/executor/EwmaFeedbackRecord.java b/storm-client/src/jvm/org/apache/storm/executor/EwmaFeedbackRecord.java index b99ce78c236..9ad55666959 100644 --- a/storm-client/src/jvm/org/apache/storm/executor/EwmaFeedbackRecord.java +++ b/storm-client/src/jvm/org/apache/storm/executor/EwmaFeedbackRecord.java @@ -1,12 +1,17 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -26,7 +31,8 @@ * @param completeJitter The {@code __complete-jitter} gauge value, or {@link #VOID} if absent. * @param executeJitter The {@code __execute-jitter} gauge value, or {@link #VOID} if absent. */ -public record EwmaFeedbackRecord(double processJitter, double completeJitter, double executeJitter) { +public record EwmaFeedbackRecord(double processJitter, double completeJitter, + double executeJitter) { // Sentinel for an absent metric. Jitter values are always >= 0, so a negative value can never // collide with a real measurement and unambiguously marks "gauge missing / not a Number". diff --git a/storm-client/src/jvm/org/apache/storm/executor/Executor.java b/storm-client/src/jvm/org/apache/storm/executor/Executor.java index 9a319bf07cc..fa4b02cbc63 100644 --- a/storm-client/src/jvm/org/apache/storm/executor/Executor.java +++ b/storm-client/src/jvm/org/apache/storm/executor/Executor.java @@ -1,12 +1,17 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -124,7 +129,8 @@ public abstract class Executor implements Callable, JCQueue.Consumer { protected final Boolean isDebug; protected final Boolean hasEventLoggers; protected final boolean ackingEnabled; - protected final MpscChunkedArrayQueue pendingEmits = new MpscChunkedArrayQueue<>(1024, (int) Math.pow(2, 30)); + protected final MpscChunkedArrayQueue pendingEmits = + new MpscChunkedArrayQueue<>(1024, (int) Math.pow(2, 30)); private final AddressedTuple flushTuple; protected ExecutorTransfer executorTransfer; protected ArrayList idToTask; @@ -140,7 +146,9 @@ public abstract class Executor implements Callable, JCQueue.Consumer { // task ids of all upstream (source component) tasks, recipients of the periodic feedback tick protected final List upstreamTaskIds; protected final ChildEwmaStats childEwmaStats; - protected Executor(WorkerState workerData, List executorId, Map credentials, String type) { + + protected Executor(WorkerState workerData, List executorId, Map credentials, String type) { this.workerData = workerData; this.executorId = executorId; this.type = type; @@ -148,7 +156,8 @@ protected Executor(WorkerState workerData, List executorId, Map executorId, Map(); this.taskToComponent = workerData.getTaskToComponent(); - this.streamToComponentToGrouper = outboundComponents(workerTopologyContext, componentId, topoConf); + this.streamToComponentToGrouper = outboundComponents(workerTopologyContext, componentId, + topoConf); if (this.streamToComponentToGrouper != null) { this.groupers = streamToComponentToGrouper.values().stream() .filter(Objects::nonNull) - .flatMap(m -> m.values().stream()).collect(Collectors.toList()); + .flatMap(m -> m.values().stream()) + .collect(Collectors.toList()); } else { this.groupers = Collections.emptyList(); } - this.reportError = new ReportError(topoConf, stormClusterState, stormId, componentId, workerTopologyContext); + this.reportError = new ReportError(topoConf, stormClusterState, stormId, componentId, + workerTopologyContext); this.reportErrorDie = new ReportErrorAndDie(reportError, suicideFn); this.sampler = ConfigUtils.mkStatsSampler(topoConf); this.isDebug = ObjectReader.getBoolean(topoConf.get(Config.TOPOLOGY_DEBUG), false); @@ -193,11 +205,14 @@ protected Executor(WorkerState workerData, List executorId, Map executorId, Map executorId, Map credentials) { + public static Executor mkExecutor(WorkerState workerState, List executorId, Map credentials) { Executor executor; WorkerTopologyContext workerTopologyContext = workerState.getWorkerTopologyContext(); @@ -241,7 +257,8 @@ public static Executor mkExecutor(WorkerState workerState, List executorId return executor; } - private static String getExecutorType(WorkerTopologyContext workerTopologyContext, String componentId) { + private static String getExecutorType(WorkerTopologyContext workerTopologyContext, + String componentId) { StormTopology topology = workerTopologyContext.getRawTopology(); Map spouts = topology.get_spouts(); Map bolts = topology.get_bolts(); @@ -255,7 +272,8 @@ private static String getExecutorType(WorkerTopologyContext workerTopologyContex } /** - * Retrieves all values of all static fields of {@link Config} which represent all available configuration keys through reflection. The + * Retrieves all values of all static fields of {@link Config} which represent all available + * configuration keys through reflection. The * method assumes that they are {@code String}s through reflection. * * @return the list of retrieved field values @@ -283,14 +301,15 @@ public Queue getPendingEmits() { } /** - * separated from mkExecutor in order to replace executor transfer in executor data for testing. + * Separated from mkExecutor in order to replace executor transfer in executor data for testing. */ public ExecutorShutdown execute() throws Exception { LOG.info("Loading executor tasks " + componentId + ":" + executorId); String handlerName = componentId + "-executor" + executorId; Utils.SmartThread handler = - Utils.asyncLoop(this, false, reportErrorDie, Thread.NORM_PRIORITY, true, true, handlerName); + Utils.asyncLoop(this, false, reportErrorDie, Thread.NORM_PRIORITY, true, true, + handlerName); LOG.info("Finished loading executor " + componentId + ":" + executorId); return new ExecutorShutdown(this, Lists.newArrayList(handler), idToTask, receiveQueue); @@ -334,9 +353,11 @@ protected void updateExecCredsIfRequired() { this.needToRefreshCreds.set(false); LOG.info("The credentials are being updated {}.", executorId); Credentials creds = this.workerData.getCredentials(); - idToTask.stream().map(Task::getTaskObject).filter(taskObject -> taskObject instanceof ICredentialsListener) + idToTask.stream().map(Task::getTaskObject) + .filter(taskObject -> taskObject instanceof ICredentialsListener) .forEach(taskObject -> { - ((ICredentialsListener) taskObject).setCredentials(creds == null ? null : creds.get_creds()); + ((ICredentialsListener) taskObject).setCredentials(creds == null + ? null : creds.get_creds()); }); } } @@ -350,7 +371,8 @@ public void metricsTick(Task task, TupleImpl tuple) { try { Integer interval = tuple.getInteger(0); int taskId = task.getTaskId(); - Map> taskToMetricToRegistry = intervalToTaskToMetricToRegistry.get(interval); + Map> taskToMetricToRegistry = + intervalToTaskToMetricToRegistry.get(interval); Map nameToRegistry = null; if (taskToMetricToRegistry != null) { nameToRegistry = taskToMetricToRegistry.get(taskId); @@ -363,7 +385,8 @@ public void metricsTick(Task task, TupleImpl tuple) { Object value = metric.getValueAndReset(); Map dimensions = metric.getDimensions(); if (value != null) { - IMetricsConsumer.DataPoint dataPoint = new IMetricsConsumer.DataPoint(name, value, dimensions); + IMetricsConsumer.DataPoint dataPoint = new IMetricsConsumer.DataPoint(name, + value, dimensions); dataPoints.add(dataPoint); } } @@ -391,7 +414,8 @@ public void metricsTick(Task task, TupleImpl tuple) { * and a default interval of -1 (indicating an on-demand, non-periodic-metrics tick), * followed by the {@link EwmaFeedbackRecord} snapshot. The snapshot is emitted on every tick: * the tick frequency ({@code topology.upstream.feedback.freq.secs}) is the rate limit, and an - * unconditional resend keeps a restarted/reassigned upstream task from being stranded with stale + * unconditional resend keeps a restarted/reassigned upstream task from being stranded with + * stale * or empty stats when the metric value happens to be stable.

    * * @param taskId The ID of the task for which metrics are being collected. @@ -399,7 +423,8 @@ public void metricsTick(Task task, TupleImpl tuple) { * feedback stream schema declared by {@link StormCommon#upstreamFeedbackFields()}). */ public Values buildUpstreamFeedbackTuple(int taskId) { - EwmaFeedbackRecord statsRecord = EwmaFeedbackRecord.fromWorkerState(this.workerData, taskId); + EwmaFeedbackRecord statsRecord = EwmaFeedbackRecord.fromWorkerState(this.workerData, + taskId); IMetricsConsumer.TaskInfo taskInfo = new IMetricsConsumer.TaskInfo( hostname, workerTopologyContext.getThisWorkerPort(), componentId, taskId, Time.currentTimeSecs(), -1); @@ -438,13 +463,15 @@ public void updateChildEwmaStats(Task task, TupleImpl tuple) { // Safe type check replaces unchecked cast and suppression if (!(values.get(0) instanceof IMetricsConsumer.TaskInfo taskInfo)) { LOG.warn("Unexpected type at index 0 in feedbackTuple for task {}: {}", - task.getTaskId(), values.get(0) == null ? "null" : values.get(0).getClass().getName()); + task.getTaskId(), values.get(0) == null ? "null" : values.get(0).getClass() + .getName()); return; } if (!(values.get(1) instanceof EwmaFeedbackRecord feedback)) { LOG.warn("Unexpected type at index 1 in feedbackTuple for task {}: {}", - task.getTaskId(), values.get(1) == null ? "null" : values.get(1).getClass().getName()); + task.getTaskId(), values.get(1) == null ? "null" : values.get(1).getClass() + .getName()); return; } @@ -452,7 +479,8 @@ public void updateChildEwmaStats(Task task, TupleImpl tuple) { } // updates v1 metric dataPoints with v2 metric API data - private void addV2Metrics(int taskId, List dataPoints, int interval) { + private void addV2Metrics(int taskId, List dataPoints, + int interval) { if (!enableV2MetricsDataPoints) { return; } @@ -480,7 +508,8 @@ private void processGauges(int taskId, List dataPoin v = gauge.getValue(); } if (v instanceof Number) { - IMetricsConsumer.DataPoint dataPoint = new IMetricsConsumer.DataPoint(entry.getKey(), v); + IMetricsConsumer.DataPoint dataPoint = new IMetricsConsumer.DataPoint(entry + .getKey(), v); dataPoints.add(dataPoint); } } @@ -490,17 +519,20 @@ private void processCounters(int taskId, List dataPo Map counters = workerData.getMetricRegistry().getTaskCounters(taskId); for (Map.Entry entry : counters.entrySet()) { Object value = entry.getValue().getCount(); - IMetricsConsumer.DataPoint dataPoint = new IMetricsConsumer.DataPoint(entry.getKey(), value); + IMetricsConsumer.DataPoint dataPoint = new IMetricsConsumer.DataPoint(entry.getKey(), + value); dataPoints.add(dataPoint); } } private void processHistograms(int taskId, List dataPoints) { - Map histograms = workerData.getMetricRegistry().getTaskHistograms(taskId); + Map histograms = workerData.getMetricRegistry() + .getTaskHistograms(taskId); for (Map.Entry entry : histograms.entrySet()) { Snapshot snapshot = entry.getValue().getSnapshot(); addSnapshotDatapoints(entry.getKey(), snapshot, dataPoints); - IMetricsConsumer.DataPoint dataPoint = new IMetricsConsumer.DataPoint(entry.getKey() + ".count", entry.getValue().getCount()); + IMetricsConsumer.DataPoint dataPoint = new IMetricsConsumer.DataPoint(entry.getKey() + + ".count", entry.getValue().getCount()); dataPoints.add(dataPoint); } } @@ -521,16 +553,20 @@ private void processTimers(int taskId, List dataPoin } } - private void addMeteredDatapoints(String baseName, Metered metered, List dataPoints) { - IMetricsConsumer.DataPoint dataPoint = new IMetricsConsumer.DataPoint(baseName + ".count", metered.getCount()); + private void addMeteredDatapoints(String baseName, Metered metered, + List dataPoints) { + IMetricsConsumer.DataPoint dataPoint = new IMetricsConsumer.DataPoint(baseName + ".count", + metered.getCount()); dataPoints.add(dataPoint); addConvertedMetric(baseName, ".m1_rate", metered.getOneMinuteRate(), dataPoints, false); addConvertedMetric(baseName, ".m5_rate", metered.getFiveMinuteRate(), dataPoints, false); - addConvertedMetric(baseName, ".m15_rate", metered.getFifteenMinuteRate(), dataPoints, false); + addConvertedMetric(baseName, ".m15_rate", metered.getFifteenMinuteRate(), dataPoints, + false); addConvertedMetric(baseName, ".mean_rate", metered.getMeanRate(), dataPoints, false); } - private void addSnapshotDatapoints(String baseName, Snapshot snapshot, List dataPoints) { + private void addSnapshotDatapoints(String baseName, Snapshot snapshot, + List dataPoints) { addConvertedMetric(baseName, ".max", snapshot.getMax(), dataPoints, true); addConvertedMetric(baseName, ".mean", snapshot.getMean(), dataPoints, true); addConvertedMetric(baseName, ".min", snapshot.getMin(), dataPoints, true); @@ -546,7 +582,8 @@ private void addSnapshotDatapoints(String baseName, Snapshot snapshot, List dataPoints, boolean needConversion) { IMetricsConsumer.DataPoint dataPoint - = new IMetricsConsumer.DataPoint(baseName + suffix, needConversion ? convertDuration(value) : value); + = new IMetricsConsumer.DataPoint(baseName + suffix, needConversion + ? convertDuration(value) : value); dataPoints.add(dataPoint); } @@ -575,9 +612,11 @@ private void scheduleMetricsTick(int interval) { timerTask.scheduleRecurring(interval, interval, () -> { TupleImpl tuple = - new TupleImpl(workerTopologyContext, new Values(interval), Constants.SYSTEM_COMPONENT_ID, + new TupleImpl(workerTopologyContext, new Values(interval), + Constants.SYSTEM_COMPONENT_ID, (int) Constants.SYSTEM_TASK_ID, Constants.METRICS_TICK_STREAM_ID); - AddressedTuple metricsTickTuple = new AddressedTuple(AddressedTuple.BROADCAST_DEST, tuple); + AddressedTuple metricsTickTuple = new AddressedTuple(AddressedTuple.BROADCAST_DEST, + tuple); publishTimerTuple(metricsTickTuple, "metrics tick tuple"); } ); @@ -614,7 +653,8 @@ private List computeUpstreamTaskIds() { } /** - * Schedules a recurring internal tick on {@link Constants#FEEDBACK_TICK_STREAM_ID}. Handling the + * Schedules a recurring internal tick on {@link Constants#FEEDBACK_TICK_STREAM_ID}. Handling + * the * tick (see BoltExecutor.tupleActionFn) triggers {@link #sendUpstreamFeedback(Task)}, replacing * the former probabilistic per-emit trigger with a deterministic periodic one. */ @@ -623,9 +663,11 @@ protected void scheduleUpstreamFeedbackTick(int interval) { timerTask.scheduleRecurring(interval, interval, () -> { TupleImpl tuple = - new TupleImpl(workerTopologyContext, new Values(interval), Constants.SYSTEM_COMPONENT_ID, + new TupleImpl(workerTopologyContext, new Values(interval), + Constants.SYSTEM_COMPONENT_ID, (int) Constants.SYSTEM_TASK_ID, Constants.FEEDBACK_TICK_STREAM_ID); - AddressedTuple feedbackTickTuple = new AddressedTuple(AddressedTuple.BROADCAST_DEST, tuple); + AddressedTuple feedbackTickTuple = new AddressedTuple(AddressedTuple.BROADCAST_DEST, + tuple); publishTimerTuple(feedbackTickTuple, "upstream feedback tick tuple"); } ); @@ -647,9 +689,11 @@ public void sendUpstreamFeedback(Task task) { } protected void setupTicks(boolean isSpout) { - final Integer tickTimeSecs = ObjectReader.getInt(topoConf.get(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS), null); + final Integer tickTimeSecs = ObjectReader.getInt(topoConf + .get(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS), null); if (tickTimeSecs != null) { - boolean enableMessageTimeout = (Boolean) topoConf.get(Config.TOPOLOGY_ENABLE_MESSAGE_TIMEOUTS); + boolean enableMessageTimeout = (Boolean) topoConf + .get(Config.TOPOLOGY_ENABLE_MESSAGE_TIMEOUTS); boolean isAcker = Acker.ACKER_COMPONENT_ID.equals(componentId); if ((!isAcker && Utils.isSystemId(componentId)) || (!enableMessageTimeout && isSpout) @@ -659,11 +703,13 @@ protected void setupTicks(boolean isSpout) { StormTimer timerTask = workerData.getUserTimer(); timerTask.scheduleRecurring(tickTimeSecs, tickTimeSecs, () -> { - TupleImpl tuple = new TupleImpl(workerTopologyContext, new Values(tickTimeSecs), + TupleImpl tuple = new TupleImpl(workerTopologyContext, + new Values(tickTimeSecs), Constants.SYSTEM_COMPONENT_ID, (int) Constants.SYSTEM_TASK_ID, Constants.SYSTEM_TICK_STREAM_ID); - AddressedTuple tickTuple = new AddressedTuple(AddressedTuple.BROADCAST_DEST, tuple); + AddressedTuple tickTuple = new AddressedTuple(AddressedTuple.BROADCAST_DEST, + tuple); publishTimerTuple(tickTuple, "tick tuple"); } ); @@ -677,14 +723,17 @@ public void reflectNewLoadMapping(LoadMapping loadMapping) { } } - // Called by flush-tuple-timer thread. Publishes to the control lane when enabled (insulating the flush signal - // from the data backlog); otherwise falls back to an un-batched write to the recvQueue as before. + // Called by flush-tuple-timer thread. Publishes to the control lane when enabled (insulating + // the flush signal + // from the data backlog); otherwise falls back to an un-batched write to the recvQueue as + // before. public boolean publishFlushTuple() { if (receiveQueue.tryPublishControl(flushTuple)) { LOG.debug("Published Flush tuple to: {} ", getComponentId()); return true; } else { - LOG.debug("Target queue (control lane or RecvQ) is currently full, will retry publishing Flush Tuple later to : {}", + LOG.debug("Target queue (control lane or RecvQ) is currently full, will retry " + + "publishing Flush Tuple later to : {}", getComponentId()); return false; } @@ -697,12 +746,15 @@ private Map> outboundComponen WorkerTopologyContext workerTopologyContext, String componentId, Map topoConf) { Map> ret = new HashMap<>(); - Map> outputGroupings = workerTopologyContext.getTargets(componentId); + Map> outputGroupings = workerTopologyContext + .getTargets(componentId); for (Map.Entry> entry : outputGroupings.entrySet()) { String streamId = entry.getKey(); Map componentGrouping = entry.getValue(); - Fields outFields = workerTopologyContext.getComponentOutputFields(componentId, streamId); - Map componentGrouper = new HashMap(); + Fields outFields = workerTopologyContext.getComponentOutputFields(componentId, + streamId); + Map componentGrouper = + new HashMap(); for (Map.Entry cg : componentGrouping.entrySet()) { String component = cg.getKey(); Grouping grouping = cg.getValue(); @@ -716,7 +768,8 @@ private Map> outboundComponen } } - for (String stream : workerTopologyContext.getComponentCommon(componentId).get_streams().keySet()) { + for (String stream : workerTopologyContext.getComponentCommon(componentId).get_streams() + .keySet()) { if (!ret.containsKey(stream)) { ret.put(stream, null); } diff --git a/storm-client/src/jvm/org/apache/storm/executor/ExecutorShutdown.java b/storm-client/src/jvm/org/apache/storm/executor/ExecutorShutdown.java index f6ecbe3bc94..3932191f352 100644 --- a/storm-client/src/jvm/org/apache/storm/executor/ExecutorShutdown.java +++ b/storm-client/src/jvm/org/apache/storm/executor/ExecutorShutdown.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -41,7 +47,8 @@ public class ExecutorShutdown implements Shutdownable, IRunningExecutor { private final ArrayList taskDatas; private final JCQueue receiveQueue; - public ExecutorShutdown(Executor executor, List threads, ArrayList taskDatas, JCQueue recvQueue) { + public ExecutorShutdown(Executor executor, List threads, + ArrayList taskDatas, JCQueue recvQueue) { this.executor = executor; this.threads = threads; this.taskDatas = taskDatas; @@ -81,20 +88,24 @@ public boolean publishFlushTuple() { @Override public void shutdown() { try { - LOG.info("Shutting down executor " + executor.getComponentId() + ":" + executor.getExecutorId()); + LOG.info("Shutting down executor " + executor.getComponentId() + ":" + executor + .getExecutorId()); executor.getReceiveQueue().close(); for (Utils.SmartThread t : threads) { t.interrupt(); } for (Utils.SmartThread t : threads) { - LOG.debug("Executor " + executor.getComponentId() + ":" + executor.getExecutorId() + " joining thread " + t.getName()); - //Don't wait forever. - //This is to avoid the deadlock between the executor thread (t) and the shutdown hook (which invokes Worker::shutdown) - //when it is the executor thread (t) who invokes the shutdown hook. See STORM-3658. + LOG.debug("Executor " + executor.getComponentId() + ":" + executor.getExecutorId() + + " joining thread " + t.getName()); + // Don't wait forever. + // This is to avoid the deadlock between the executor thread (t) and the shutdown + // hook (which invokes Worker::shutdown) + // when it is the executor thread (t) who invokes the shutdown hook. See STORM-3658. long waitMs = 100; t.join(waitMs); if (t.isAlive()) { - LOG.warn("Thread {} is still alive ({} ms after interruption). Stop waiting for it.", t.getName(), waitMs); + LOG.warn("Thread {} is still alive ({} ms after interruption). Stop waiting " + + "for it.", t.getName(), waitMs); } } executor.getStats().cleanupStats(); @@ -123,7 +134,8 @@ public void shutdown() { } } } - LOG.info("Shut down executor " + executor.getComponentId() + ":" + executor.getExecutorId()); + LOG.info("Shut down executor " + executor.getComponentId() + ":" + executor + .getExecutorId()); } catch (Exception e) { throw Utils.wrapInRuntime(e); } diff --git a/storm-client/src/jvm/org/apache/storm/executor/ExecutorTransfer.java b/storm-client/src/jvm/org/apache/storm/executor/ExecutorTransfer.java index 7052dce4ac4..14c51349148 100644 --- a/storm-client/src/jvm/org/apache/storm/executor/ExecutorTransfer.java +++ b/storm-client/src/jvm/org/apache/storm/executor/ExecutorTransfer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -39,24 +45,30 @@ public class ExecutorTransfer { private int indexingBase = 0; private ArrayList localReceiveQueues; // [taskId-indexingBase] => queue : List of all recvQs local to this worker private AtomicReferenceArray queuesToFlush; - // [taskId-indexingBase] => queue, some entries can be null. : outbound Qs for this executor instance + // [taskId-indexingBase] => queue, some entries can be null. : outbound Qs for this executor + // instance public ExecutorTransfer(WorkerState workerData, Map topoConf) { this.workerData = workerData; WorkerTopologyContext workerTopologyContext = workerData.getWorkerTopologyContext(); - this.threadLocalSerializer = ThreadLocal.withInitial(() -> new KryoTupleSerializer(topoConf, workerTopologyContext)); + this.threadLocalSerializer = ThreadLocal.withInitial(() -> new KryoTupleSerializer(topoConf, + workerTopologyContext)); this.isDebug = ObjectReader.getBoolean(topoConf.get(Config.TOPOLOGY_DEBUG), false); } - // to be called after all Executor objects in the worker are created and before this object is used + // to be called after all Executor objects in the worker are created and before this object is + // used public void initLocalRecvQueues() { - Integer minTaskId = workerData.getLocalReceiveQueues().keySet().stream().min(Integer::compareTo).get(); - this.localReceiveQueues = Utils.convertToArray(workerData.getLocalReceiveQueues(), minTaskId); + Integer minTaskId = workerData.getLocalReceiveQueues().keySet().stream() + .min(Integer::compareTo).get(); + this.localReceiveQueues = Utils.convertToArray(workerData.getLocalReceiveQueues(), + minTaskId); this.indexingBase = minTaskId; this.queuesToFlush = new AtomicReferenceArray(localReceiveQueues.size()); } - // adds addressedTuple to destination Q if it is not full. else adds to pendingEmits (if its not null) + // adds addressedTuple to destination Q if it is not full. else adds to pendingEmits (if its not + // null) public boolean tryTransfer(AddressedTuple addressedTuple, Queue pendingEmits) { if (isDebug) { LOG.info("TRANSFERRING tuple {}", addressedTuple); @@ -66,10 +78,10 @@ public boolean tryTransfer(AddressedTuple addressedTuple, Queue if (localQueue != null) { return tryTransferLocal(addressedTuple, localQueue, pendingEmits); } - return workerData.tryTransferRemote(addressedTuple, pendingEmits, threadLocalSerializer.get()); + return workerData.tryTransferRemote(addressedTuple, pendingEmits, threadLocalSerializer + .get()); } - // flushes local and remote messages public void flush() throws InterruptedException { flushLocal(); @@ -86,7 +98,6 @@ private void flushLocal() throws InterruptedException { } } - public JCQueue getLocalQueue(AddressedTuple tuple) { if ((tuple.dest - indexingBase) >= localReceiveQueues.size()) { return null; @@ -95,15 +106,21 @@ public JCQueue getLocalQueue(AddressedTuple tuple) { } /** - * Adds tuple to localQueue (if overflow is empty). If localQueue is full adds to pendingEmits instead. pendingEmits can be null. + * Adds tuple to localQueue (if overflow is empty). If localQueue is full adds to pendingEmits + * instead. pendingEmits can be null. * Returns false if unable to add to localQueue. */ - public boolean tryTransferLocal(AddressedTuple tuple, JCQueue localQueue, Queue pendingEmits) { + public boolean tryTransferLocal(AddressedTuple tuple, JCQueue localQueue, + Queue pendingEmits) { workerData.checkSerialize(threadLocalSerializer.get(), tuple); - if (localQueue.isControlLaneEnabled() && Constants.isControlStreamId(tuple.getTuple().getSourceStreamId())) { - // control tuples bypass batching, pendingEmits ordering and backpressure. If the control lane is full - // the tuple is dropped and counted; control signals are periodic, so the next one arrives within its - // period. Report the tuple as handled either way so it is never queued behind the data backlog. + if (localQueue.isControlLaneEnabled() && Constants.isControlStreamId(tuple.getTuple() + .getSourceStreamId())) { + // control tuples bypass batching, pendingEmits ordering and backpressure. If the + // control lane is full + // the tuple is dropped and counted; control signals are periodic, so the next one + // arrives within its + // period. Report the tuple as handled either way so it is never queued behind the data + // backlog. localQueue.tryPublishControl(tuple); return true; } diff --git a/storm-client/src/jvm/org/apache/storm/executor/IRunningExecutor.java b/storm-client/src/jvm/org/apache/storm/executor/IRunningExecutor.java index e62fada8d78..5173f4310f9 100644 --- a/storm-client/src/jvm/org/apache/storm/executor/IRunningExecutor.java +++ b/storm-client/src/jvm/org/apache/storm/executor/IRunningExecutor.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/executor/LocalExecutor.java b/storm-client/src/jvm/org/apache/storm/executor/LocalExecutor.java index 050b56dff57..665e1bc496b 100644 --- a/storm-client/src/jvm/org/apache/storm/executor/LocalExecutor.java +++ b/storm-client/src/jvm/org/apache/storm/executor/LocalExecutor.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

    - * http://www.apache.org/licenses/LICENSE-2.0 - *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

    http://www.apache.org/licenses/LICENSE-2.0 + * + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -24,14 +30,17 @@ public class LocalExecutor { private static volatile String trackId = null; - public static Executor mkExecutor(WorkerState workerState, List executorId, Map initialCredentials) + public static Executor mkExecutor(WorkerState workerState, List executorId, Map initialCredentials) throws Exception { Executor executor = Executor.mkExecutor(workerState, executorId, initialCredentials); - executor.setLocalExecutorTransfer(new ExecutorTransfer(workerState, executor.getTopoConf()) { + executor.setLocalExecutorTransfer(new ExecutorTransfer(workerState, executor + .getTopoConf()) { @Override public boolean tryTransfer(AddressedTuple tuple, Queue pendingEmits) { if (null != trackId) { - ((AtomicInteger) ((Map) RegisteredGlobalState.getState(trackId)).get("transferred")).incrementAndGet(); + ((AtomicInteger) ((Map) RegisteredGlobalState.getState(trackId)) + .get("transferred")).incrementAndGet(); } return super.tryTransfer(tuple, pendingEmits); } diff --git a/storm-client/src/jvm/org/apache/storm/executor/TupleInfo.java b/storm-client/src/jvm/org/apache/storm/executor/TupleInfo.java index ca53b33a9c3..0797f45ccfd 100644 --- a/storm-client/src/jvm/org/apache/storm/executor/TupleInfo.java +++ b/storm-client/src/jvm/org/apache/storm/executor/TupleInfo.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/executor/bolt/BoltExecutor.java b/storm-client/src/jvm/org/apache/storm/executor/bolt/BoltExecutor.java index 47a4940e72f..4eeff45a2a1 100644 --- a/storm-client/src/jvm/org/apache/storm/executor/bolt/BoltExecutor.java +++ b/storm-client/src/jvm/org/apache/storm/executor/bolt/BoltExecutor.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

    - * http://www.apache.org/licenses/LICENSE-2.0 - *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

    http://www.apache.org/licenses/LICENSE-2.0 + * + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -30,8 +36,8 @@ import org.apache.storm.hooks.info.BoltExecuteInfo; import org.apache.storm.messaging.IConnection; import org.apache.storm.metric.api.IMetricsRegistrant; -import org.apache.storm.policy.IWaitStrategy; import org.apache.storm.policy.IWaitStrategy.WaitSituation; +import org.apache.storm.policy.IWaitStrategy; import org.apache.storm.policy.WaitStrategyPark; import org.apache.storm.security.auth.IAutoCredentials; import org.apache.storm.stats.BoltExecutorStats; @@ -50,7 +56,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; - public class BoltExecutor extends Executor { private static final Logger LOG = LoggerFactory.getLogger(BoltExecutor.class); @@ -62,20 +67,24 @@ public class BoltExecutor extends Executor { private final BoltExecutorStats stats; private BoltOutputCollectorImpl outputCollector; - public BoltExecutor(WorkerState workerData, List executorId, Map credentials) { + public BoltExecutor(WorkerState workerData, List executorId, Map credentials) { super(workerData, executorId, credentials, ClientStatsUtil.BOLT); this.executeSampler = ConfigUtils.mkStatsSampler(topoConf); this.isSystemBoltExecutor = (executorId == Constants.SYSTEM_EXECUTOR_ID); if (isSystemBoltExecutor) { this.consumeWaitStrategy = makeSystemBoltWaitStrategy(); } else { - this.consumeWaitStrategy = ReflectionUtils.newInstance((String) topoConf.get(Config.TOPOLOGY_BOLT_WAIT_STRATEGY)); + this.consumeWaitStrategy = ReflectionUtils.newInstance((String) topoConf + .get(Config.TOPOLOGY_BOLT_WAIT_STRATEGY)); this.consumeWaitStrategy.prepare(topoConf, WaitSituation.BOLT_WAIT); } - this.backPressureWaitStrategy = ReflectionUtils.newInstance((String) topoConf.get(Config.TOPOLOGY_BACKPRESSURE_WAIT_STRATEGY)); + this.backPressureWaitStrategy = ReflectionUtils.newInstance((String) topoConf + .get(Config.TOPOLOGY_BACKPRESSURE_WAIT_STRATEGY)); this.backPressureWaitStrategy.prepare(topoConf, WaitSituation.BACK_PRESSURE_WAIT); this.stats = new BoltExecutorStats(ConfigUtils.samplingRate(this.getTopoConf()), - ObjectReader.getInt(this.getTopoConf().get(Config.NUM_STAT_BUCKETS))); + ObjectReader.getInt(this.getTopoConf() + .get(Config.NUM_STAT_BUCKETS))); } private static IWaitStrategy makeSystemBoltWaitStrategy() { @@ -95,14 +104,14 @@ public void init(ArrayList idToTask, int idToTaskBase) throws InterruptedE executorTransfer.initLocalRecvQueues(); workerReady.await(); while (!stormActive.get()) { - //Topology may be deployed in deactivated mode, wait for activation + // Topology may be deployed in deactivated mode, wait for activation Utils.sleepNoSimulation(100); } LOG.info("Preparing bolt {}:{}", componentId, getTaskIds()); for (Task taskData : idToTask) { if (taskData == null) { - //This happens if the min id is too small + // This happens if the min id is too small continue; } IBolt boltObject = (IBolt) taskData.getTaskObject(); @@ -111,7 +120,8 @@ public void init(ArrayList idToTask, int idToTaskBase) throws InterruptedE ((ICredentialsListener) boltObject).setCredentials(credentials); } if (Constants.SYSTEM_COMPONENT_ID.equals(componentId)) { - BuiltinMetricsUtil.registerIconnectionServerMetric(workerData.getReceiver(), topoConf, userContext); + BuiltinMetricsUtil.registerIconnectionServerMetric(workerData.getReceiver(), + topoConf, userContext); // add any autocredential expiry metrics from the worker if (workerData.getAutoCredentials() != null) { @@ -124,7 +134,8 @@ public void init(ArrayList idToTask, int idToTaskBase) throws InterruptedE } } - this.outputCollector = new BoltOutputCollectorImpl(this, taskData, rand, hasEventLoggers, ackingEnabled, isDebug); + this.outputCollector = new BoltOutputCollectorImpl(this, taskData, rand, + hasEventLoggers, ackingEnabled, isDebug); boltObject.prepare(topoConf, userContext, new OutputCollector(outputCollector)); } openOrPrepareWasCalled.set(true); @@ -171,7 +182,8 @@ public Long call() throws Exception { } } else { if (bpIdleCount == 0) { // check avoids multiple log msgs when spinning in a idle loop - LOG.debug("Experiencing Back Pressure. Entering BackPressure Wait. PendingEmits = {}", pendingEmits.size()); + LOG.debug("Experiencing Back Pressure. Entering BackPressure Wait. " + + "PendingEmits = {}", pendingEmits.size()); } bpIdleCount = backPressureWaitStrategy.idle(bpIdleCount); } @@ -241,7 +253,8 @@ public void tupleActionFn(int taskId, TupleImpl tuple) throws Exception { stats.boltExecuteTuple(tuple.getSourceComponent(), tuple.getSourceStreamId(), delta, workerData.getUptime().upTime(), firstTask); Task currentTask = idToTask.get(taskId - idToTaskBase); - currentTask.getTaskMetrics().boltExecuteTuple(tuple.getSourceComponent(), tuple.getSourceStreamId(), delta); + currentTask.getTaskMetrics().boltExecuteTuple(tuple.getSourceComponent(), tuple + .getSourceStreamId(), delta); } } } diff --git a/storm-client/src/jvm/org/apache/storm/executor/bolt/BoltOutputCollectorImpl.java b/storm-client/src/jvm/org/apache/storm/executor/bolt/BoltOutputCollectorImpl.java index 886a00c7f89..2d975f5b97d 100644 --- a/storm-client/src/jvm/org/apache/storm/executor/bolt/BoltOutputCollectorImpl.java +++ b/storm-client/src/jvm/org/apache/storm/executor/bolt/BoltOutputCollectorImpl.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

    - * http://www.apache.org/licenses/LICENSE-2.0 - *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

    http://www.apache.org/licenses/LICENSE-2.0 + * + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -70,7 +76,8 @@ public List emit(String streamId, Collection anchors, List anchors, List tuple) { + public void emitDirect(int taskId, String streamId, Collection anchors, + List tuple) { try { boltEmit(streamId, anchors, tuple, taskId); } catch (InterruptedException e) { @@ -108,11 +115,13 @@ private List boltEmit(String streamId, Collection anchors, List< msgId = MessageId.makeUnanchored(); } TupleImpl tupleExt = new TupleImpl( - executor.getWorkerTopologyContext(), values, executor.getComponentId(), taskId, streamId, msgId); + executor.getWorkerTopologyContext(), values, executor + .getComponentId(), taskId, streamId, msgId); xsfer.tryTransfer(new AddressedTuple(t, tupleExt), executor.getPendingEmits()); } if (isEventLoggers) { - task.sendToEventLogger(executor, values, executor.getComponentId(), null, random, executor.getPendingEmits()); + task.sendToEventLogger(executor, values, executor.getComponentId(), null, random, + executor.getPendingEmits()); } return outTasks; } @@ -126,7 +135,8 @@ public void ack(Tuple input) { Map anchorsToIds = input.getMessageId().getAnchorsToIds(); for (Map.Entry entry : anchorsToIds.entrySet()) { task.sendUnanchored(Acker.ACKER_ACK_STREAM_ID, - new Values(entry.getKey(), Utils.bitXor(entry.getValue(), ackValue)), + new Values(entry.getKey(), Utils.bitXor(entry.getValue(), + ackValue)), executor.getExecutorTransfer(), executor.getPendingEmits()); } long delta = tupleTimeDelta((TupleImpl) input); @@ -139,8 +149,10 @@ public void ack(Tuple input) { boltAckInfo.applyOn(task.getUserContext()); } if (delta >= 0) { - executor.getStats().boltAckedTuple(input.getSourceComponent(), input.getSourceStreamId(), delta); - task.getTaskMetrics().boltAckedTuple(input.getSourceComponent(), input.getSourceStreamId(), delta); + executor.getStats().boltAckedTuple(input.getSourceComponent(), input + .getSourceStreamId(), delta); + task.getTaskMetrics().boltAckedTuple(input.getSourceComponent(), input + .getSourceStreamId(), delta); } } @@ -152,7 +164,8 @@ public void fail(Tuple input) { Set roots = input.getMessageId().getAnchors(); for (Long root : roots) { task.sendUnanchored(Acker.ACKER_FAIL_STREAM_ID, - new Values(root), executor.getExecutorTransfer(), executor.getPendingEmits()); + new Values(root), executor.getExecutorTransfer(), executor + .getPendingEmits()); } long delta = tupleTimeDelta((TupleImpl) input); if (isDebug) { @@ -161,8 +174,10 @@ public void fail(Tuple input) { BoltFailInfo boltFailInfo = new BoltFailInfo(input, taskId, delta); boltFailInfo.applyOn(task.getUserContext()); if (delta >= 0) { - executor.getStats().boltFailedTuple(input.getSourceComponent(), input.getSourceStreamId()); - task.getTaskMetrics().boltFailedTuple(input.getSourceComponent(), input.getSourceStreamId()); + executor.getStats().boltFailedTuple(input.getSourceComponent(), input + .getSourceStreamId()); + task.getTaskMetrics().boltFailedTuple(input.getSourceComponent(), input + .getSourceStreamId()); } } diff --git a/storm-client/src/jvm/org/apache/storm/executor/error/IReportError.java b/storm-client/src/jvm/org/apache/storm/executor/error/IReportError.java index 13c84d34718..6b8dd737f6c 100644 --- a/storm-client/src/jvm/org/apache/storm/executor/error/IReportError.java +++ b/storm-client/src/jvm/org/apache/storm/executor/error/IReportError.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/executor/error/ReportError.java b/storm-client/src/jvm/org/apache/storm/executor/error/ReportError.java index cc2095ccdd2..546a681faf5 100644 --- a/storm-client/src/jvm/org/apache/storm/executor/error/ReportError.java +++ b/storm-client/src/jvm/org/apache/storm/executor/error/ReportError.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -39,15 +45,18 @@ public class ReportError implements IReportError { private AtomicInteger intervalStartTime; private AtomicInteger intervalErrors; - public ReportError(Map topoConf, IStormClusterState stormClusterState, String stormId, String componentId, + public ReportError(Map topoConf, IStormClusterState stormClusterState, + String stormId, String componentId, WorkerTopologyContext workerTopologyContext) { this.topoConf = topoConf; this.stormClusterState = stormClusterState; this.stormId = stormId; this.componentId = componentId; this.workerTopologyContext = workerTopologyContext; - this.errorIntervalSecs = ObjectReader.getInt(topoConf.get(Config.TOPOLOGY_ERROR_THROTTLE_INTERVAL_SECS)); - this.maxPerInterval = ObjectReader.getInt(topoConf.get(Config.TOPOLOGY_MAX_ERROR_REPORT_PER_INTERVAL)); + this.errorIntervalSecs = ObjectReader.getInt(topoConf + .get(Config.TOPOLOGY_ERROR_THROTTLE_INTERVAL_SECS)); + this.maxPerInterval = ObjectReader.getInt(topoConf + .get(Config.TOPOLOGY_MAX_ERROR_REPORT_PER_INTERVAL)); this.intervalStartTime = new AtomicInteger(Time.currentTimeSecs()); this.intervalErrors = new AtomicInteger(0); } @@ -62,7 +71,8 @@ public void report(Throwable error) { if (intervalErrors.incrementAndGet() <= maxPerInterval) { try { stormClusterState.reportError(stormId, componentId, Utils.hostname(), - workerTopologyContext.getThisWorkerPort().longValue(), error); + workerTopologyContext.getThisWorkerPort() + .longValue(), error); } catch (UnknownHostException e) { throw Utils.wrapInRuntime(e); } diff --git a/storm-client/src/jvm/org/apache/storm/executor/error/ReportErrorAndDie.java b/storm-client/src/jvm/org/apache/storm/executor/error/ReportErrorAndDie.java index b4679274272..ac9ced5c7ea 100644 --- a/storm-client/src/jvm/org/apache/storm/executor/error/ReportErrorAndDie.java +++ b/storm-client/src/jvm/org/apache/storm/executor/error/ReportErrorAndDie.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/executor/spout/SpoutExecutor.java b/storm-client/src/jvm/org/apache/storm/executor/spout/SpoutExecutor.java index 2eab9b08816..72939ddd371 100644 --- a/storm-client/src/jvm/org/apache/storm/executor/spout/SpoutExecutor.java +++ b/storm-client/src/jvm/org/apache/storm/executor/spout/SpoutExecutor.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -29,8 +35,8 @@ import org.apache.storm.hooks.info.SpoutAckInfo; import org.apache.storm.hooks.info.SpoutFailInfo; import org.apache.storm.metrics2.RateCounter; -import org.apache.storm.policy.IWaitStrategy; import org.apache.storm.policy.IWaitStrategy.WaitSituation; +import org.apache.storm.policy.IWaitStrategy; import org.apache.storm.spout.ISpout; import org.apache.storm.spout.SpoutOutputCollector; import org.apache.storm.stats.ClientStatsUtil; @@ -68,11 +74,14 @@ public class SpoutExecutor extends Executor { private final RateCounter skippedInactiveMs; private final RateCounter skippedBackpressureMs; - public SpoutExecutor(final WorkerState workerData, final List executorId, Map credentials) { + public SpoutExecutor(final WorkerState workerData, final List executorId, Map credentials) { super(workerData, executorId, credentials, ClientStatsUtil.SPOUT); - this.spoutWaitStrategy = ReflectionUtils.newInstance((String) topoConf.get(Config.TOPOLOGY_SPOUT_WAIT_STRATEGY)); + this.spoutWaitStrategy = ReflectionUtils.newInstance((String) topoConf + .get(Config.TOPOLOGY_SPOUT_WAIT_STRATEGY)); this.spoutWaitStrategy.prepare(topoConf, WaitSituation.SPOUT_WAIT); - this.backPressureWaitStrategy = ReflectionUtils.newInstance((String) topoConf.get(Config.TOPOLOGY_BACKPRESSURE_WAIT_STRATEGY)); + this.backPressureWaitStrategy = ReflectionUtils.newInstance((String) topoConf + .get(Config.TOPOLOGY_BACKPRESSURE_WAIT_STRATEGY)); this.backPressureWaitStrategy.prepare(topoConf, WaitSituation.BACK_PRESSURE_WAIT); this.lastActive = new AtomicBoolean(false); @@ -80,12 +89,16 @@ public SpoutExecutor(final WorkerState workerData, final List executorId, this.emittedCount = new MutableLong(0); this.emptyEmitStreak = new MutableLong(0); this.stats = new SpoutExecutorStats( - ConfigUtils.samplingRate(this.getTopoConf()), ObjectReader.getInt(this.getTopoConf().get(Config.NUM_STAT_BUCKETS))); - this.skippedMaxSpoutMs = workerData.getMetricRegistry().rateCounter("__skipped-max-spout-ms", componentId, + ConfigUtils.samplingRate(this.getTopoConf()), ObjectReader.getInt(this.getTopoConf() + .get(Config.NUM_STAT_BUCKETS))); + this.skippedMaxSpoutMs = workerData.getMetricRegistry() + .rateCounter("__skipped-max-spout-ms", componentId, taskIds.get(0)); - this.skippedInactiveMs = workerData.getMetricRegistry().rateCounter("__skipped-inactive-ms", componentId, + this.skippedInactiveMs = workerData.getMetricRegistry().rateCounter("__skipped-inactive-ms", + componentId, taskIds.get(0)); - this.skippedBackpressureMs = workerData.getMetricRegistry().rateCounter("__skipped-backpressure-ms", componentId, + this.skippedBackpressureMs = workerData.getMetricRegistry() + .rateCounter("__skipped-backpressure-ms", componentId, taskIds.get(0)); } @@ -99,13 +112,14 @@ public void init(final ArrayList idToTask, int idToTaskBase) throws Interr executorTransfer.initLocalRecvQueues(); workerReady.await(); while (!stormActive.get()) { - //Topology may be deployed in deactivated mode, wait for activation + // Topology may be deployed in deactivated mode, wait for activation Utils.sleepNoSimulation(100); } LOG.info("Opening spout {}:{}", componentId, taskIds); this.idToTask = idToTask; - this.maxSpoutPending = ObjectReader.getInt(topoConf.get(Config.TOPOLOGY_MAX_SPOUT_PENDING), 0) * idToTask.size(); + this.maxSpoutPending = ObjectReader.getInt(topoConf.get(Config.TOPOLOGY_MAX_SPOUT_PENDING), + 0) * idToTask.size(); this.spouts = new ArrayList<>(); for (Task task : idToTask) { if (task != null) { @@ -119,7 +133,8 @@ public void expire(Long key, TupleInfo tupleInfo) { if (tupleInfo.getTimestamp() != 0) { timeDelta = Time.deltaMs(tupleInfo.getTimestamp()); } - failSpoutMsg(SpoutExecutor.this, idToTask.get(tupleInfo.getTaskId() - idToTaskBase), timeDelta, tupleInfo, "TIMEOUT"); + failSpoutMsg(SpoutExecutor.this, idToTask.get(tupleInfo.getTaskId() - idToTaskBase), + timeDelta, tupleInfo, "TIMEOUT"); } }); @@ -166,7 +181,8 @@ public Long call() throws Exception { recvqCheckSkips = 0; } long currCount = emittedCount.get(); - boolean reachedMaxSpoutPending = (maxSpoutPending != 0) && (pending.size() >= maxSpoutPending); + boolean reachedMaxSpoutPending = (maxSpoutPending != 0) && (pending + .size() >= maxSpoutPending); boolean isActive = stormActive.get(); if (!isActive) { @@ -183,7 +199,8 @@ public Long call() throws Exception { long emptyStretch = 0; if (!reachedMaxSpoutPending && pendingEmitsIsEmpty) { - for (int j = 0; j < spouts.size(); j++) { // in critical path. don't use iterators. + for (int j = 0; j < spouts + .size(); j++) { // in critical path. don't use iterators. spouts.get(j).nextTuple(); } noEmits = (currCount == emittedCount.get()); @@ -226,13 +243,15 @@ public Long call() throws Exception { private void backPressureWaitStrategy() throws InterruptedException { long start = Time.currentTimeMillis(); if (bpIdleCount == 0) { // check avoids multiple log msgs when in a idle loop - LOG.debug("Experiencing Back Pressure from downstream components. Entering BackPressure Wait."); + LOG.debug("Experiencing Back Pressure from downstream components. Entering " + + "BackPressure Wait."); } bpIdleCount = backPressureWaitStrategy.idle(bpIdleCount); skippedBackpressureMs.inc(Time.currentTimeMillis() - start); } - private void spoutWaitStrategy(boolean reachedMaxSpoutPending, long emptyStretch) throws InterruptedException { + private void spoutWaitStrategy(boolean reachedMaxSpoutPending, + long emptyStretch) throws InterruptedException { emptyEmitStreak.increment(); long start = Time.currentTimeMillis(); swIdleCount = spoutWaitStrategy.idle(swIdleCount); @@ -292,7 +311,7 @@ protected void acceptTupleAction(int taskId, TupleImpl tuple) { if (taskId != AddressedTuple.BROADCAST_DEST) { tupleActionFn(taskId, tuple); } else if (streamId.equals(Constants.SYSTEM_TICK_STREAM_ID)) { - //taskId is irrelevant here. Ensures pending.rotate() is called once per tick. + // taskId is irrelevant here. Ensures pending.rotate() is called once per tick. tupleActionFn(taskIds.get(0), tuple); } else { @@ -334,7 +353,8 @@ public void tupleActionFn(int taskId, TupleImpl tuple) throws Exception { TupleInfo tupleInfo = pending.remove(id); if (tupleInfo != null && tupleInfo.getMessageId() != null) { if (taskId != tupleInfo.getTaskId()) { - throw new RuntimeException("Fatal error, mismatched task ids: " + taskId + " " + tupleInfo.getTaskId()); + throw new RuntimeException("Fatal error, mismatched task ids: " + taskId + " " + + tupleInfo.getTaskId()); } Long timeDelta = null; if (hasAckers) { @@ -346,22 +366,27 @@ public void tupleActionFn(int taskId, TupleImpl tuple) throws Exception { if (streamId.equals(Acker.ACKER_ACK_STREAM_ID)) { ackSpoutMsg(this, idToTask.get(taskId - idToTaskBase), timeDelta, tupleInfo); } else if (streamId.equals(Acker.ACKER_FAIL_STREAM_ID)) { - failSpoutMsg(this, idToTask.get(taskId - idToTaskBase), timeDelta, tupleInfo, "FAIL-STREAM"); + failSpoutMsg(this, idToTask.get(taskId - idToTaskBase), timeDelta, tupleInfo, + "FAIL-STREAM"); } } } } - public void ackSpoutMsg(SpoutExecutor executor, Task taskData, Long timeDelta, TupleInfo tupleInfo) { + public void ackSpoutMsg(SpoutExecutor executor, Task taskData, Long timeDelta, + TupleInfo tupleInfo) { try { ISpout spout = (ISpout) taskData.getTaskObject(); int taskId = taskData.getTaskId(); if (executor.getIsDebug()) { - LOG.info("SPOUT Acking message {} {}", tupleInfo.getRootId(), tupleInfo.getMessageId()); + LOG.info("SPOUT Acking message {} {}", tupleInfo.getRootId(), tupleInfo + .getMessageId()); } spout.ack(tupleInfo.getMessageId()); - if (!taskData.getUserContext().getHooks().isEmpty()) { // avoid allocating SpoutAckInfo obj if not necessary - new SpoutAckInfo(tupleInfo.getMessageId(), taskId, timeDelta).applyOn(taskData.getUserContext()); + if (!taskData.getUserContext().getHooks() + .isEmpty()) { // avoid allocating SpoutAckInfo obj if not necessary + new SpoutAckInfo(tupleInfo.getMessageId(), taskId, timeDelta).applyOn(taskData + .getUserContext()); } if (hasAckers && timeDelta != null) { executor.getStats().spoutAckedTuple(tupleInfo.getStream(), timeDelta); @@ -372,15 +397,18 @@ public void ackSpoutMsg(SpoutExecutor executor, Task taskData, Long timeDelta, T } } - public void failSpoutMsg(SpoutExecutor executor, Task taskData, Long timeDelta, TupleInfo tupleInfo, String reason) { + public void failSpoutMsg(SpoutExecutor executor, Task taskData, Long timeDelta, + TupleInfo tupleInfo, String reason) { try { ISpout spout = (ISpout) taskData.getTaskObject(); int taskId = taskData.getTaskId(); if (executor.getIsDebug()) { - LOG.info("SPOUT Failing {} : {} REASON: {}", tupleInfo.getRootId(), tupleInfo, reason); + LOG.info("SPOUT Failing {} : {} REASON: {}", tupleInfo.getRootId(), tupleInfo, + reason); } spout.fail(tupleInfo.getMessageId()); - new SpoutFailInfo(tupleInfo.getMessageId(), taskId, timeDelta).applyOn(taskData.getUserContext()); + new SpoutFailInfo(tupleInfo.getMessageId(), taskId, timeDelta).applyOn(taskData + .getUserContext()); if (timeDelta != null) { executor.getStats().spoutFailedTuple(tupleInfo.getStream()); taskData.getTaskMetrics().spoutFailedTuple(tupleInfo.getStream()); @@ -390,7 +418,6 @@ public void failSpoutMsg(SpoutExecutor executor, Task taskData, Long timeDelta, } } - public int getSpoutRecvqCheckSkipCount() { if (ackingEnabled) { return 0; // always check recQ if ACKing enabled diff --git a/storm-client/src/jvm/org/apache/storm/executor/spout/SpoutOutputCollectorImpl.java b/storm-client/src/jvm/org/apache/storm/executor/spout/SpoutOutputCollectorImpl.java index d47b5347c04..49ae131b1cd 100644 --- a/storm-client/src/jvm/org/apache/storm/executor/spout/SpoutOutputCollectorImpl.java +++ b/storm-client/src/jvm/org/apache/storm/executor/spout/SpoutOutputCollectorImpl.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -31,7 +37,8 @@ import org.slf4j.LoggerFactory; /** - * Methods are not thread safe. Each thread expected to have a separate instance, or else synchronize externally + * Methods are not thread safe. Each thread expected to have a separate instance, or else + * synchronize externally */ public class SpoutOutputCollectorImpl implements ISpoutOutputCollector { private static final Logger LOG = LoggerFactory.getLogger(SpoutOutputCollectorImpl.class); @@ -94,7 +101,6 @@ public void flush() { } } - @Override public long getPendingCount() { return pending.size(); @@ -106,7 +112,8 @@ public void reportError(Throwable error) { executor.getReportError().report(error); } - private List sendSpoutMsg(String stream, List values, Object messageId, Integer outTaskId) throws + private List sendSpoutMsg(String stream, List values, Object messageId, + Integer outTaskId) throws InterruptedException { emittedCount.increment(); @@ -135,12 +142,14 @@ private List sendSpoutMsg(String stream, List values, Object me } final TupleImpl tuple = - new TupleImpl(executor.getWorkerTopologyContext(), values, executor.getComponentId(), this.taskId, stream, msgId); + new TupleImpl(executor.getWorkerTopologyContext(), values, executor + .getComponentId(), this.taskId, stream, msgId); AddressedTuple adrTuple = new AddressedTuple(t, tuple); executor.getExecutorTransfer().tryTransfer(adrTuple, executor.getPendingEmits()); } if (isEventLoggers) { - taskData.sendToEventLogger(executor, values, executor.getComponentId(), messageId, random, executor.getPendingEmits()); + taskData.sendToEventLogger(executor, values, executor.getComponentId(), messageId, + random, executor.getPendingEmits()); } if (needAck) { @@ -159,13 +168,17 @@ private List sendSpoutMsg(String stream, List values, Object me pending.put(rootId, info); List ackInitTuple = new Values(rootId, Utils.bitXorVals(ackSeq), this.taskId); - taskData.sendUnanchored(Acker.ACKER_INIT_STREAM_ID, ackInitTuple, executor.getExecutorTransfer(), executor.getPendingEmits()); + taskData.sendUnanchored(Acker.ACKER_INIT_STREAM_ID, ackInitTuple, executor + .getExecutorTransfer(), executor.getPendingEmits()); } else if (messageId != null) { - // Reusing TupleInfo object as we directly call executor.ackSpoutMsg() & are not sending msgs. perf critical + // Reusing TupleInfo object as we directly call executor.ackSpoutMsg() & are not sending + // msgs. perf critical if (isDebug) { if (spoutExecutorThdId != Thread.currentThread().getId()) { - throw new RuntimeException("Detected background thread emitting tuples for the spout. " - + "Spout Output Collector should only emit from the main spout executor thread."); + throw new RuntimeException("Detected background thread emitting tuples for " + + "the spout. " + + "Spout Output Collector should only emit from the main spout " + + "executor thread."); } } globalTupleInfo.clear(); diff --git a/storm-client/src/jvm/org/apache/storm/grouping/CustomStreamGrouping.java b/storm-client/src/jvm/org/apache/storm/grouping/CustomStreamGrouping.java index 9630b21c5ae..e6294a74b6b 100644 --- a/storm-client/src/jvm/org/apache/storm/grouping/CustomStreamGrouping.java +++ b/storm-client/src/jvm/org/apache/storm/grouping/CustomStreamGrouping.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -20,7 +26,8 @@ public interface CustomStreamGrouping extends Serializable { /** - * Tells the stream grouping at runtime the tasks in the target bolt. This information should be used in chooseTasks to determine the + * Tells the stream grouping at runtime the tasks in the target bolt. This information should be + * used in chooseTasks to determine the * target tasks. * *

    It also tells the grouping the metadata on the stream this grouping will be used on. @@ -28,7 +35,8 @@ public interface CustomStreamGrouping extends Serializable { void prepare(WorkerTopologyContext context, GlobalStreamId stream, List targetTasks); /** - * This function implements a custom stream grouping. It takes in as input the number of tasks in the target bolt in prepare and returns + * This function implements a custom stream grouping. It takes in as input the number of tasks + * in the target bolt in prepare and returns * the tasks to send the tuples to. * * @param values the values to group on diff --git a/storm-client/src/jvm/org/apache/storm/grouping/JitterAwareStreamGrouping.java b/storm-client/src/jvm/org/apache/storm/grouping/JitterAwareStreamGrouping.java index b0c723cb2fa..4cc00050be6 100644 --- a/storm-client/src/jvm/org/apache/storm/grouping/JitterAwareStreamGrouping.java +++ b/storm-client/src/jvm/org/apache/storm/grouping/JitterAwareStreamGrouping.java @@ -1,12 +1,17 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -23,35 +28,48 @@ import org.apache.storm.task.WorkerTopologyContext; /** - * A {@link LoadAwareCustomStreamGrouping} that steers each tuple toward the downstream (child) task with + * A {@link LoadAwareCustomStreamGrouping} that steers each tuple toward the downstream (child) task + * with * lower jitter, as reported back to the emitting task through upstream feedback and aggregated by - * {@link ChildEwmaStats}. Jitter is compared with {@link ChildEwmaStats#compareByJitter}, so a lower + * {@link ChildEwmaStats}. Jitter is compared with {@link ChildEwmaStats#compareByJitter}, so a + * lower * {@code __execute-jitter} wins first, then {@code __process-jitter}, then {@code __complete-jitter}. * - *

    Steering uses power-of-two-choices: for each tuple two target tasks are sampled at random and + *

    Steering uses power-of-two-choices: for each tuple two target tasks are sampled at + * random and * the lower-jitter one wins. Random sampling keeps the best task from receiving every tuple (the - * "thundering herd" a plain arg-min selection would cause) while still biasing traffic toward the good + * "thundering herd" a plain arg-min selection would cause) while still biasing traffic toward the + * good * tasks. * - *

    Whenever jitter cannot pick a winner — the sampled pair ties (equal jitter, or neither has reported), - * no feedback exists yet for the source task, or no {@link ChildEwmaStats} was registered — the decision is - * delegated to an embedded {@link LoadAwareShuffleGrouping}. When all targets carry equal jitter, - * every sampled pair ties, so the grouping behaves as a pure load-aware shuffle. {@link #refreshLoad} is + *

    Whenever jitter cannot pick a winner — the sampled pair ties (equal jitter, or neither has + * reported), + * no feedback exists yet for the source task, or no {@link ChildEwmaStats} was registered — the + * decision is + * delegated to an embedded {@link LoadAwareShuffleGrouping}. When all targets carry equal + * jitter, + * every sampled pair ties, so the grouping behaves as a pure load-aware shuffle. {@link + * #refreshLoad} is * forwarded to that delegate, so the fallback path honours real system load and locality. * - *

    Ordering: like any load-aware shuffle, this grouping does not preserve tuple ordering. - * It is a routing policy layered on top of {@link LoadAwareShuffleGrouping} and inherits the same (lack of) + *

    Ordering: like any load-aware shuffle, this grouping does not preserve tuple + * ordering. + * It is a routing policy layered on top of {@link LoadAwareShuffleGrouping} and inherits the same + * (lack of) * ordering semantics: successive tuples emitted by the same source task may be steered to different - * downstream tasks, so consumers must not rely on receiving tuples in emission order. Use a fields grouping + * downstream tasks, so consumers must not rely on receiving tuples in emission order. Use a fields + * grouping * if per-key ordering is required. * - *

    Requirements: this grouping is purely opt-in and only steers while the upstream feedback loop is + *

    Requirements: this grouping is purely opt-in and only steers while the upstream + * feedback loop is * active. The feedback records carry EWMA jitter stats, which are produced solely when * {@code topology.stats.ewma.enable=true}; enabling {@code topology.upstream.feedback.enable} without EWMA * is therefore rejected at config validation (see {@code ConfigValidation.UpstreamFeedbackValidator}), since * the grouping would otherwise silently degrade to a plain load-aware shuffle forever. * - *

    Performance: benchmarks show the feedback signal adds negligible overhead and, at moderate load yields + *

    Performance: benchmarks show the feedback signal adds negligible overhead and, at + * moderate load yields * a directionally lower complete latency than a plain load-aware shuffle at comparable throughput. * At higher load the advantage disappears and results are neutral-to-slightly-worse. * Treat it as an opt-in latency-smoothing policy rather than a general throughput improvement. @@ -77,9 +95,11 @@ public void registerEwmaStats(ChildEwmaStats childEwmaStats) { } @Override - public void prepare(WorkerTopologyContext context, GlobalStreamId stream, List targetTasks) { + public void prepare(WorkerTopologyContext context, GlobalStreamId stream, + List targetTasks) { this.targetTasks = targetTasks; - // The fallback rejects an empty target list; chooseTasks short-circuits that case before delegating. + // The fallback rejects an empty target list; chooseTasks short-circuits that case before + // delegating. if (targetTasks != null && !targetTasks.isEmpty()) { fallback.prepare(context, stream, targetTasks); } @@ -123,7 +143,8 @@ public List chooseTasks(int taskId, List values) { if (cmp > 0) { return Collections.singletonList(b); } - // Tie (equal jitter, or both unreported): no jitter winner -> defer to the load-aware fallback. + // Tie (equal jitter, or both unreported): no jitter winner -> defer to the load-aware + // fallback. return fallback.chooseTasks(taskId, values); } diff --git a/storm-client/src/jvm/org/apache/storm/grouping/Load.java b/storm-client/src/jvm/org/apache/storm/grouping/Load.java index a80926785a1..799d1adef21 100644 --- a/storm-client/src/jvm/org/apache/storm/grouping/Load.java +++ b/storm-client/src/jvm/org/apache/storm/grouping/Load.java @@ -1,31 +1,39 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.grouping; /** - * Represents the load that a Bolt is currently under to help in deciding where to route a tuple, to help balance the load. + * Represents the load that a Bolt is currently under to help in deciding where to route a tuple, to + * help balance the load. */ public class Load { private boolean hasMetrics = false; - private double boltLoad = 0.0; //0 no load to 1 fully loaded - private double connectionLoad = 0.0; //0 no load to 1 fully loaded + private double boltLoad = 0.0; // 0 no load to 1 fully loaded + private double connectionLoad = 0.0; // 0 no load to 1 fully loaded /** * Create a new load. * * @param hasMetrics have metrics been reported yet? * @param boltLoad the load as reported by the bolt 0.0 no load 1.0 fully loaded - * @param connectionLoad the load as reported by the connection to the bolt 0.0 no load 1.0 fully loaded. + * @param connectionLoad the load as reported by the connection to the bolt 0.0 no load 1.0 + * fully loaded. */ public Load(boolean hasMetrics, double boltLoad, double connectionLoad) { this.hasMetrics = hasMetrics; @@ -35,6 +43,7 @@ public Load(boolean hasMetrics, double boltLoad, double connectionLoad) { /** * Check whether has metrics. + * * @return true if metrics have been reported so far. */ public boolean hasMetrics() { @@ -43,6 +52,7 @@ public boolean hasMetrics() { /** * Get bolt load. + * * @return the load as reported by the bolt. */ public double getBoltLoad() { @@ -51,6 +61,7 @@ public double getBoltLoad() { /** * Get connection load. + * * @return the load as reported by the connection */ public double getConnectionLoad() { @@ -59,6 +70,7 @@ public double getConnectionLoad() { /** * Get load. + * * @return the load that is a combination of sub loads. */ public double getLoad() { diff --git a/storm-client/src/jvm/org/apache/storm/grouping/LoadAwareCustomStreamGrouping.java b/storm-client/src/jvm/org/apache/storm/grouping/LoadAwareCustomStreamGrouping.java index 7edeec83f33..853e9755a5d 100644 --- a/storm-client/src/jvm/org/apache/storm/grouping/LoadAwareCustomStreamGrouping.java +++ b/storm-client/src/jvm/org/apache/storm/grouping/LoadAwareCustomStreamGrouping.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/grouping/LoadAwareShuffleGrouping.java b/storm-client/src/jvm/org/apache/storm/grouping/LoadAwareShuffleGrouping.java index 0aa15fe7f45..19ff5ed9b2a 100644 --- a/storm-client/src/jvm/org/apache/storm/grouping/LoadAwareShuffleGrouping.java +++ b/storm-client/src/jvm/org/apache/storm/grouping/LoadAwareShuffleGrouping.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -59,15 +65,18 @@ public class LoadAwareShuffleGrouping implements LoadAwareCustomStreamGrouping, private double lowerBound; @Override - public void prepare(WorkerTopologyContext context, GlobalStreamId stream, List targetTasks) { + public void prepare(WorkerTopologyContext context, GlobalStreamId stream, + List targetTasks) { random = new Random(); - sourceNodeInfo = new NodeInfo(context.getAssignmentId(), Sets.newHashSet((long) context.getThisWorkerPort())); + sourceNodeInfo = new NodeInfo(context.getAssignmentId(), Sets.newHashSet((long) context + .getThisWorkerPort())); taskToNodePort = context.getTaskToNodePort(); nodeToHost = context.getNodeToHost(); this.targetTasks = targetTasks; capacity = targetTasks.size() == 1 ? 1 : Math.max(1000, targetTasks.size() * 5); conf = context.getConf(); - dnsToSwitchMapping = ReflectionUtils.newInstance((String) conf.get(Config.STORM_NETWORK_TOPOGRAPHY_PLUGIN)); + dnsToSwitchMapping = ReflectionUtils.newInstance((String) conf + .get(Config.STORM_NETWORK_TOPOGRAPHY_PLUGIN)); localityGroup = new HashMap<>(); currentScope = LocalityScope.WORKER_LOCAL; higherBound = ObjectReader.getDouble(conf.get(Config.TOPOLOGY_LOCALITYAWARE_HIGHER_BOUND)); @@ -101,7 +110,7 @@ public List chooseTasks(int taskId, List values) { current.set(0); return rets[choices[0]]; } - //race condition with another thread, and we lost + // race condition with another thread, and we lost // try again } } @@ -117,12 +126,14 @@ private void refreshLocalityGroup() { Map cachedTaskToNodePort = taskToNodePort.get(); Map cachedNodeToHost = nodeToHost.get(); - Map hostToRack = getHostToRackMapping(cachedTaskToNodePort, cachedNodeToHost); + Map hostToRack = getHostToRackMapping(cachedTaskToNodePort, + cachedNodeToHost); localityGroup.values().stream().forEach(v -> v.clear()); for (int target : targetTasks) { - LocalityScope scope = calculateScope(cachedTaskToNodePort, cachedNodeToHost, hostToRack, target); + LocalityScope scope = calculateScope(cachedTaskToNodePort, cachedNodeToHost, hostToRack, + target); LOG.debug("targetTask {} is in LocalityScope {}", target, scope); if (!localityGroup.containsKey(scope)) { localityGroup.put(scope, new ArrayList<>()); @@ -149,7 +160,8 @@ private LocalityScope transition(LoadMapping load) { if (targetInScope.isEmpty()) { LocalityScope upScope = LocalityScope.upgrade(currentScope); if (upScope == currentScope) { - throw new RuntimeException("The current scope " + currentScope + " has no target tasks."); + throw new RuntimeException("The current scope " + currentScope + + " has no target tasks."); } currentScope = upScope; return transition(load); @@ -159,7 +171,8 @@ private LocalityScope transition(LoadMapping load) { return currentScope; } - double avg = targetInScope.stream().mapToDouble((key) -> load.get(key)).average().getAsDouble(); + double avg = targetInScope.stream().mapToDouble((key) -> load.get(key)).average() + .getAsDouble(); LocalityScope nextScope = currentScope; if (avg > higherBound) { @@ -168,7 +181,8 @@ private LocalityScope transition(LoadMapping load) { LocalityScope lowerScope = LocalityScope.downgrade(currentScope); List lowerTargets = getTargetsInScope(lowerScope); if (!lowerTargets.isEmpty()) { - double lowerAvg = lowerTargets.stream().mapToDouble((key) -> load.get(key)).average().getAsDouble(); + double lowerAvg = lowerTargets.stream().mapToDouble((key) -> load.get(key)) + .average().getAsDouble(); if (lowerAvg < lowerBound) { nextScope = lowerScope; } @@ -183,29 +197,31 @@ private synchronized void updateRing(LoadMapping load) { LocalityScope prevScope = currentScope; currentScope = transition(load); if (currentScope != prevScope) { - //reset all the weights + // reset all the weights orig.values().stream().forEach(o -> o.resetWeight()); } List targetsInScope = getTargetsInScope(currentScope); - //We will adjust weights based off of the minimum load - double min = load == null ? 0 : targetsInScope.stream().mapToDouble((key) -> load.get(key)).min().getAsDouble(); + // We will adjust weights based off of the minimum load + double min = load == null ? 0 : targetsInScope.stream().mapToDouble((key) -> load.get(key)) + .min().getAsDouble(); for (int target : targetsInScope) { IndexAndWeights val = orig.get(target); double l = load == null ? 0.0 : load.get(target); if (l <= min + (0.05)) { - //We assume that within 5% of the minimum congestion is still fine. - //Not congested we grow (but slowly) + // We assume that within 5% of the minimum congestion is still fine. + // Not congested we grow (but slowly) val.weight = Math.min(MAX_WEIGHT, val.weight + 1); } else { - //Congested we contract much more quickly + // Congested we contract much more quickly val.weight = Math.max(0, val.weight - 10); } } - //Now we need to build the array - long weightSum = targetsInScope.stream().mapToLong((target) -> orig.get(target).weight).sum(); - //Now we can calculate a percentage + // Now we need to build the array + long weightSum = targetsInScope.stream().mapToLong((target) -> orig.get(target).weight) + .sum(); + // Now we can calculate a percentage int currentIdx = 0; if (weightSum > 0) { @@ -219,14 +235,15 @@ private synchronized void updateRing(LoadMapping load) { } if (currentIdx > 0) { - //in case we didn't fill in enough + // in case we didn't fill in enough for (; currentIdx < capacity; currentIdx++) { prepareChoices[currentIdx] = prepareChoices[random.nextInt(currentIdx)]; } } } if (currentIdx == 0) { - //This really should be impossible, because we go off of the min load, and inc anything within 5% of it. + // This really should be impossible, because we go off of the min load, and inc anything + // within 5% of it. // But just to be sure it is never an issue, especially with float rounding etc. for (; currentIdx < capacity; currentIdx++) { prepareChoices[currentIdx] = currentIdx % rets.length; @@ -256,7 +273,8 @@ private void swap(int[] arr, int i, int j) { arr[j] = tmp; } - private LocalityScope calculateScope(Map taskToNodePort, Map nodeToHost, + private LocalityScope calculateScope(Map taskToNodePort, Map nodeToHost, Map hostToRack, int target) { NodeInfo targetNodeInfo = taskToNodePort.get(target); @@ -284,11 +302,12 @@ private LocalityScope calculateScope(Map taskToNodePort, Map< } } - private Map getHostToRackMapping(Map taskToNodePort, Map nodeToHost) { + private Map getHostToRackMapping(Map taskToNodePort, + Map nodeToHost) { Set hosts = new HashSet<>(); for (int task : targetTasks) { - //if this task containing worker will be killed by a assignments sync, - //taskToNodePort will be an empty map which is refreshed by WorkerState + // if this task containing worker will be killed by a assignments sync, + // taskToNodePort will be an empty map which is refreshed by WorkerState if (taskToNodePort.containsKey(task)) { String node = taskToNodePort.get(task).get_node(); String hostname = nodeToHost.get(node); @@ -308,7 +327,7 @@ private Map getHostToRackMapping(Map taskToNo return dnsToSwitchMapping.resolve(new ArrayList<>(hosts)); } - //only for test + // only for test public int getCapacity() { return capacity; } diff --git a/storm-client/src/jvm/org/apache/storm/grouping/LoadMapping.java b/storm-client/src/jvm/org/apache/storm/grouping/LoadMapping.java index 569d1c4402e..cd5bb024a49 100644 --- a/storm-client/src/jvm/org/apache/storm/grouping/LoadMapping.java +++ b/storm-client/src/jvm/org/apache/storm/grouping/LoadMapping.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -21,8 +27,10 @@ */ public class LoadMapping { private static final Load NOT_CONNECTED = new Load(false, 1.0, 1.0); - private final AtomicReference> local = new AtomicReference>(new HashMap()); - private final AtomicReference> remote = new AtomicReference>(new HashMap()); + private final AtomicReference> local = + new AtomicReference>(new HashMap()); + private final AtomicReference> remote = + new AtomicReference>(new HashMap()); public void setLocal(Map local) { Map newLocal = new HashMap(); diff --git a/storm-client/src/jvm/org/apache/storm/grouping/PartialKeyGrouping.java b/storm-client/src/jvm/org/apache/storm/grouping/PartialKeyGrouping.java index 1c6c212ba12..81044ba7a72 100644 --- a/storm-client/src/jvm/org/apache/storm/grouping/PartialKeyGrouping.java +++ b/storm-client/src/jvm/org/apache/storm/grouping/PartialKeyGrouping.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -25,13 +31,17 @@ import org.apache.storm.tuple.Fields; /** - * A variation on FieldGrouping. This grouping operates on a partitioning of the incoming tuples (like a FieldGrouping), + * A variation on FieldGrouping. This grouping operates on a partitioning of the incoming tuples + * (like a FieldGrouping), * but it can send Tuples from a given partition to multiple downstream tasks. * - *

    Given a total pool of target tasks, this grouping will always send Tuples with a given key to one member of a - * subset of those tasks. Each key is assigned a subset of tasks. Each tuple is then sent to one task from that subset. + *

    Given a total pool of target tasks, this grouping will always send Tuples with a given key to + * one member of a + * subset of those tasks. Each key is assigned a subset of tasks. Each tuple is then sent to one + * task from that subset. * - *

    Notes: - the default TaskSelector ensures each task gets as close to a balanced number of Tuples as possible - the + *

    Notes: - the default TaskSelector ensures each task gets as close to a balanced number of + * Tuples as possible - the * default AssignmentCreator hashes the key and produces an assignment of two tasks */ public class PartialKeyGrouping implements CustomStreamGrouping, Serializable { @@ -55,14 +65,16 @@ public PartialKeyGrouping(Fields fields, AssignmentCreator assignmentCreator) { this(fields, assignmentCreator, new BalancedTargetSelector()); } - public PartialKeyGrouping(Fields fields, AssignmentCreator assignmentCreator, TargetSelector targetSelector) { + public PartialKeyGrouping(Fields fields, AssignmentCreator assignmentCreator, + TargetSelector targetSelector) { this.fields = fields; this.assignmentCreator = assignmentCreator; this.targetSelector = targetSelector; } @Override - public void prepare(WorkerTopologyContext context, GlobalStreamId stream, List targetTasks) { + public void prepare(WorkerTopologyContext context, GlobalStreamId stream, + List targetTasks) { this.targetTasks = targetTasks; if (this.fields != null) { this.outFields = context.getComponentOutputFields(stream); @@ -75,7 +87,8 @@ public List chooseTasks(int taskId, List values) { if (values.size() > 0) { final byte[] rawKeyBytes = getKeyBytes(values); - final int[] taskAssignmentForKey = assignmentCreator.createAssignment(this.targetTasks, rawKeyBytes); + final int[] taskAssignmentForKey = assignmentCreator.createAssignment(this.targetTasks, + rawKeyBytes); final int selectedTask = targetSelector.chooseTask(taskAssignmentForKey); boltIds.add(selectedTask); @@ -83,7 +96,6 @@ public List chooseTasks(int taskId, List values) { return boltIds; } - /** * Extract the key from the input Tuple. */ @@ -129,10 +141,13 @@ private byte[] getKeyBytes(List values) { // Helper Classes /** - * This interface is responsible for choosing a subset of the target tasks to use for a given key. + * This interface is responsible for choosing a subset of the target tasks to use for a given + * key. * - *

    NOTE: whatever scheme you use to create the assignment should be deterministic. This may be executed on - * multiple Storm Workers, thus each of them needs to come up with the same assignment for a given key. + *

    NOTE: whatever scheme you use to create the assignment should be deterministic. This may + * be executed on + * multiple Storm Workers, thus each of them needs to come up with the same assignment for a + * given key. */ public interface AssignmentCreator extends Serializable { int[] createAssignment(List targetTasks, byte[] key); @@ -156,7 +171,8 @@ public static class RandomTwoTaskAssignmentCreator implements AssignmentCreator */ @Override public int[] createAssignment(List tasks, byte[] key) { - // It is necessary that this produce a deterministic assignment based on the key, so seed the Random from the key + // It is necessary that this produce a deterministic assignment based on the key, so + // seed the Random from the key final long seedForRandom = Arrays.hashCode(key); final Random random = new Random(seedForRandom); final int choice1 = random.nextInt(tasks.size()); @@ -168,14 +184,16 @@ public int[] createAssignment(List tasks, byte[] key) { } /** - * A basic implementation of target selection. This strategy chooses the task within the assignment that has received the fewest Tuples + * A basic implementation of target selection. This strategy chooses the task within the + * assignment that has received the fewest Tuples * overall from this instance of the grouping. */ public static class BalancedTargetSelector implements TargetSelector { private Map targetTaskStats = Maps.newHashMap(); /** - * Chooses one of the incoming tasks and selects the one that has been selected the fewest times so far. + * Chooses one of the incoming tasks and selects the one that has been selected the fewest + * times so far. */ @Override public Integer chooseTask(int[] assignedTasks) { @@ -190,7 +208,8 @@ public Integer chooseTask(int[] assignedTasks) { } } - targetTaskStats.put(taskIdWithMinLoad, targetTaskStats.getOrDefault(taskIdWithMinLoad, 0L) + 1); + targetTaskStats.put(taskIdWithMinLoad, targetTaskStats.getOrDefault(taskIdWithMinLoad, + 0L) + 1); return taskIdWithMinLoad; } } diff --git a/storm-client/src/jvm/org/apache/storm/grouping/ShuffleGrouping.java b/storm-client/src/jvm/org/apache/storm/grouping/ShuffleGrouping.java index 217eabcdba9..f4e71961a3b 100644 --- a/storm-client/src/jvm/org/apache/storm/grouping/ShuffleGrouping.java +++ b/storm-client/src/jvm/org/apache/storm/grouping/ShuffleGrouping.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -22,13 +28,13 @@ import org.apache.storm.generated.GlobalStreamId; import org.apache.storm.task.WorkerTopologyContext; - public class ShuffleGrouping implements CustomStreamGrouping, Serializable { private ArrayList> choices; private AtomicInteger current; @Override - public void prepare(WorkerTopologyContext context, GlobalStreamId stream, List targetTasks) { + public void prepare(WorkerTopologyContext context, GlobalStreamId stream, + List targetTasks) { choices = new ArrayList>(targetTasks.size()); for (Integer i : targetTasks) { choices.add(Arrays.asList(i)); diff --git a/storm-client/src/jvm/org/apache/storm/hooks/BaseTaskHook.java b/storm-client/src/jvm/org/apache/storm/hooks/BaseTaskHook.java index 5213e357507..b1a62c47264 100644 --- a/storm-client/src/jvm/org/apache/storm/hooks/BaseTaskHook.java +++ b/storm-client/src/jvm/org/apache/storm/hooks/BaseTaskHook.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/hooks/BaseWorkerHook.java b/storm-client/src/jvm/org/apache/storm/hooks/BaseWorkerHook.java index 8e8e4483332..cf505445e05 100644 --- a/storm-client/src/jvm/org/apache/storm/hooks/BaseWorkerHook.java +++ b/storm-client/src/jvm/org/apache/storm/hooks/BaseWorkerHook.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -17,7 +23,8 @@ import org.apache.storm.task.WorkerTopologyContext; /** - * A BaseWorkerHook is a noop implementation of IWorkerHook. You may extends this class and implement any and/or all + * A BaseWorkerHook is a noop implementation of IWorkerHook. You may extends this class and + * implement any and/or all * methods you need for your workers. */ public class BaseWorkerHook implements IWorkerHook, Serializable { diff --git a/storm-client/src/jvm/org/apache/storm/hooks/ITaskHook.java b/storm-client/src/jvm/org/apache/storm/hooks/ITaskHook.java index ba0310820b3..9e74e53c889 100644 --- a/storm-client/src/jvm/org/apache/storm/hooks/ITaskHook.java +++ b/storm-client/src/jvm/org/apache/storm/hooks/ITaskHook.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/hooks/IWorkerHook.java b/storm-client/src/jvm/org/apache/storm/hooks/IWorkerHook.java index 61719c1b0e3..034bcf230ca 100644 --- a/storm-client/src/jvm/org/apache/storm/hooks/IWorkerHook.java +++ b/storm-client/src/jvm/org/apache/storm/hooks/IWorkerHook.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -18,18 +24,23 @@ import org.apache.storm.task.WorkerUserContext; /** - * An IWorkerHook represents a topology component that can be executed when a worker starts, and when a worker shuts down. It can be useful - * when you want to execute operations before topology processing starts, or cleanup operations before your workers shut down. + * An IWorkerHook represents a topology component that can be executed when a worker starts, and + * when a worker shuts down. It can be useful + * when you want to execute operations before topology processing starts, or cleanup operations + * before your workers shut down. */ public interface IWorkerHook extends Serializable { /** - * This method is called when a worker is started and can be used to do necessary prep-processing and allow initialization of shared + * This method is called when a worker is started and can be used to do necessary + * prep-processing and allow initialization of shared * application state. * * @param topoConf The Storm configuration for this worker - * @param context This object can be used to get information about this worker's place within the topology and exposes - * {@link WorkerUserContext#setResource(String, Object)} to set the shared application state. + * @param context This object can be used to get information about this worker's place within + * the topology and exposes + * {@link WorkerUserContext#setResource(String, Object)} to set the shared application + * state. */ default void start(Map topoConf, WorkerUserContext context) { // NOOP diff --git a/storm-client/src/jvm/org/apache/storm/hooks/SubmitterHookException.java b/storm-client/src/jvm/org/apache/storm/hooks/SubmitterHookException.java index 08f2be8e297..2a764778199 100644 --- a/storm-client/src/jvm/org/apache/storm/hooks/SubmitterHookException.java +++ b/storm-client/src/jvm/org/apache/storm/hooks/SubmitterHookException.java @@ -20,7 +20,8 @@ package org.apache.storm.hooks; /** - * This Exception is thrown when registered {@link org.apache.storm.ISubmitterHook} could not be initialized or invoked. + * This Exception is thrown when registered {@link org.apache.storm.ISubmitterHook} could not be + * initialized or invoked. */ public class SubmitterHookException extends RuntimeException { diff --git a/storm-client/src/jvm/org/apache/storm/hooks/info/BoltAckInfo.java b/storm-client/src/jvm/org/apache/storm/hooks/info/BoltAckInfo.java index 3bfd501abd7..35aa457d2d7 100644 --- a/storm-client/src/jvm/org/apache/storm/hooks/info/BoltAckInfo.java +++ b/storm-client/src/jvm/org/apache/storm/hooks/info/BoltAckInfo.java @@ -1,19 +1,24 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.hooks.info; import java.util.List; - import org.apache.storm.hooks.ITaskHook; import org.apache.storm.task.TopologyContext; import org.apache.storm.tuple.Tuple; diff --git a/storm-client/src/jvm/org/apache/storm/hooks/info/BoltExecuteInfo.java b/storm-client/src/jvm/org/apache/storm/hooks/info/BoltExecuteInfo.java index 9c5659eaf79..3fd468af3db 100644 --- a/storm-client/src/jvm/org/apache/storm/hooks/info/BoltExecuteInfo.java +++ b/storm-client/src/jvm/org/apache/storm/hooks/info/BoltExecuteInfo.java @@ -1,19 +1,24 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.hooks.info; import java.util.List; - import org.apache.storm.hooks.ITaskHook; import org.apache.storm.task.TopologyContext; import org.apache.storm.tuple.Tuple; diff --git a/storm-client/src/jvm/org/apache/storm/hooks/info/BoltFailInfo.java b/storm-client/src/jvm/org/apache/storm/hooks/info/BoltFailInfo.java index 59c7e195c17..3328714fadf 100644 --- a/storm-client/src/jvm/org/apache/storm/hooks/info/BoltFailInfo.java +++ b/storm-client/src/jvm/org/apache/storm/hooks/info/BoltFailInfo.java @@ -1,19 +1,24 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.hooks.info; import java.util.List; - import org.apache.storm.hooks.ITaskHook; import org.apache.storm.task.TopologyContext; import org.apache.storm.tuple.Tuple; diff --git a/storm-client/src/jvm/org/apache/storm/hooks/info/EmitInfo.java b/storm-client/src/jvm/org/apache/storm/hooks/info/EmitInfo.java index 35d225633d9..6cf0043dc35 100644 --- a/storm-client/src/jvm/org/apache/storm/hooks/info/EmitInfo.java +++ b/storm-client/src/jvm/org/apache/storm/hooks/info/EmitInfo.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/hooks/info/SpoutAckInfo.java b/storm-client/src/jvm/org/apache/storm/hooks/info/SpoutAckInfo.java index ac053beb80e..e4ba6a60723 100644 --- a/storm-client/src/jvm/org/apache/storm/hooks/info/SpoutAckInfo.java +++ b/storm-client/src/jvm/org/apache/storm/hooks/info/SpoutAckInfo.java @@ -1,19 +1,24 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.hooks.info; import java.util.List; - import org.apache.storm.hooks.ITaskHook; import org.apache.storm.task.TopologyContext; diff --git a/storm-client/src/jvm/org/apache/storm/hooks/info/SpoutFailInfo.java b/storm-client/src/jvm/org/apache/storm/hooks/info/SpoutFailInfo.java index df7ef294947..ea0335530e2 100644 --- a/storm-client/src/jvm/org/apache/storm/hooks/info/SpoutFailInfo.java +++ b/storm-client/src/jvm/org/apache/storm/hooks/info/SpoutFailInfo.java @@ -1,19 +1,24 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.hooks.info; import java.util.List; - import org.apache.storm.hooks.ITaskHook; import org.apache.storm.task.TopologyContext; diff --git a/storm-client/src/jvm/org/apache/storm/lambda/LambdaBiConsumerBolt.java b/storm-client/src/jvm/org/apache/storm/lambda/LambdaBiConsumerBolt.java index 4035f0855c9..2a1cb05b00a 100644 --- a/storm-client/src/jvm/org/apache/storm/lambda/LambdaBiConsumerBolt.java +++ b/storm-client/src/jvm/org/apache/storm/lambda/LambdaBiConsumerBolt.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -24,7 +30,8 @@ public class LambdaBiConsumerBolt extends BaseBasicBolt { private String[] fields; - public LambdaBiConsumerBolt(SerializableBiConsumer biConsumer, String[] fields) { + public LambdaBiConsumerBolt(SerializableBiConsumer biConsumer, + String[] fields) { this.biConsumer = biConsumer; this.fields = fields; } diff --git a/storm-client/src/jvm/org/apache/storm/lambda/LambdaConsumerBolt.java b/storm-client/src/jvm/org/apache/storm/lambda/LambdaConsumerBolt.java index 6dc94c8b2a7..95f432dfdb4 100644 --- a/storm-client/src/jvm/org/apache/storm/lambda/LambdaConsumerBolt.java +++ b/storm-client/src/jvm/org/apache/storm/lambda/LambdaConsumerBolt.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/lambda/LambdaSpout.java b/storm-client/src/jvm/org/apache/storm/lambda/LambdaSpout.java index bec9d467447..c67571a8caf 100644 --- a/storm-client/src/jvm/org/apache/storm/lambda/LambdaSpout.java +++ b/storm-client/src/jvm/org/apache/storm/lambda/LambdaSpout.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -29,7 +35,8 @@ public LambdaSpout(SerializableSupplier supplier) { } @Override - public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { + public void open(Map conf, TopologyContext context, + SpoutOutputCollector collector) { this.collector = collector; } diff --git a/storm-client/src/jvm/org/apache/storm/lambda/SerializableBiConsumer.java b/storm-client/src/jvm/org/apache/storm/lambda/SerializableBiConsumer.java index c311be8178e..501a1e338f1 100644 --- a/storm-client/src/jvm/org/apache/storm/lambda/SerializableBiConsumer.java +++ b/storm-client/src/jvm/org/apache/storm/lambda/SerializableBiConsumer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/lambda/SerializableCallable.java b/storm-client/src/jvm/org/apache/storm/lambda/SerializableCallable.java index 905c4980bb9..68cd6fbf5ce 100644 --- a/storm-client/src/jvm/org/apache/storm/lambda/SerializableCallable.java +++ b/storm-client/src/jvm/org/apache/storm/lambda/SerializableCallable.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/lambda/SerializableConsumer.java b/storm-client/src/jvm/org/apache/storm/lambda/SerializableConsumer.java index 224edf9b278..2389d52d920 100644 --- a/storm-client/src/jvm/org/apache/storm/lambda/SerializableConsumer.java +++ b/storm-client/src/jvm/org/apache/storm/lambda/SerializableConsumer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/lambda/SerializableSupplier.java b/storm-client/src/jvm/org/apache/storm/lambda/SerializableSupplier.java index 410db38cccd..6f366548e32 100644 --- a/storm-client/src/jvm/org/apache/storm/lambda/SerializableSupplier.java +++ b/storm-client/src/jvm/org/apache/storm/lambda/SerializableSupplier.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/messaging/ConnectionWithStatus.java b/storm-client/src/jvm/org/apache/storm/messaging/ConnectionWithStatus.java index f6823357fb9..ad8a5096144 100644 --- a/storm-client/src/jvm/org/apache/storm/messaging/ConnectionWithStatus.java +++ b/storm-client/src/jvm/org/apache/storm/messaging/ConnectionWithStatus.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -15,15 +21,17 @@ public abstract class ConnectionWithStatus implements IConnection { /** - * whether this connection is available to transfer data. + * Whether this connection is available to transfer data. */ public abstract Status status(); public enum Status { /** - * we are establishing a active connection with target host. The new data sending request can be buffered for future sending, or - * dropped(cases like there is no enough memory). It varies with difference IConnection implementations. + * We are establishing a active connection with target host. The new data sending request + * can be buffered for future sending, or + * dropped(cases like there is no enough memory). It varies with difference IConnection + * implementations. */ Connecting, @@ -33,7 +41,8 @@ public enum Status { Ready, /** - * The connection channel is closed or being closed. We don't accept further data sending or receiving. All data sending request + * The connection channel is closed or being closed. We don't accept further data sending or + * receiving. All data sending request * will be dropped. */ Closed diff --git a/storm-client/src/jvm/org/apache/storm/messaging/DeserializingConnectionCallback.java b/storm-client/src/jvm/org/apache/storm/messaging/DeserializingConnectionCallback.java index 6a8464f432c..5f88dafe16d 100644 --- a/storm-client/src/jvm/org/apache/storm/messaging/DeserializingConnectionCallback.java +++ b/storm-client/src/jvm/org/apache/storm/messaging/DeserializingConnectionCallback.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -35,16 +41,18 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; - /** * A class that is called when a TaskMessage arrives. */ public class DeserializingConnectionCallback implements IConnectionCallback, IMetric { - private static final Logger LOG = LoggerFactory.getLogger(DeserializingConnectionCallback.class); + private static final Logger LOG = LoggerFactory + .getLogger(DeserializingConnectionCallback.class); - // A tuple that cannot be decoded is dropped instead of killing the worker; anything outside this set keeps + // A tuple that cannot be decoded is dropped instead of killing the worker; anything outside + // this set keeps // the fatal handling in StormServerHandler. - private static final Set> TOLERATED_DESERIALIZATION_FAILURES = new HashSet<>(Arrays.asList( + private static final Set> TOLERATED_DESERIALIZATION_FAILURES = new HashSet<>(Arrays + .asList( IOException.class, KryoException.class, IllegalArgumentException.class, @@ -81,12 +89,14 @@ protected KryoTupleDeserializer initialValue() { private final AtomicLong totalDropCount = new AtomicLong(0L); private final AtomicLong consecutiveDropCount = new AtomicLong(0L); - public DeserializingConnectionCallback(final Map conf, final GeneralTopologyContext context, + public DeserializingConnectionCallback(final Map conf, + final GeneralTopologyContext context, WorkerState.ILocalTransferCallback callback) { this.conf = conf; this.context = context; cb = callback; - sizeMetricsEnabled = ObjectReader.getBoolean(conf.get(Config.TOPOLOGY_SERIALIZED_MESSAGE_SIZE_METRICS), false); + sizeMetricsEnabled = ObjectReader.getBoolean(conf + .get(Config.TOPOLOGY_SERIALIZED_MESSAGE_SIZE_METRICS), false); } @@ -110,7 +120,8 @@ public void recv(List batch) { deserializationFailures.incrementAndGet(); long totalDrops = totalDropCount.incrementAndGet(); if (totalDrops <= INDIVIDUAL_DROP_LOG_LIMIT) { - LOG.error("Failed to deserialize a message of {} bytes destined for task {}, dropping it", + LOG.error("Failed to deserialize a message of {} bytes destined for task {}, " + + "dropping it", message.message().length, message.task(), e); } else if ((totalDrops - INDIVIDUAL_DROP_LOG_LIMIT) % DROP_LOG_SUMMARY_INTERVAL == 0) { diff --git a/storm-client/src/jvm/org/apache/storm/messaging/IConnection.java b/storm-client/src/jvm/org/apache/storm/messaging/IConnection.java index 11e98acff62..a96d6890961 100644 --- a/storm-client/src/jvm/org/apache/storm/messaging/IConnection.java +++ b/storm-client/src/jvm/org/apache/storm/messaging/IConnection.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -34,7 +40,7 @@ public interface IConnection extends AutoCloseable { void sendBackPressureStatus(BackPressureStatus bpStatus); /** - * send batch messages. + * Send batch messages. */ void send(Iterator msgs); @@ -55,7 +61,7 @@ public interface IConnection extends AutoCloseable { int getPort(); /** - * close this connection. + * Close this connection. */ @Override void close(); diff --git a/storm-client/src/jvm/org/apache/storm/messaging/IConnectionCallback.java b/storm-client/src/jvm/org/apache/storm/messaging/IConnectionCallback.java index 4ccda6df1fd..0d48abcc260 100644 --- a/storm-client/src/jvm/org/apache/storm/messaging/IConnectionCallback.java +++ b/storm-client/src/jvm/org/apache/storm/messaging/IConnectionCallback.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/messaging/IContext.java b/storm-client/src/jvm/org/apache/storm/messaging/IContext.java index 58b09752bfd..d3aa61d5c6b 100644 --- a/storm-client/src/jvm/org/apache/storm/messaging/IContext.java +++ b/storm-client/src/jvm/org/apache/storm/messaging/IContext.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -22,7 +28,8 @@ * *

    Messaging plugin is specified via Storm config parameter, storm.messaging.transport. * - *

    A messaging plugin should have a default constructor and implements IContext interface. Upon construction, we will invoke + *

    A messaging plugin should have a default constructor and implements IContext interface. Upon + * construction, we will invoke * IContext::prepare(topoConf) to enable context to be configured according to storm configuration. */ public interface IContext { @@ -55,11 +62,13 @@ default void prepare(Map topoConf, StormMetricRegistry metricReg * @param stormId topology ID * @param port port # * @param cb The callback to deliver received messages to - * @param newConnectionResponse Supplier of the initial message to send to new client connections. If authentication + * @param newConnectionResponse Supplier of the initial message to send to new client + * connections. If authentication * is required, the message will be sent after authentication is complete. * @return server side connection */ - IConnection bind(String stormId, int port, IConnectionCallback cb, Supplier newConnectionResponse); + IConnection bind(String stormId, int port, IConnectionCallback cb, + Supplier newConnectionResponse); /** * This method establish a client side connection to a remote server diff --git a/storm-client/src/jvm/org/apache/storm/messaging/TaskMessage.java b/storm-client/src/jvm/org/apache/storm/messaging/TaskMessage.java index 9c2975d66fe..1bfa26aedb2 100644 --- a/storm-client/src/jvm/org/apache/storm/messaging/TaskMessage.java +++ b/storm-client/src/jvm/org/apache/storm/messaging/TaskMessage.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/messaging/TransportFactory.java b/storm-client/src/jvm/org/apache/storm/messaging/TransportFactory.java index fde5344c0d7..9c208d8a54a 100644 --- a/storm-client/src/jvm/org/apache/storm/messaging/TransportFactory.java +++ b/storm-client/src/jvm/org/apache/storm/messaging/TransportFactory.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -22,32 +28,35 @@ public class TransportFactory { public static final Logger LOG = LoggerFactory.getLogger(TransportFactory.class); - public static IContext makeContext(Map topoConf, StormMetricRegistry metricRegistry) { + public static IContext makeContext(Map topoConf, + StormMetricRegistry metricRegistry) { - //get factory class name + // get factory class name String transportPluginClassName = (String) topoConf.get(Config.STORM_MESSAGING_TRANSPORT); LOG.info("Storm peer transport plugin:" + transportPluginClassName); IContext transport; try { - //create a factory class + // create a factory class Class klass = Class.forName(transportPluginClassName); - //obtain a context object + // obtain a context object Object obj = klass.newInstance(); if (obj instanceof IContext) { - //case 1: plugin is a IContext class + // case 1: plugin is a IContext class transport = (IContext) obj; - //initialize with storm configuration + // initialize with storm configuration transport.prepare(topoConf, metricRegistry); } else { - //case 2: Non-IContext plugin must have a makeContext(topoConf) method that returns IContext object + // case 2: Non-IContext plugin must have a makeContext(topoConf) method that returns + // IContext object // StormMetricRegistry is ignored if IContext is created this way Method method = klass.getMethod("makeContext", Map.class); LOG.debug("object:" + obj + " method:" + method); transport = (IContext) method.invoke(obj, topoConf); } } catch (Exception e) { - throw new RuntimeException("Fail to construct messaging plugin from plugin " + transportPluginClassName, e); + throw new RuntimeException("Fail to construct messaging plugin from plugin " + + transportPluginClassName, e); } return transport; } diff --git a/storm-client/src/jvm/org/apache/storm/messaging/local/Context.java b/storm-client/src/jvm/org/apache/storm/messaging/local/Context.java index 67ab6ddfda9..f463e19061c 100644 --- a/storm-client/src/jvm/org/apache/storm/messaging/local/Context.java +++ b/storm-client/src/jvm/org/apache/storm/messaging/local/Context.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -48,7 +54,7 @@ private LocalServer createLocalServer(String nodeId, int port, IConnectionCallba LocalServer ret = new LocalServer(port, cb); LocalServer existing = registry.put(key, ret); if (existing != null) { - //Can happen if worker is restarted in the same topology, e.g. due to blob update + // Can happen if worker is restarted in the same topology, e.g. due to blob update LOG.info("Replacing existing server for key {}", existing, ret, key); } return ret; @@ -56,22 +62,24 @@ private LocalServer createLocalServer(String nodeId, int port, IConnectionCallba @Override public void prepare(Map topoConf) { - //NOOP + // NOOP } @Override - public IConnection bind(String stormId, int port, IConnectionCallback cb, Supplier newConnectionResponse) { + public IConnection bind(String stormId, int port, IConnectionCallback cb, + Supplier newConnectionResponse) { return createLocalServer(stormId, port, cb); } @Override - public IConnection connect(String stormId, String host, int port, AtomicBoolean[] remoteBpStatus) { + public IConnection connect(String stormId, String host, int port, + AtomicBoolean[] remoteBpStatus) { return new LocalClient(stormId, port); } @Override public void term() { - //NOOP + // NOOP } private class LocalServer implements IConnection { @@ -108,7 +116,8 @@ public void sendLoadMetrics(Map taskToLoad) { @Override public void sendBackPressureStatus(BackPressureStatus bpStatus) { - throw new RuntimeException("Local Server connection should not send BackPressure status"); + throw new RuntimeException("Local Server connection should not send BackPressure " + + "status"); } @Override @@ -118,12 +127,12 @@ public int getPort() { @Override public void close() { - //NOOP + // NOOP } } private class LocalClient implements IConnection { - //Messages sent before the server registered a callback + // Messages sent before the server registered a callback private final LinkedBlockingQueue pendingDueToUnregisteredServer; private final ScheduledExecutorService pendingFlusher; private final int port; @@ -146,10 +155,11 @@ public Thread newThread(Runnable runnable) { @Override public void run() { try { - //Ensure messages are flushed even if no more sends are performed + // Ensure messages are flushed even if no more sends are performed flushPending(); } catch (Throwable t) { - LOG.error("Uncaught throwable in pending message flusher thread, messages may be lost", t); + LOG.error("Uncaught throwable in pending message flusher thread, messages " + + "may be lost", t); throw new RuntimeException(t); } } @@ -157,7 +167,7 @@ public void run() { } private void flushPending() { - //Can't cache server in client, server can change when workers restart. + // Can't cache server in client, server can change when workers restart. LocalServer server = registry.get(registryKey); if (server != null && !pendingDueToUnregisteredServer.isEmpty()) { ArrayList ret = new ArrayList<>(); @@ -202,7 +212,8 @@ public void sendLoadMetrics(Map taskToLoad) { @Override public void sendBackPressureStatus(BackPressureStatus bpStatus) { - throw new RuntimeException("Local Client connection should not send BackPressure status"); + throw new RuntimeException("Local Client connection should not send BackPressure " + + "status"); } @Override diff --git a/storm-client/src/jvm/org/apache/storm/messaging/netty/BackPressureStatus.java b/storm-client/src/jvm/org/apache/storm/messaging/netty/BackPressureStatus.java index 5ead7405d27..2f9540fae11 100644 --- a/storm-client/src/jvm/org/apache/storm/messaging/netty/BackPressureStatus.java +++ b/storm-client/src/jvm/org/apache/storm/messaging/netty/BackPressureStatus.java @@ -26,7 +26,8 @@ import org.apache.storm.shade.io.netty.buffer.ByteBuf; import org.apache.storm.shade.io.netty.buffer.ByteBufAllocator; -// Instances of this type are sent from NettyWorker to upstream WorkerTransfer to indicate BackPressure situation +// Instances of this type are sent from NettyWorker to upstream WorkerTransfer to indicate +// BackPressure situation public class BackPressureStatus { public static final short IDENTIFIER = (short) -600; private static final int SIZE_OF_ID = 2; // size if IDENTIFIER @@ -45,7 +46,8 @@ public BackPressureStatus() { /** * Constructor. */ - public BackPressureStatus(String workerId, Collection bpTasks, Collection nonBpTasks) { + public BackPressureStatus(String workerId, Collection bpTasks, + Collection nonBpTasks) { this.workerId = workerId; this.id = bpCount.incrementAndGet(); this.bpTasks = bpTasks; @@ -58,7 +60,8 @@ public static BackPressureStatus read(byte[] bytes, KryoValuesDeserializer deser @Override public String toString() { - return "{worker=" + workerId + ", bpStatusId=" + id + ", bpTasks=" + bpTasks + ", nonBpTasks=" + nonBpTasks + '}'; + return "{worker=" + workerId + ", bpStatusId=" + id + ", bpTasks=" + bpTasks + + ", nonBpTasks=" + nonBpTasks + '}'; } /** diff --git a/storm-client/src/jvm/org/apache/storm/messaging/netty/BackPressureStatusEncoder.java b/storm-client/src/jvm/org/apache/storm/messaging/netty/BackPressureStatusEncoder.java index be4fff9a724..e7b6b1f1929 100644 --- a/storm-client/src/jvm/org/apache/storm/messaging/netty/BackPressureStatusEncoder.java +++ b/storm-client/src/jvm/org/apache/storm/messaging/netty/BackPressureStatusEncoder.java @@ -30,7 +30,8 @@ public BackPressureStatusEncoder(KryoValuesSerializer ser) { } @Override - protected void encode(ChannelHandlerContext ctx, BackPressureStatus msg, List out) throws Exception { + protected void encode(ChannelHandlerContext ctx, BackPressureStatus msg, + List out) throws Exception { out.add(msg.buffer(ctx.alloc(), ser)); } diff --git a/storm-client/src/jvm/org/apache/storm/messaging/netty/Client.java b/storm-client/src/jvm/org/apache/storm/messaging/netty/Client.java index dd579bb6332..87b16ce72ad 100644 --- a/storm-client/src/jvm/org/apache/storm/messaging/netty/Client.java +++ b/storm-client/src/jvm/org/apache/storm/messaging/netty/Client.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -37,8 +43,8 @@ import org.apache.storm.messaging.ConnectionWithStatus; import org.apache.storm.messaging.TaskMessage; import org.apache.storm.metrics2.StormMetricRegistry; -import org.apache.storm.policy.IWaitStrategy; import org.apache.storm.policy.IWaitStrategy.WaitSituation; +import org.apache.storm.policy.IWaitStrategy; import org.apache.storm.policy.WaitStrategyProgressive; import org.apache.storm.shade.io.netty.bootstrap.Bootstrap; import org.apache.storm.shade.io.netty.buffer.PooledByteBufAllocator; @@ -64,9 +70,12 @@ * *

    Implementation details: * - *

    Sending messages, i.e. writing to the channel, is performed asynchronously. Messages are sent in batches to optimize for network - * throughput at the expense of network latency. The message batch size is configurable. Connecting and reconnecting are performed - * asynchronously. Note: The current implementation drops any messages that are being enqueued for sending if the connection to the remote + *

    Sending messages, i.e. writing to the channel, is performed asynchronously. Messages are sent + * in batches to optimize for network + * throughput at the expense of network latency. The message batch size is configurable. Connecting + * and reconnecting are performed + * asynchronously. Note: The current implementation drops any messages that are being enqueued for + * sending if the connection to the remote * destination is currently unavailable. */ public class Client extends ConnectionWithStatus implements ISaslClient { @@ -135,16 +144,24 @@ public class Client extends ConnectionWithStatus implements ISaslClient { this.topoConf = topoConf; closing = false; this.scheduler = scheduler; - int bufferSize = ObjectReader.getInt(topoConf.get(Config.STORM_MESSAGING_NETTY_BUFFER_SIZE)); - int lowWatermark = ObjectReader.getInt(topoConf.get(Config.STORM_MESSAGING_NETTY_BUFFER_LOW_WATERMARK)); - int highWatermark = ObjectReader.getInt(topoConf.get(Config.STORM_MESSAGING_NETTY_BUFFER_HIGH_WATERMARK)); - // if SASL authentication is disabled, saslChannelReady is initialized as true; otherwise false - saslChannelReady.set(!ObjectReader.getBoolean(topoConf.get(Config.STORM_MESSAGING_NETTY_AUTHENTICATION), false)); - LOG.info("Creating Netty Client, connecting to {}:{}, bufferSize: {}, lowWatermark: {}, highWatermark: {}", + int bufferSize = ObjectReader.getInt(topoConf + .get(Config.STORM_MESSAGING_NETTY_BUFFER_SIZE)); + int lowWatermark = ObjectReader.getInt(topoConf + .get(Config.STORM_MESSAGING_NETTY_BUFFER_LOW_WATERMARK)); + int highWatermark = ObjectReader.getInt(topoConf + .get(Config.STORM_MESSAGING_NETTY_BUFFER_HIGH_WATERMARK)); + // if SASL authentication is disabled, saslChannelReady is initialized as true; otherwise + // false + saslChannelReady.set(!ObjectReader.getBoolean(topoConf + .get(Config.STORM_MESSAGING_NETTY_AUTHENTICATION), false)); + LOG.info("Creating Netty Client, connecting to {}:{}, bufferSize: {}, lowWatermark: {}, " + + "highWatermark: {}", host, port, bufferSize, lowWatermark, highWatermark); - int minWaitMs = ObjectReader.getInt(topoConf.get(Config.STORM_MESSAGING_NETTY_MIN_SLEEP_MS)); - int maxWaitMs = ObjectReader.getInt(topoConf.get(Config.STORM_MESSAGING_NETTY_MAX_SLEEP_MS)); + int minWaitMs = ObjectReader.getInt(topoConf + .get(Config.STORM_MESSAGING_NETTY_MIN_SLEEP_MS)); + int maxWaitMs = ObjectReader.getInt(topoConf + .get(Config.STORM_MESSAGING_NETTY_MAX_SLEEP_MS)); retryPolicy = new StormBoundedExponentialBackoffRetry(minWaitMs, maxWaitMs, -1); SslContext sslContext = NettyTlsUtils.createSslContext(topoConf, false); @@ -158,16 +175,21 @@ public class Client extends ConnectionWithStatus implements ISaslClient { .option(ChannelOption.TCP_NODELAY, true) .option(ChannelOption.SO_SNDBUF, bufferSize) .option(ChannelOption.SO_KEEPALIVE, true) - .option(ChannelOption.WRITE_BUFFER_WATER_MARK, new WriteBufferWaterMark(lowWatermark, highWatermark)) + .option(ChannelOption.WRITE_BUFFER_WATER_MARK, new WriteBufferWaterMark(lowWatermark, + highWatermark)) .option(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT) - .handler(new StormClientPipelineFactory(this, remoteBpStatus, topoConf, sslContext, host, port)); + .handler(new StormClientPipelineFactory(this, remoteBpStatus, topoConf, sslContext, + host, port)); dstAddress = new InetSocketAddress(host, port); dstAddressPrefixedName = prefixedName(dstAddress); launchChannelAliveThread(); scheduleConnect(NO_DELAY_MS); - int messageBatchSize = ObjectReader.getInt(topoConf.get(Config.STORM_NETTY_MESSAGE_BATCH_SIZE), 262144); - pendingMessagesFlushTimeoutMs = ObjectReader.getLong(topoConf.get(Config.STORM_MESSAGING_NETTY_FLUSH_TIMEOUT_MS), 600000L); - pendingMessagesFlushIntervalMs = (long) (pendingMessagesFlushFactor * pendingMessagesFlushTimeoutMs); + int messageBatchSize = ObjectReader.getInt(topoConf + .get(Config.STORM_NETTY_MESSAGE_BATCH_SIZE), 262144); + pendingMessagesFlushTimeoutMs = ObjectReader.getLong(topoConf + .get(Config.STORM_MESSAGING_NETTY_FLUSH_TIMEOUT_MS), 600000L); + pendingMessagesFlushIntervalMs = + (long) (pendingMessagesFlushFactor * pendingMessagesFlushTimeoutMs); batcher = new MessageBuffer(messageBatchSize); String clazz = (String) topoConf.get(Config.TOPOLOGY_BACKPRESSURE_WAIT_STRATEGY); if (clazz == null) { @@ -178,9 +200,11 @@ public class Client extends ConnectionWithStatus implements ISaslClient { waitStrategy.prepare(topoConf, WaitSituation.BACK_PRESSURE_WAIT); this.metricRegistry = metricRegistry; - // it's possible to be passed a null metric registry if users are using their own IContext implementation. + // it's possible to be passed a null metric registry if users are using their own IContext + // implementation. boolean reportMetrics = this.metricRegistry != null - && ObjectReader.getBoolean(topoConf.get(Config.TOPOLOGY_ENABLE_SEND_ICONNECTION_METRICS), true); + && ObjectReader.getBoolean(topoConf + .get(Config.TOPOLOGY_ENABLE_SEND_ICONNECTION_METRICS), true); if (reportMetrics) { Gauge reconnects = new Gauge() { @@ -226,8 +250,10 @@ public Integer getValue() { } /** - * This thread helps us to check for channel connection periodically. This is performed just to know whether the destination address is - * alive or attempts to refresh connections if not alive. This solution is better than what we have now in case of a bad channel. + * This thread helps us to check for channel connection periodically. This is performed just to + * know whether the destination address is + * alive or attempts to refresh connections if not alive. This solution is better than what we + * have now in case of a bad channel. */ private void launchChannelAliveThread() { // netty TimerTask is already defined and hence a fully @@ -275,7 +301,8 @@ private boolean connectionEstablished(Channel channel) { } /** - * Note: Storm will check via this method whether a worker can be activated safely during the initial startup of a topology. The + * Note: Storm will check via this method whether a worker can be activated safely during the + * initial startup of a topology. The * worker will only be activated once all of the its connections are ready. */ @Override @@ -310,7 +337,8 @@ public void sendBackPressureStatus(BackPressureStatus bpStatus) { public void send(Iterator msgs) { if (closing) { int numMessages = iteratorSize(msgs); - LOG.error("Dropping {} messages because the Netty client to {} is being closed", numMessages, + LOG.error("Dropping {} messages because the Netty client to {} is being closed", + numMessages, dstAddressPrefixedName); return; } @@ -322,10 +350,13 @@ public void send(Iterator msgs) { Channel channel = getConnectedChannel(); if (channel == null) { /* - * Connection is unavailable. We will drop pending messages and let at-least-once message replay kick in. + * Connection is unavailable. We will drop pending messages and let at-least-once + * message replay kick in. * - * Another option would be to buffer the messages in memory. But this option has the risk of causing OOM errors, - * especially for topologies that disable message acking because we don't know whether the connection recovery will + * Another option would be to buffer the messages in memory. But this option has the + * risk of causing OOM errors, + * especially for topologies that disable message acking because we don't know whether + * the connection recovery will * succeed or not, and how long the recovery will take. */ dropMessages(msgs); @@ -431,7 +462,8 @@ public void operationComplete(ChannelFuture future) throws Exception { LOG.debug("sent {} messages to {}", numMessages, dstAddressPrefixedName); messagesSent.getAndAdd(batch.size()); } else { - LOG.error("failed to send {} messages to {}: {}", numMessages, dstAddressPrefixedName, + LOG.error("failed to send {} messages to {}: {}", numMessages, + dstAddressPrefixedName, future.cause()); closeChannelAndReconnect(future.channel()); messagesLost.getAndAdd(numMessages); @@ -442,7 +474,8 @@ public void operationComplete(ChannelFuture future) throws Exception { } /** - * Schedule a reconnect if we closed a non-null channel, and acquired the right to provide a replacement by successfully setting a null + * Schedule a reconnect if we closed a non-null channel, and acquired the right to provide a + * replacement by successfully setting a null * to the channel field. * * @param channel the channel to close @@ -492,7 +525,8 @@ private void waitForPendingMessagesToBeSent() { try { long deltaMs = System.currentTimeMillis() - startMs; if (deltaMs > pendingMessagesFlushTimeoutMs) { - LOG.error("failed to send all pending messages to {} within timeout, {} of {} messages were not " + LOG.error("failed to send all pending messages to {} within timeout, {} of {} " + + "messages were not " + "sent", dstAddressPrefixedName, pendingMessages.get(), totalPendingMsgs); break; } @@ -569,7 +603,8 @@ public String toString() { } /** - * Asynchronously establishes a Netty connection to the remote address This task runs on a single thread shared among all clients, and + * Asynchronously establishes a Netty connection to the remote address This task runs on a + * single thread shared among all clients, and * thus should not perform operations that block. */ private class Connect implements TimerTask { @@ -589,7 +624,6 @@ private void reschedule(Throwable t) { scheduleConnect(nextDelayMs); } - @Override public void run(Timeout timeout) throws Exception { if (reconnectingAllowed()) { @@ -607,10 +641,12 @@ public void operationComplete(ChannelFuture future) throws Exception { if (future.isSuccess() && connectionEstablished(newChannel)) { boolean setChannel = channelRef.compareAndSet(null, newChannel); checkState(setChannel); - LOG.debug("successfully connected to {}, {} [attempt {}]", address.toString(), newChannel.toString(), + LOG.debug("successfully connected to {}, {} [attempt {}]", address + .toString(), newChannel.toString(), connectionAttempt); if (messagesLost.get() > 0) { - LOG.warn("Re-connection to {} was successful but {} messages has been lost so far", address.toString(), + LOG.warn("Re-connection to {} was successful but {} messages has " + + "been lost so far", address.toString(), messagesLost.get()); } } else { @@ -624,8 +660,10 @@ public void operationComplete(ChannelFuture future) throws Exception { }); } else { close(); - throw new RuntimeException("Giving up to scheduleConnect to " + dstAddressPrefixedName + " after " - + connectionAttempts + " failed attempts. " + messagesLost.get() + " messages were lost"); + throw new RuntimeException("Giving up to scheduleConnect to " + + dstAddressPrefixedName + " after " + + connectionAttempts + " failed attempts. " + messagesLost.get() + + " messages were lost"); } } diff --git a/storm-client/src/jvm/org/apache/storm/messaging/netty/Context.java b/storm-client/src/jvm/org/apache/storm/messaging/netty/Context.java index a1384cb8d48..320f8f85185 100644 --- a/storm-client/src/jvm/org/apache/storm/messaging/netty/Context.java +++ b/storm-client/src/jvm/org/apache/storm/messaging/netty/Context.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -36,7 +42,7 @@ public class Context implements IContext { private StormMetricRegistry metricRegistry = null; /** - * initialization per Storm configuration. + * Initialization per Storm configuration. */ @Override public void prepare(Map topoConf) { @@ -48,38 +54,43 @@ public void prepare(Map topoConf, StormMetricRegistry metricRegi this.topoConf = topoConf; serverConnections = new ArrayList<>(); - //each context will have a single client channel worker event loop group - int maxWorkers = ObjectReader.getInt(topoConf.get(Config.STORM_MESSAGING_NETTY_CLIENT_WORKER_THREADS)); + // each context will have a single client channel worker event loop group + int maxWorkers = ObjectReader.getInt(topoConf + .get(Config.STORM_MESSAGING_NETTY_CLIENT_WORKER_THREADS)); ThreadFactory workerFactory = new NettyRenameThreadFactory("client" + "-worker"); // 0 means DEFAULT_EVENT_LOOP_THREADS // https://github.com/netty/netty/blob/netty-4.1.24.Final/transport/src/main/java/io/netty/channel/MultithreadEventLoopGroup.java#L40 - this.workerEventLoopGroup = new NioEventLoopGroup(maxWorkers > 0 ? maxWorkers : 0, workerFactory); + this.workerEventLoopGroup = new NioEventLoopGroup(maxWorkers > 0 ? maxWorkers : 0, + workerFactory); - clientScheduleService = new HashedWheelTimer(new NettyRenameThreadFactory("client-schedule-service")); + clientScheduleService = + new HashedWheelTimer(new NettyRenameThreadFactory("client-schedule-service")); this.metricRegistry = metricRegistry; } /** - * establish a server with a binding port. + * Establish a server with a binding port. */ @Override - public synchronized IConnection bind(String stormId, int port, IConnectionCallback cb, Supplier newConnectionResponse) { + public synchronized IConnection bind(String stormId, int port, IConnectionCallback cb, + Supplier newConnectionResponse) { Server server = new Server(topoConf, port, cb, newConnectionResponse); serverConnections.add(server); return server; } /** - * establish a connection to a remote server. + * Establish a connection to a remote server. */ @Override - public IConnection connect(String stormId, String host, int port, AtomicBoolean[] remoteBpStatus) { + public IConnection connect(String stormId, String host, int port, + AtomicBoolean[] remoteBpStatus) { return new Client(topoConf, remoteBpStatus, workerEventLoopGroup, clientScheduleService, host, port, metricRegistry); } /** - * terminate this context. + * Terminate this context. */ @Override public synchronized void term() { @@ -90,7 +101,7 @@ public synchronized void term() { } serverConnections = null; - //we need to release resources associated with the worker event loop group + // we need to release resources associated with the worker event loop group workerEventLoopGroup.shutdownGracefully().awaitUninterruptibly(); } diff --git a/storm-client/src/jvm/org/apache/storm/messaging/netty/ControlMessage.java b/storm-client/src/jvm/org/apache/storm/messaging/netty/ControlMessage.java index 5bda0e74d88..72b84d7a79f 100644 --- a/storm-client/src/jvm/org/apache/storm/messaging/netty/ControlMessage.java +++ b/storm-client/src/jvm/org/apache/storm/messaging/netty/ControlMessage.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -31,6 +37,7 @@ public enum ControlMessage implements INettySerializable { /** * Create message. + * * @param encoded status code * @return a control message per an encoded status code */ @@ -54,7 +61,7 @@ public static ControlMessage read(byte[] serial) { @Override public int encodeLength() { - return 2; //short + return 2; // short } @Override diff --git a/storm-client/src/jvm/org/apache/storm/messaging/netty/INettySerializable.java b/storm-client/src/jvm/org/apache/storm/messaging/netty/INettySerializable.java index a12a96c0795..a9d3a992024 100644 --- a/storm-client/src/jvm/org/apache/storm/messaging/netty/INettySerializable.java +++ b/storm-client/src/jvm/org/apache/storm/messaging/netty/INettySerializable.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -17,6 +23,7 @@ public interface INettySerializable { /** * Serialize this object to ByteBuf. + * * @param dest The ByteBuf to serialize to */ void write(ByteBuf dest); diff --git a/storm-client/src/jvm/org/apache/storm/messaging/netty/ISaslClient.java b/storm-client/src/jvm/org/apache/storm/messaging/netty/ISaslClient.java index 813448a1394..3f33e13ca2f 100644 --- a/storm-client/src/jvm/org/apache/storm/messaging/netty/ISaslClient.java +++ b/storm-client/src/jvm/org/apache/storm/messaging/netty/ISaslClient.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/messaging/netty/ISaslServer.java b/storm-client/src/jvm/org/apache/storm/messaging/netty/ISaslServer.java index 07f47a54260..af7772a9f3b 100644 --- a/storm-client/src/jvm/org/apache/storm/messaging/netty/ISaslServer.java +++ b/storm-client/src/jvm/org/apache/storm/messaging/netty/ISaslServer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/messaging/netty/IServer.java b/storm-client/src/jvm/org/apache/storm/messaging/netty/IServer.java index 4b095d68087..d547b595e49 100644 --- a/storm-client/src/jvm/org/apache/storm/messaging/netty/IServer.java +++ b/storm-client/src/jvm/org/apache/storm/messaging/netty/IServer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/messaging/netty/KerberosSaslClientHandler.java b/storm-client/src/jvm/org/apache/storm/messaging/netty/KerberosSaslClientHandler.java index 450dc693939..80a5ff58d1d 100644 --- a/storm-client/src/jvm/org/apache/storm/messaging/netty/KerberosSaslClientHandler.java +++ b/storm-client/src/jvm/org/apache/storm/messaging/netty/KerberosSaslClientHandler.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -33,7 +39,8 @@ public class KerberosSaslClientHandler extends ChannelInboundHandlerAdapter { private final String jaasSection; private final String host; - public KerberosSaslClientHandler(ISaslClient client, Map topoConf, String jaasSection, String host) throws + public KerberosSaslClientHandler(ISaslClient client, Map topoConf, + String jaasSection, String host) throws IOException { this.client = client; this.topoConf = topoConf; @@ -51,16 +58,19 @@ public void channelActive(ChannelHandlerContext ctx) { channel.localAddress(), channel.remoteAddress()); try { - KerberosSaslNettyClient saslNettyClient = channel.attr(KerberosSaslNettyClientState.KERBEROS_SASL_NETTY_CLIENT).get(); + KerberosSaslNettyClient saslNettyClient = channel + .attr(KerberosSaslNettyClientState.KERBEROS_SASL_NETTY_CLIENT).get(); if (saslNettyClient == null) { LOG.debug("Creating saslNettyClient now for channel: {}", channel); saslNettyClient = new KerberosSaslNettyClient(topoConf, jaasSection, host); - channel.attr(KerberosSaslNettyClientState.KERBEROS_SASL_NETTY_CLIENT).set(saslNettyClient); + channel.attr(KerberosSaslNettyClientState.KERBEROS_SASL_NETTY_CLIENT) + .set(saslNettyClient); } LOG.debug("Going to initiate Kerberos negotiations."); - byte[] initialChallenge = saslNettyClient.saslResponse(new SaslMessageToken(new byte[0])); + byte[] initialChallenge = saslNettyClient + .saslResponse(new SaslMessageToken(new byte[0])); LOG.debug("Sending initial challenge: {}", initialChallenge); channel.writeAndFlush(new SaslMessageToken(initialChallenge), channel.voidPromise()); } catch (Exception e) { @@ -85,22 +95,26 @@ public void channelRead(ChannelHandlerContext ctx, Object message) throws Except private KerberosSaslNettyClient getChannelSaslClient(Channel channel) throws Exception { // Generate SASL response to server using Channel-local SASL client. - KerberosSaslNettyClient saslNettyClient = channel.attr(KerberosSaslNettyClientState.KERBEROS_SASL_NETTY_CLIENT).get(); + KerberosSaslNettyClient saslNettyClient = channel + .attr(KerberosSaslNettyClientState.KERBEROS_SASL_NETTY_CLIENT).get(); if (saslNettyClient == null) { throw new Exception("saslNettyClient was unexpectedly null for channel:" + channel); } return saslNettyClient; } - private void handleControlMessage(ChannelHandlerContext ctx, ControlMessage controlMessage) throws Exception { + private void handleControlMessage(ChannelHandlerContext ctx, + ControlMessage controlMessage) throws Exception { Channel channel = ctx.channel(); KerberosSaslNettyClient saslNettyClient = getChannelSaslClient(channel); if (controlMessage == ControlMessage.SASL_COMPLETE_REQUEST) { - LOG.debug("Server has sent us the SaslComplete message. Allowing normal work to proceed."); + LOG.debug("Server has sent us the SaslComplete message. Allowing normal work to " + + "proceed."); if (!saslNettyClient.isComplete()) { String errorMessage = - "Server returned a Sasl-complete message, but as far as we can tell, we are not authenticated yet."; + "Server returned a Sasl-complete message, but as far as we can tell, we are " + + "not authenticated yet."; LOG.error(errorMessage); throw new Exception(errorMessage); } @@ -116,10 +130,12 @@ private void handleControlMessage(ChannelHandlerContext ctx, ControlMessage cont } } - private void handleSaslMessageToken(ChannelHandlerContext ctx, SaslMessageToken saslMessageToken) throws Exception { + private void handleSaslMessageToken(ChannelHandlerContext ctx, + SaslMessageToken saslMessageToken) throws Exception { Channel channel = ctx.channel(); KerberosSaslNettyClient saslNettyClient = getChannelSaslClient(channel); - LOG.debug("Responding to server's token of length: {}", saslMessageToken.getSaslToken().length); + LOG.debug("Responding to server's token of length: {}", saslMessageToken + .getSaslToken().length); // Generate SASL response (but we only actually send the response if // it's non-null. @@ -131,7 +147,8 @@ private void handleSaslMessageToken(ChannelHandlerContext ctx, SaslMessageToken LOG.debug("Response to server is null: authentication should now be complete."); if (!saslNettyClient.isComplete()) { LOG.warn("Generated a null response, but authentication is not complete."); - throw new Exception("Our reponse to the server is null, but as far as we can tell, we are not authenticated yet."); + throw new Exception("Our reponse to the server is null, but as far as we can " + + "tell, we are not authenticated yet."); } this.client.channelReady(channel); } else { diff --git a/storm-client/src/jvm/org/apache/storm/messaging/netty/KerberosSaslNettyClient.java b/storm-client/src/jvm/org/apache/storm/messaging/netty/KerberosSaslNettyClient.java index 6fec202eda0..c849ffc520d 100644 --- a/storm-client/src/jvm/org/apache/storm/messaging/netty/KerberosSaslNettyClient.java +++ b/storm-client/src/jvm/org/apache/storm/messaging/netty/KerberosSaslNettyClient.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -39,7 +45,8 @@ public class KerberosSaslNettyClient { .getLogger(KerberosSaslNettyClient.class); /** - * Used to respond to server's counterpart, SaslServer with SASL tokens represented as byte arrays. + * Used to respond to server's counterpart, SaslServer with SASL tokens represented as byte + * arrays. */ private SaslClient saslClient; private Subject subject; @@ -70,7 +77,7 @@ public KerberosSaslNettyClient(Map topoConf, String jaasSection, throw new RuntimeException(ex); } - //check the credential of our principal + // check the credential of our principal if (subject.getPrivateCredentials(KerberosTicket.class).isEmpty()) { LOG.error("Failed to verify user principal."); throw new RuntimeException("Fail to verify user principal with section \"" @@ -147,14 +154,16 @@ public byte[] saslResponse(SaslMessageToken saslTokenMessage) { } /** - * Implementation of javax.security.auth.callback.CallbackHandler that works with Storm topology tokens. + * Implementation of javax.security.auth.callback.CallbackHandler that works with Storm topology + * tokens. */ private static class SaslClientCallbackHandler implements CallbackHandler { /** * Implementation used to respond to SASL tokens from server. * - * @param callbacks objects that indicate what credential information the server's SaslServer requires from the client. + * @param callbacks objects that indicate what credential information the server's + * SaslServer requires from the client. */ @Override public void handle(Callback[] callbacks) throws UnsupportedCallbackException { diff --git a/storm-client/src/jvm/org/apache/storm/messaging/netty/KerberosSaslNettyClientState.java b/storm-client/src/jvm/org/apache/storm/messaging/netty/KerberosSaslNettyClientState.java index 6c923b595a9..19b8bfc6638 100644 --- a/storm-client/src/jvm/org/apache/storm/messaging/netty/KerberosSaslNettyClientState.java +++ b/storm-client/src/jvm/org/apache/storm/messaging/netty/KerberosSaslNettyClientState.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/messaging/netty/KerberosSaslNettyServer.java b/storm-client/src/jvm/org/apache/storm/messaging/netty/KerberosSaslNettyServer.java index d3652e25e4d..297859afc9d 100644 --- a/storm-client/src/jvm/org/apache/storm/messaging/netty/KerberosSaslNettyServer.java +++ b/storm-client/src/jvm/org/apache/storm/messaging/netty/KerberosSaslNettyServer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -35,7 +41,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; - class KerberosSaslNettyServer { private static final Logger LOG = LoggerFactory @@ -45,15 +50,17 @@ class KerberosSaslNettyServer { private Subject subject; private List authorizedUsers; - KerberosSaslNettyServer(Map topoConf, String jaasSection, List authorizedUsers) { + KerberosSaslNettyServer(Map topoConf, String jaasSection, + List authorizedUsers) { this.authorizedUsers = authorizedUsers; LOG.debug("KerberosSaslNettyServer: authmethod {}", SaslUtils.KERBEROS); - KerberosSaslCallbackHandler ch = new KerberosSaslNettyServer.KerberosSaslCallbackHandler(authorizedUsers); + KerberosSaslCallbackHandler ch = new KerberosSaslNettyServer + .KerberosSaslCallbackHandler(authorizedUsers); String jaasConfFile = ClientAuthUtils.getJaasConf(topoConf); - //login our principal + // login our principal subject = null; try { LOG.debug("Trying to login using {}.", jaasConfFile); @@ -65,7 +72,7 @@ class KerberosSaslNettyServer { throw new RuntimeException(ex); } - //check the credential of our principal + // check the credential of our principal if (subject.getPrivateCredentials(KerberosTicket.class).isEmpty()) { LOG.error("Failed to verifyuser principal."); throw new RuntimeException("Fail to verify user principal with section \"" diff --git a/storm-client/src/jvm/org/apache/storm/messaging/netty/KerberosSaslNettyServerState.java b/storm-client/src/jvm/org/apache/storm/messaging/netty/KerberosSaslNettyServerState.java index 77cb8643aa4..f1d9bceaa52 100644 --- a/storm-client/src/jvm/org/apache/storm/messaging/netty/KerberosSaslNettyServerState.java +++ b/storm-client/src/jvm/org/apache/storm/messaging/netty/KerberosSaslNettyServerState.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/messaging/netty/KerberosSaslServerHandler.java b/storm-client/src/jvm/org/apache/storm/messaging/netty/KerberosSaslServerHandler.java index c2274488fef..dbfd79fa2c1 100644 --- a/storm-client/src/jvm/org/apache/storm/messaging/netty/KerberosSaslServerHandler.java +++ b/storm-client/src/jvm/org/apache/storm/messaging/netty/KerberosSaslServerHandler.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -33,7 +39,8 @@ public class KerberosSaslServerHandler extends ChannelInboundHandlerAdapter { private final String jaasSection; private final List authorizedUsers; - public KerberosSaslServerHandler(ISaslServer server, Map topoConf, String jaasSection, + public KerberosSaslServerHandler(ISaslServer server, Map topoConf, + String jaasSection, List authorizedUsers) throws IOException { this.server = server; this.topoConf = topoConf; @@ -56,14 +63,19 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception try { LOG.debug("Got SaslMessageToken!"); - KerberosSaslNettyServer saslNettyServer = channel.attr(KerberosSaslNettyServerState.KERBOROS_SASL_NETTY_SERVER).get(); + KerberosSaslNettyServer saslNettyServer = channel + .attr(KerberosSaslNettyServerState.KERBOROS_SASL_NETTY_SERVER).get(); if (saslNettyServer == null) { - LOG.debug("No saslNettyServer for {} yet; creating now, with topology token: ", channel); + LOG.debug("No saslNettyServer for {} yet; creating now, with topology token: ", + channel); try { - saslNettyServer = new KerberosSaslNettyServer(topoConf, jaasSection, authorizedUsers); - channel.attr(KerberosSaslNettyServerState.KERBOROS_SASL_NETTY_SERVER).set(saslNettyServer); + saslNettyServer = new KerberosSaslNettyServer(topoConf, jaasSection, + authorizedUsers); + channel.attr(KerberosSaslNettyServerState.KERBOROS_SASL_NETTY_SERVER) + .set(saslNettyServer); } catch (RuntimeException ioe) { - LOG.error("Error occurred while creating saslNettyServer on server {} for client {}", + LOG.error("Error occurred while creating saslNettyServer on server {} for " + + "client {}", channel.localAddress(), channel.remoteAddress()); throw ioe; } @@ -78,7 +90,8 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception SaslMessageToken saslTokenMessageRequest = new SaslMessageToken(responseBytes); if (saslTokenMessageRequest.getSaslToken() == null) { - channel.writeAndFlush(ControlMessage.SASL_COMPLETE_REQUEST, channel.voidPromise()); + channel.writeAndFlush(ControlMessage.SASL_COMPLETE_REQUEST, channel + .voidPromise()); } else { // Send response to client. channel.writeAndFlush(saslTokenMessageRequest, channel.voidPromise()); @@ -89,8 +102,10 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception // SASL-Complete message to the client. LOG.info("SASL authentication is complete for client with username: {}", saslNettyServer.getUserName()); - channel.writeAndFlush(ControlMessage.SASL_COMPLETE_REQUEST, channel.voidPromise()); - LOG.debug("Removing SaslServerHandler from pipeline since SASL authentication is complete."); + channel.writeAndFlush(ControlMessage.SASL_COMPLETE_REQUEST, channel + .voidPromise()); + LOG.debug("Removing SaslServerHandler from pipeline since SASL authentication " + + "is complete."); ctx.pipeline().remove(this); server.authenticated(channel); } diff --git a/storm-client/src/jvm/org/apache/storm/messaging/netty/Login.java b/storm-client/src/jvm/org/apache/storm/messaging/netty/Login.java index c3e3d6fb481..ca4b11e44bd 100644 --- a/storm-client/src/jvm/org/apache/storm/messaging/netty/Login.java +++ b/storm-client/src/jvm/org/apache/storm/messaging/netty/Login.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -31,7 +37,8 @@ import org.apache.storm.shade.org.apache.zookeeper.client.ZooKeeperSaslClient; /** - * This class is responsible for refreshing Kerberos credentials for logins for both Zookeeper client and server. See ZooKeeperSaslServer + * This class is responsible for refreshing Kerberos credentials for logins for both Zookeeper + * client and server. See ZooKeeperSaslServer * for server-side usage. See ZooKeeperSaslClient for client-side usage. This class is a copied from * https://github.com/apache/zookeeper/blob/branch-3.4/src/java/main/org/apache/zookeeper/Login.java with the difference that refresh thread * does not die. @@ -40,13 +47,14 @@ public class Login { // Login will sleep until 80% of time from last refresh to // ticket's expiry has been reached, at which time it will wake // and try to renew the ticket. - private static final float TICKET_RENEW_WINDOW = 0.80f; + private static final float TICKET_RENEW_WINDOW = 0.80F; /** * Percentage of random jitter added to the renewal time. */ - private static final float TICKET_RENEW_JITTER = 0.05f; + private static final float TICKET_RENEW_JITTER = 0.05F; // Regardless of TICKET_RENEW_WINDOW setting above and the ticket expiry time, - // thread will not sleep between refresh attempts any less than 1 minute (60*1000 milliseconds = 1 minute). + // thread will not sleep between refresh attempts any less than 1 minute (60*1000 milliseconds = + // 1 minute). // Change the '1' to e.g. 5, to change this to 5 minutes. private static final long MIN_TIME_BEFORE_RELOGIN = 1 * 60 * 1000L; /** @@ -69,6 +77,7 @@ public class Login { /** * Login constructor. The constructor starts the thread used * to periodically re-login to the Kerberos Ticket Granting Server. + * * @param loginContextName * name of section in JAAS file that will be used to login. * Passed as first param to javax.security.auth.login.LoginContext(). @@ -78,7 +87,8 @@ public class Login { * @throws javax.security.auth.login.LoginException * Thrown if authentication fails. */ - public Login(final String loginContextName, CallbackHandler callbackHandler, String jaasConfFile) + public Login(final String loginContextName, CallbackHandler callbackHandler, + String jaasConfFile) throws LoginException { this.loginContextName = loginContextName; this.callbackHandler = callbackHandler; @@ -91,7 +101,8 @@ public Login(final String loginContextName, CallbackHandler callbackHandler, Str AppConfigurationEntry[] entries = configuration.getAppConfigurationEntry(loginContextName); for (AppConfigurationEntry entry : entries) { - // there will only be a single entry, so this for() loop will only be iterated through once. + // there will only be a single entry, so this for() loop will only be iterated through + // once. if (entry.getOptions().get("useTicketCache") != null) { String val = (String) entry.getOptions().get("useTicketCache"); if (val.equals("true")) { @@ -109,9 +120,12 @@ public Login(final String loginContextName, CallbackHandler callbackHandler, Str return; } - // Refresh the Ticket Granting Ticket (TGT) periodically. How often to refresh is determined by the - // TGT's existing expiry date and the configured MIN_TIME_BEFORE_RELOGIN. For testing and development, - // you can decrease the interval of expiration of tickets (for example, to 3 minutes) by running : + // Refresh the Ticket Granting Ticket (TGT) periodically. How often to refresh is determined + // by the + // TGT's existing expiry date and the configured MIN_TIME_BEFORE_RELOGIN. For testing and + // development, + // you can decrease the interval of expiration of tickets (for example, to 3 minutes) by + // running : // "modprinc -maxlife 3mins " in kadmin. this.thread = new Thread(new Runnable() { @Override @@ -131,18 +145,27 @@ public void run() { long expiry = tgt.getEndTime().getTime(); Date expiryDate = new Date(expiry); if ((isUsingTicketCache) && (tgt.getEndTime().equals(tgt.getRenewTill()))) { - LOG.error("The TGT cannot be renewed beyond the next expiry date: " + expiryDate + "." - + "This process will not be able to authenticate new SASL connections after that " - + "time (for example, it will not be authenticate a new connection with a Zookeeper " - + "Quorum member). Ask your system administrator to either increase the " - + "'renew until' time by doing : 'modprinc -maxrenewlife " + principal + "' within " - + "kadmin, or instead, to generate a keytab for " + principal + ". Because the TGT's " - + "expiry cannot be further extended by refreshing, exiting refresh thread now."); + LOG.error("The TGT cannot be renewed beyond the next expiry date: " + + expiryDate + "." + + "This process will not be able to authenticate new SASL " + + "connections after that " + + "time (for example, it will not be authenticate a new " + + "connection with a Zookeeper " + + "Quorum member). Ask your system administrator to either " + + "increase the " + + "'renew until' time by doing : 'modprinc -maxrenewlife " + + principal + "' within " + + "kadmin, or instead, to generate a keytab for " + principal + + ". Because the TGT's " + + "expiry cannot be further extended by refreshing, exiting " + + "refresh thread now."); return; } // determine how long to sleep from looking at ticket's expiry. - // We should not allow the ticket to expire, but we should take into consideration - // MIN_TIME_BEFORE_RELOGIN. Will not sleep less than MIN_TIME_BEFORE_RELOGIN, unless doing so + // We should not allow the ticket to expire, but we should take into + // consideration + // MIN_TIME_BEFORE_RELOGIN. Will not sleep less than + // MIN_TIME_BEFORE_RELOGIN, unless doing so // would cause ticket expiration. if ((nextRefresh > expiry) || ((now + MIN_TIME_BEFORE_RELOGIN) > expiry)) { @@ -150,11 +173,14 @@ public void run() { nextRefresh = now; } else { if (nextRefresh < (now + MIN_TIME_BEFORE_RELOGIN)) { - // next scheduled refresh is sooner than (now + MIN_TIME_BEFORE_LOGIN). + // next scheduled refresh is sooner than (now + + // MIN_TIME_BEFORE_LOGIN). Date until = new Date(nextRefresh); Date newuntil = new Date(now + MIN_TIME_BEFORE_RELOGIN); - LOG.warn("TGT refresh thread time adjusted from : " + until + " to : " + newuntil + " since " - + "the former is sooner than the minimum refresh interval (" + LOG.warn("TGT refresh thread time adjusted from : " + until + + " to : " + newuntil + " since " + + "the former is sooner than the minimum refresh " + + "interval (" + MIN_TIME_BEFORE_RELOGIN / 1000 + " seconds) from now."); } nextRefresh = Math.max(nextRefresh, now + MIN_TIME_BEFORE_RELOGIN); @@ -163,7 +189,8 @@ public void run() { if (tgt != null && now > tgt.getEndTime().getTime()) { if ((now - tgt.getEndTime().getTime()) < (10 * MIN_TIME_BEFORE_RELOGIN)) { Date until = new Date(now + MIN_TIME_BEFORE_RELOGIN); - LOG.info("TGT already expired but giving additional 10 minutes past TGT expiry, refresh " + LOG.info("TGT already expired but giving additional 10 minutes past " + + "TGT expiry, refresh " + "sleeping until: " + until.toString()); try { @@ -173,9 +200,12 @@ public void run() { return; } } else { - LOG.error("nextRefresh:" + new Date(nextRefresh) + " is in the past: exiting refresh thread. Check" - + " clock sync between this host and KDC - (KDC's clock is likely ahead of this host)." - + " Manual intervention will be required for this client to successfully authenticate." + LOG.error("nextRefresh:" + new Date(nextRefresh) + + " is in the past: exiting refresh thread. Check" + + " clock sync between this host and KDC - (KDC's clock is " + + "likely ahead of this host)." + + " Manual intervention will be required for this client to " + + "successfully authenticate." + " Exiting worker!."); Runtime.getRuntime().exit(-3); } @@ -199,7 +229,8 @@ public void run() { int retry = 1; while (retry >= 0) { try { - LOG.debug("running ticket cache refresh command: " + cmd + " " + kinitArgs); + LOG.debug("running ticket cache refresh command: " + cmd + " " + + kinitArgs); Shell.execCommand(cmd, kinitArgs); break; } catch (Exception e) { @@ -209,12 +240,15 @@ public void run() { try { Thread.sleep(10 * 1000); } catch (InterruptedException ie) { - LOG.error("Interrupted while renewing TGT, exiting Login thread"); + LOG.error("Interrupted while renewing TGT, exiting Login " + + "thread"); return; } } else { - LOG.warn("Could not renew TGT due to problem running shell command: '" + cmd - + " " + kinitArgs + "'" + "; exception was:" + e + ". Exiting refresh thread.", e); + LOG.warn("Could not renew TGT due to problem running shell " + + "command: '" + cmd + + " " + kinitArgs + "'" + "; exception was:" + e + + ". Exiting refresh thread.", e); return; } } @@ -233,11 +267,13 @@ public void run() { try { Thread.sleep(10 * 1000); } catch (InterruptedException e) { - LOG.error("Interrupted during login retry after LoginException:", le); + LOG.error("Interrupted during login retry after " + + "LoginException:", le); throw le; } } else { - LOG.error("Could not refresh TGT for principal: " + principal + ".", le); + LOG.error("Could not refresh TGT for principal: " + principal + + ".", le); } } } @@ -297,7 +333,8 @@ private synchronized LoginContext login() throws LoginException { + "Please check your java.security.login.auth.config (=" + System.getProperty("java.security.login.auth.config") + ") and your " + ZooKeeperSaslClient.LOGIN_CONTEXT_NAME_KEY + "(=" - + System.getProperty(ZooKeeperSaslClient.LOGIN_CONTEXT_NAME_KEY, "Client") + ")"); + + System.getProperty(ZooKeeperSaslClient.LOGIN_CONTEXT_NAME_KEY, "Client") + + ")"); } LoginContext loginContext; try { @@ -308,7 +345,8 @@ private synchronized LoginContext login() throws LoginException { LOG.error("Login using jaas conf " + jaasConfFile + " failed"); throw e; } - LOG.info("Successfully logged in to context " + loginContextName + " using " + jaasConfFile); + LOG.info("Successfully logged in to context " + loginContextName + " using " + + jaasConfFile); return loginContext; } @@ -321,7 +359,8 @@ private long getRefreshTime(KerberosTicket tgt) { long proposedRefresh = start + (long) ((expires - start) * (TICKET_RENEW_WINDOW + (TICKET_RENEW_JITTER * rng.nextDouble()))); if (proposedRefresh > expires) { - // proposedRefresh is too far in the future: it's after ticket expires: simply return now. + // proposedRefresh is too far in the future: it's after ticket expires: simply return + // now. return System.currentTimeMillis(); } else { return proposedRefresh; @@ -345,7 +384,8 @@ private void sleepUntilSufficientTimeElapsed() { long now = System.currentTimeMillis(); if (now - getLastLogin() < MIN_TIME_BEFORE_RELOGIN) { LOG.warn("Not attempting to re-login since the last re-login was " - + "attempted less than " + (MIN_TIME_BEFORE_RELOGIN / 1000) + " seconds before."); + + "attempted less than " + (MIN_TIME_BEFORE_RELOGIN / 1000) + + " seconds before."); try { Thread.sleep(MIN_TIME_BEFORE_RELOGIN - (now - getLastLogin())); } catch (InterruptedException e) { @@ -359,6 +399,7 @@ private void sleepUntilSufficientTimeElapsed() { /** * Returns login object. + * * @return login */ private LoginContext getLogin() { @@ -374,6 +415,7 @@ private void setLogin(LoginContext login) { /** * Get the time of the last login. + * * @return the number of milliseconds since the beginning of time. */ private long getLastLogin() { @@ -382,6 +424,7 @@ private long getLastLogin() { /** * Set the last login time. + * * @param time the number of milliseconds since the beginning of time */ private void setLastLogin(long time) { @@ -390,6 +433,7 @@ private void setLastLogin(long time) { /** * Re-login a principal. This method assumes that {@link #login()} has happened already. + * * @throws javax.security.auth.login.LoginException on a failure */ // c.f. HADOOP-6559 @@ -405,17 +449,19 @@ private synchronized void reLogin() sleepUntilSufficientTimeElapsed(); LOG.info("Initiating logout for " + principal); synchronized (Login.class) { - //clear up the kerberos state. But the tokens are not cleared! As per - //the Java kerberos login module code, only the kerberos credentials - //are cleared + // clear up the kerberos state. But the tokens are not cleared! As per + // the Java kerberos login module code, only the kerberos credentials + // are cleared login.logout(); - //login with original callback handler and config, and also update the - //subject field of this instance to have the new credentials (pass it - //to the LoginContext constructor) - login = new LoginContext(loginContextName, getSubject(), callbackHandler, configuration); + // login with original callback handler and config, and also update the + // subject field of this instance to have the new credentials (pass it + // to the LoginContext constructor) + login = new LoginContext(loginContextName, getSubject(), callbackHandler, + configuration); LOG.info("Initiating re-login for " + principal); login.login(); - LOG.info("Successfully re-logged in to context " + loginContextName + " using " + jaasConfFile); + LOG.info("Successfully re-logged in to context " + loginContextName + " using " + + jaasConfFile); setLogin(login); } } diff --git a/storm-client/src/jvm/org/apache/storm/messaging/netty/MessageBatch.java b/storm-client/src/jvm/org/apache/storm/messaging/netty/MessageBatch.java index 6b15788c0a5..3301fa6f29d 100644 --- a/storm-client/src/jvm/org/apache/storm/messaging/netty/MessageBatch.java +++ b/storm-client/src/jvm/org/apache/storm/messaging/netty/MessageBatch.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -42,7 +48,7 @@ private int msgEncodeLength(TaskMessage taskMsg) { return 0; } - int size = 6; //INT + SHORT + int size = 6; // INT + SHORT if (taskMsg.message() != null) { size += taskMsg.message().length; } @@ -51,6 +57,7 @@ private int msgEncodeLength(TaskMessage taskMsg) { /** * Check whether full. + * * @return true if this batch used up allowed buffer size */ boolean isFull() { @@ -59,6 +66,7 @@ boolean isFull() { /** * Check whether empty. + * * @return true if this batch doesn't have any messages */ boolean isEmpty() { @@ -67,6 +75,7 @@ boolean isEmpty() { /** * Get size. + * * @return number of msgs in this batch */ int size() { @@ -79,7 +88,7 @@ public int encodeLength() { } /** - * create a buffer containing the encoding of this batch. + * Create a buffer containing the encoding of this batch. */ @Override public void write(ByteBuf dest) { @@ -87,12 +96,12 @@ public void write(ByteBuf dest) { writeTaskMessage(dest, msg); } - //add a END_OF_BATCH indicator + // add a END_OF_BATCH indicator ControlMessage.EOB_MESSAGE.write(dest); } /** - * write a TaskMessage into a buffer. + * Write a TaskMessage into a buffer. * *

    Each TaskMessage is encoded as: task ... short(2) len ... int(4) payload ... byte[] * */ diff --git a/storm-client/src/jvm/org/apache/storm/messaging/netty/MessageBuffer.java b/storm-client/src/jvm/org/apache/storm/messaging/netty/MessageBuffer.java index f17beb5eb1b..0bcd9cf4d06 100644 --- a/storm-client/src/jvm/org/apache/storm/messaging/netty/MessageBuffer.java +++ b/storm-client/src/jvm/org/apache/storm/messaging/netty/MessageBuffer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/messaging/netty/MessageDecoder.java b/storm-client/src/jvm/org/apache/storm/messaging/netty/MessageDecoder.java index 7dcf206d058..f5f9edd7abd 100644 --- a/storm-client/src/jvm/org/apache/storm/messaging/netty/MessageDecoder.java +++ b/storm-client/src/jvm/org/apache/storm/messaging/netty/MessageDecoder.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -47,11 +53,12 @@ public MessageDecoder(KryoValuesDeserializer deser, boolean serverAuthRequired) * payload ... byte[] * */ @Override - protected void decode(ChannelHandlerContext ctx, ByteBuf buf, List out) throws Exception { + protected void decode(ChannelHandlerContext ctx, ByteBuf buf, + List out) throws Exception { // Make sure that we have received at least a short long available = buf.readableBytes(); if (available < 2) { - //need more data + // need more data return; } @@ -82,11 +89,11 @@ protected void decode(ChannelHandlerContext ctx, ByteBuf buf, List out) } } - //case 2: SaslTokenMessageRequest + // case 2: SaslTokenMessageRequest if (code == SaslMessageToken.IDENTIFIER) { // Make sure that we have received at least an integer (length) if (buf.readableBytes() < 4) { - //need more data + // need more data buf.resetReaderIndex(); return; } @@ -126,7 +133,7 @@ protected void decode(ChannelHandlerContext ctx, ByteBuf buf, List out) } available = buf.readableBytes(); if (available < 4) { - //Need more data + // Need more data buf.resetReaderIndex(); return; } @@ -192,7 +199,8 @@ private boolean gateFrames(ChannelHandlerContext ctx) { if (!serverAuthRequired) { return false; } - SaslNettyServer saslNettyServer = ctx.channel().attr(SaslNettyServerState.SASL_NETTY_SERVER).get(); + SaslNettyServer saslNettyServer = ctx.channel().attr(SaslNettyServerState.SASL_NETTY_SERVER) + .get(); return saslNettyServer == null || !saslNettyServer.isComplete(); } @@ -204,7 +212,8 @@ private static void discardAndClose(ChannelHandlerContext ctx, ByteBuf buf, Stri @Override public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { - LOG.error("Exception thrown while decoding messages in channel {}; exception: ", ctx.channel(), cause); + LOG.error("Exception thrown while decoding messages in channel {}; exception: ", ctx + .channel(), cause); ctx.close(); } } diff --git a/storm-client/src/jvm/org/apache/storm/messaging/netty/NettyRenameThreadFactory.java b/storm-client/src/jvm/org/apache/storm/messaging/netty/NettyRenameThreadFactory.java index 1bc59c83a74..ec62a4e0c1a 100644 --- a/storm-client/src/jvm/org/apache/storm/messaging/netty/NettyRenameThreadFactory.java +++ b/storm-client/src/jvm/org/apache/storm/messaging/netty/NettyRenameThreadFactory.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -17,7 +23,8 @@ public class NettyRenameThreadFactory implements ThreadFactory { - private static final NettyUncaughtExceptionHandler UNCAUGHT_EXCEPTION_HANDLER = new NettyUncaughtExceptionHandler(); + private static final NettyUncaughtExceptionHandler UNCAUGHT_EXCEPTION_HANDLER = + new NettyUncaughtExceptionHandler(); private final ThreadGroup group; private final AtomicInteger index = new AtomicInteger(1); diff --git a/storm-client/src/jvm/org/apache/storm/messaging/netty/NettySerializableMessageEncoder.java b/storm-client/src/jvm/org/apache/storm/messaging/netty/NettySerializableMessageEncoder.java index 0eab349ceef..22bfe9d4881 100644 --- a/storm-client/src/jvm/org/apache/storm/messaging/netty/NettySerializableMessageEncoder.java +++ b/storm-client/src/jvm/org/apache/storm/messaging/netty/NettySerializableMessageEncoder.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -20,17 +26,20 @@ @ChannelHandler.Sharable public class NettySerializableMessageEncoder extends MessageToByteEncoder { - public static final NettySerializableMessageEncoder INSTANCE = new NettySerializableMessageEncoder(); + public static final NettySerializableMessageEncoder INSTANCE = + new NettySerializableMessageEncoder(); private NettySerializableMessageEncoder() {} @Override - protected void encode(ChannelHandlerContext ctx, INettySerializable msg, ByteBuf out) throws Exception { + protected void encode(ChannelHandlerContext ctx, INettySerializable msg, + ByteBuf out) throws Exception { msg.write(out); } @Override - protected ByteBuf allocateBuffer(ChannelHandlerContext ctx, INettySerializable msg, boolean preferDirect) throws Exception { + protected ByteBuf allocateBuffer(ChannelHandlerContext ctx, INettySerializable msg, + boolean preferDirect) throws Exception { return ctx.alloc().ioBuffer(msg.encodeLength()); } diff --git a/storm-client/src/jvm/org/apache/storm/messaging/netty/NettyTlsUtils.java b/storm-client/src/jvm/org/apache/storm/messaging/netty/NettyTlsUtils.java index c2fa4e678d2..6fcc02f23ed 100644 --- a/storm-client/src/jvm/org/apache/storm/messaging/netty/NettyTlsUtils.java +++ b/storm-client/src/jvm/org/apache/storm/messaging/netty/NettyTlsUtils.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -31,17 +37,18 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; - public class NettyTlsUtils { private static final Logger LOG = LoggerFactory.getLogger(NettyTlsUtils.class); public static SslContext createSslContext(Map topoConf, boolean forServer) { - boolean enableTls = ObjectReader.getBoolean(topoConf.get(Config.STORM_MESSAGING_NETTY_TLS_ENABLE), false); + boolean enableTls = ObjectReader.getBoolean(topoConf + .get(Config.STORM_MESSAGING_NETTY_TLS_ENABLE), false); if (!enableTls) { return null; } - boolean requireOpenSsl = ObjectReader.getBoolean(topoConf.get(Config.STORM_MESSAGING_NETTY_TLS_REQUIRE_OPEN_SSL), false); + boolean requireOpenSsl = ObjectReader.getBoolean(topoConf + .get(Config.STORM_MESSAGING_NETTY_TLS_REQUIRE_OPEN_SSL), false); if (requireOpenSsl) { OpenSsl.ensureAvailability(); } @@ -49,40 +56,55 @@ public static SslContext createSslContext(Map topoConf, boolean final Set ciphers; if (topoConf.containsKey(Config.STORM_MESSAGING_NETTY_TLS_CIPHERS)) { ciphers = new HashSet<>(); - ciphers.addAll(ObjectReader.getStrings(topoConf.get(Config.STORM_MESSAGING_NETTY_TLS_CIPHERS))); + ciphers.addAll(ObjectReader.getStrings(topoConf + .get(Config.STORM_MESSAGING_NETTY_TLS_CIPHERS))); } else { // TLSv1.3 ciphers available with OpenSSL from testing ciphers = Collections.unmodifiableSet(new LinkedHashSet( - Arrays.asList("TLS_AES_256_GCM_SHA384", "TLS_CHACHA20_POLY1305_SHA256", "TLS_AES_128_GCM_SHA256"))); + Arrays.asList("TLS_AES_256_GCM_SHA384", "TLS_CHACHA20_POLY1305_SHA256", + "TLS_AES_128_GCM_SHA256"))); } - String protocols = ObjectReader.getString(topoConf.get(Config.STORM_MESSAGING_NETTY_TLS_SSL_PROTOCOLS), "TLSv1.3"); + String protocols = ObjectReader.getString(topoConf + .get(Config.STORM_MESSAGING_NETTY_TLS_SSL_PROTOCOLS), "TLSv1.3"); SslContext sslContext = null; try { SslContextBuilder builder; if (forServer) { LOG.info("Building SSL context for Netty server"); - String keystorePath = ObjectReader.getString(topoConf.get(Config.STORM_MESSAGING_NETTY_TLS_KEYSTORE_PATH)); - String keystorePassword = ObjectReader.getString(topoConf.get(Config.STORM_MESSAGING_NETTY_TLS_KEYSTORE_PASSWORD)); - String truststorePath = ObjectReader.getString(topoConf.get(Config.STORM_MESSAGING_NETTY_TLS_TRUSTSTORE_PATH)); - String truststorePassword = ObjectReader.getString(topoConf.get(Config.STORM_MESSAGING_NETTY_TLS_TRUSTSTORE_PASSWORD)); - builder = SslContextBuilder.forServer(new ReloadableX509KeyManager(keystorePath, keystorePassword)) - .trustManager(new ReloadableX509TrustManager(truststorePath, truststorePassword)) + String keystorePath = ObjectReader.getString(topoConf + .get(Config.STORM_MESSAGING_NETTY_TLS_KEYSTORE_PATH)); + String keystorePassword = ObjectReader.getString(topoConf + .get(Config.STORM_MESSAGING_NETTY_TLS_KEYSTORE_PASSWORD)); + String truststorePath = ObjectReader.getString(topoConf + .get(Config.STORM_MESSAGING_NETTY_TLS_TRUSTSTORE_PATH)); + String truststorePassword = ObjectReader.getString(topoConf + .get(Config.STORM_MESSAGING_NETTY_TLS_TRUSTSTORE_PASSWORD)); + builder = SslContextBuilder.forServer(new ReloadableX509KeyManager(keystorePath, + keystorePassword)) + .trustManager(new ReloadableX509TrustManager(truststorePath, + truststorePassword)) .clientAuth(ClientAuth.REQUIRE); } else { LOG.info("Building SSL context for Netty client"); - String clientKeystorePath = ObjectReader.getString(topoConf.get(Config.STORM_MESSAGING_NETTY_TLS_CLIENT_KEYSTORE_PATH)); + String clientKeystorePath = ObjectReader.getString(topoConf + .get(Config.STORM_MESSAGING_NETTY_TLS_CLIENT_KEYSTORE_PATH)); String clientKeystorePassword = - ObjectReader.getString(topoConf.get(Config.STORM_MESSAGING_NETTY_TLS_CLIENT_KEYSTORE_PASSWORD)); + ObjectReader.getString(topoConf + .get(Config.STORM_MESSAGING_NETTY_TLS_CLIENT_KEYSTORE_PASSWORD)); String clientTruststorePath = - ObjectReader.getString(topoConf.get(Config.STORM_MESSAGING_NETTY_TLS_CLIENT_TRUSTSTORE_PATH)); + ObjectReader.getString(topoConf + .get(Config.STORM_MESSAGING_NETTY_TLS_CLIENT_TRUSTSTORE_PATH)); String clientTruststorePassword = - ObjectReader.getString(topoConf.get(Config.STORM_MESSAGING_NETTY_TLS_CLIENT_TRUSTSTORE_PASSWORD)); + ObjectReader.getString(topoConf + .get(Config.STORM_MESSAGING_NETTY_TLS_CLIENT_TRUSTSTORE_PASSWORD)); builder = SslContextBuilder.forClient(); - builder.keyManager(new ReloadableX509KeyManager(clientKeystorePath, clientKeystorePassword)) - .trustManager(new ReloadableX509TrustManager(clientTruststorePath, clientTruststorePassword)); + builder.keyManager(new ReloadableX509KeyManager(clientKeystorePath, + clientKeystorePassword)) + .trustManager(new ReloadableX509TrustManager(clientTruststorePath, + clientTruststorePassword)); } builder.ciphers(ciphers) diff --git a/storm-client/src/jvm/org/apache/storm/messaging/netty/NettyUncaughtExceptionHandler.java b/storm-client/src/jvm/org/apache/storm/messaging/netty/NettyUncaughtExceptionHandler.java index e7bd779dd8b..10f2a1f226e 100644 --- a/storm-client/src/jvm/org/apache/storm/messaging/netty/NettyUncaughtExceptionHandler.java +++ b/storm-client/src/jvm/org/apache/storm/messaging/netty/NettyUncaughtExceptionHandler.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -24,7 +30,8 @@ public void uncaughtException(Thread t, Throwable e) { try { LOG.error("Uncaught exception in netty " + e.getCause()); } catch (Throwable err) { - // Doing nothing (probably due to an oom issue) and hoping Utils.handleUncaughtException will handle it + // Doing nothing (probably due to an oom issue) and hoping Utils.handleUncaughtException + // will handle it } try { diff --git a/storm-client/src/jvm/org/apache/storm/messaging/netty/SaslMessageToken.java b/storm-client/src/jvm/org/apache/storm/messaging/netty/SaslMessageToken.java index 7e8ecf20eca..42de08e85da 100644 --- a/storm-client/src/jvm/org/apache/storm/messaging/netty/SaslMessageToken.java +++ b/storm-client/src/jvm/org/apache/storm/messaging/netty/SaslMessageToken.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -52,7 +58,8 @@ public static SaslMessageToken read(byte[] serial) { } short identifier = smBuffer.readShort(); int payloadLen = smBuffer.readInt(); - if (identifier != IDENTIFIER || payloadLen < 0 || payloadLen > smBuffer.readableBytes()) { + if (identifier != IDENTIFIER || payloadLen < 0 || payloadLen > smBuffer + .readableBytes()) { return null; } byte[] token = new byte[payloadLen]; @@ -87,7 +94,7 @@ public int encodeLength() { } /** - * encode the current SaslToken Message into a ByteBuf. + * Encode the current SaslToken Message into a ByteBuf. * *

    SaslTokenMessageRequest is encoded as: identifier .... short(2) payload * length .... int payload .... byte[] diff --git a/storm-client/src/jvm/org/apache/storm/messaging/netty/SaslNettyClient.java b/storm-client/src/jvm/org/apache/storm/messaging/netty/SaslNettyClient.java index 0af993a713c..a910ca54576 100644 --- a/storm-client/src/jvm/org/apache/storm/messaging/netty/SaslNettyClient.java +++ b/storm-client/src/jvm/org/apache/storm/messaging/netty/SaslNettyClient.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -35,7 +41,8 @@ public class SaslNettyClient { .getLogger(SaslNettyClient.class); /** - * Used to respond to server's counterpart, SaslServer with SASL tokens represented as byte arrays. + * Used to respond to server's counterpart, SaslServer with SASL tokens represented as byte + * arrays. */ private SaslClient saslClient; @@ -81,7 +88,8 @@ public byte[] saslResponse(SaslMessageToken saslTokenMessage) { } /** - * Implementation of javax.security.auth.callback.CallbackHandler that works with Storm topology tokens. + * Implementation of javax.security.auth.callback.CallbackHandler that works with Storm topology + * tokens. */ private static class SaslClientCallbackHandler implements CallbackHandler { /** @@ -105,7 +113,8 @@ private static class SaslClientCallbackHandler implements CallbackHandler { /** * Implementation used to respond to SASL tokens from server. * - * @param callbacks objects that indicate what credential information the server's SaslServer requires from the client. + * @param callbacks objects that indicate what credential information the server's + * SaslServer requires from the client. */ @Override public void handle(Callback[] callbacks) @@ -124,7 +133,8 @@ public void handle(Callback[] callbacks) rc = (RealmCallback) callback; } else { throw new UnsupportedCallbackException(callback, - "handle: Unrecognized SASL client callback"); + "handle: Unrecognized SASL client " + + "callback"); } } if (nc != null) { diff --git a/storm-client/src/jvm/org/apache/storm/messaging/netty/SaslNettyClientState.java b/storm-client/src/jvm/org/apache/storm/messaging/netty/SaslNettyClientState.java index 1818dfaf89f..bbf94109a29 100644 --- a/storm-client/src/jvm/org/apache/storm/messaging/netty/SaslNettyClientState.java +++ b/storm-client/src/jvm/org/apache/storm/messaging/netty/SaslNettyClientState.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -16,6 +22,7 @@ final class SaslNettyClientState { - public static final AttributeKey SASL_NETTY_CLIENT = AttributeKey.valueOf("sasl.netty.client"); + public static final AttributeKey SASL_NETTY_CLIENT = AttributeKey + .valueOf("sasl.netty.client"); } diff --git a/storm-client/src/jvm/org/apache/storm/messaging/netty/SaslNettyServer.java b/storm-client/src/jvm/org/apache/storm/messaging/netty/SaslNettyServer.java index ae6372118c8..d48aca45660 100644 --- a/storm-client/src/jvm/org/apache/storm/messaging/netty/SaslNettyServer.java +++ b/storm-client/src/jvm/org/apache/storm/messaging/netty/SaslNettyServer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -89,7 +95,8 @@ public static class SaslDigestCallbackHandler implements CallbackHandler { private String userName; SaslDigestCallbackHandler(String topologyName, byte[] token) { - LOG.debug("SaslDigestCallback: Creating SaslDigestCallback handler with topology token: {}", topologyName); + LOG.debug("SaslDigestCallback: Creating SaslDigestCallback handler with topology " + + "token: {}", topologyName); this.userName = topologyName; this.userPassword = token; } @@ -112,12 +119,14 @@ public void handle(Callback[] callbacks) throws IOException, continue; // realm is ignored } else { throw new UnsupportedCallbackException(callback, - "handle: Unrecognized SASL DIGEST-MD5 Callback"); + "handle: Unrecognized SASL DIGEST-MD5 " + + "Callback"); } } if (nc != null) { - LOG.debug("handle: SASL server DIGEST-MD5 callback: setting username for client: {}", + LOG.debug("handle: SASL server DIGEST-MD5 callback: setting username for client: " + + "{}", userName); nc.setName(userName); } @@ -141,7 +150,8 @@ public void handle(Callback[] callbacks) throws IOException, } if (ac.isAuthorized()) { - LOG.debug("handle: SASL server DIGEST-MD5 callback: setting canonicalized client ID: ", + LOG.debug("handle: SASL server DIGEST-MD5 callback: setting canonicalized " + + "client ID: ", userName); ac.setAuthorizedID(authzid); } diff --git a/storm-client/src/jvm/org/apache/storm/messaging/netty/SaslNettyServerState.java b/storm-client/src/jvm/org/apache/storm/messaging/netty/SaslNettyServerState.java index 2c7951900d7..0ed90f7c2a2 100644 --- a/storm-client/src/jvm/org/apache/storm/messaging/netty/SaslNettyServerState.java +++ b/storm-client/src/jvm/org/apache/storm/messaging/netty/SaslNettyServerState.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -16,5 +22,6 @@ final class SaslNettyServerState { - public static final AttributeKey SASL_NETTY_SERVER = AttributeKey.valueOf("sasl.netty.server"); + public static final AttributeKey SASL_NETTY_SERVER = AttributeKey + .valueOf("sasl.netty.server"); } diff --git a/storm-client/src/jvm/org/apache/storm/messaging/netty/SaslStormClientHandler.java b/storm-client/src/jvm/org/apache/storm/messaging/netty/SaslStormClientHandler.java index ca1e57602d3..eb52b1acaf2 100644 --- a/storm-client/src/jvm/org/apache/storm/messaging/netty/SaslStormClientHandler.java +++ b/storm-client/src/jvm/org/apache/storm/messaging/netty/SaslStormClientHandler.java @@ -1,13 +1,19 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -45,7 +51,8 @@ public void channelActive(ChannelHandlerContext ctx) { LOG.info("Connection established from " + channel.localAddress() + " to " + channel.remoteAddress()); try { - SaslNettyClient saslNettyClient = channel.attr(SaslNettyClientState.SASL_NETTY_CLIENT).get(); + SaslNettyClient saslNettyClient = channel.attr(SaslNettyClientState.SASL_NETTY_CLIENT) + .get(); if (saslNettyClient == null) { LOG.debug("Creating saslNettyClient now " + "for channel: " @@ -78,7 +85,8 @@ public void channelRead(ChannelHandlerContext ctx, Object message) throws Except private SaslNettyClient getChannelSaslNettyClient(Channel channel) throws Exception { // Generate SASL response to server using Channel-local SASL client. - SaslNettyClient saslNettyClient = channel.attr(SaslNettyClientState.SASL_NETTY_CLIENT).get(); + SaslNettyClient saslNettyClient = channel.attr(SaslNettyClientState.SASL_NETTY_CLIENT) + .get(); if (saslNettyClient == null) { throw new Exception("saslNettyClient was unexpectedly " + "null for channel: " + channel); @@ -86,7 +94,8 @@ private SaslNettyClient getChannelSaslNettyClient(Channel channel) throws Except return saslNettyClient; } - private void handleControlMessage(ChannelHandlerContext ctx, ControlMessage controlMessage) throws Exception { + private void handleControlMessage(ChannelHandlerContext ctx, + ControlMessage controlMessage) throws Exception { SaslNettyClient saslNettyClient = getChannelSaslNettyClient(ctx.channel()); if (controlMessage == ControlMessage.SASL_COMPLETE_REQUEST) { LOG.debug("Server has sent us the SaslComplete " @@ -111,7 +120,8 @@ private void handleControlMessage(ChannelHandlerContext ctx, ControlMessage cont } } - private void handleSaslMessageToken(ChannelHandlerContext ctx, SaslMessageToken saslMessageToken) throws Exception { + private void handleSaslMessageToken(ChannelHandlerContext ctx, + SaslMessageToken saslMessageToken) throws Exception { Channel channel = ctx.channel(); SaslNettyClient saslNettyClient = getChannelSaslNettyClient(channel); LOG.debug("Responding to server's token of length: " diff --git a/storm-client/src/jvm/org/apache/storm/messaging/netty/SaslStormServerAuthorizeHandler.java b/storm-client/src/jvm/org/apache/storm/messaging/netty/SaslStormServerAuthorizeHandler.java index 64a8bae9142..9ab41d54245 100644 --- a/storm-client/src/jvm/org/apache/storm/messaging/netty/SaslStormServerAuthorizeHandler.java +++ b/storm-client/src/jvm/org/apache/storm/messaging/netty/SaslStormServerAuthorizeHandler.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -18,7 +24,8 @@ import org.slf4j.LoggerFactory; /** - * Authorize or deny client requests based on existence and completeness of client's SASL authentication. + * Authorize or deny client requests based on existence and completeness of client's SASL + * authentication. */ public class SaslStormServerAuthorizeHandler extends ChannelInboundHandlerAdapter { @@ -37,11 +44,13 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception return; } - LOG.debug("messageReceived: Checking whether the client is authorized to send messages to the server "); + LOG.debug("messageReceived: Checking whether the client is authorized to send messages to " + + "the server "); // Authorize: client is allowed to doRequest() if and only if the client // has successfully authenticated with this server. - SaslNettyServer saslNettyServer = ctx.channel().attr(SaslNettyServerState.SASL_NETTY_SERVER).get(); + SaslNettyServer saslNettyServer = ctx.channel().attr(SaslNettyServerState.SASL_NETTY_SERVER) + .get(); if (saslNettyServer == null) { LOG.warn("messageReceived: This client is *NOT* authorized to perform " diff --git a/storm-client/src/jvm/org/apache/storm/messaging/netty/SaslStormServerHandler.java b/storm-client/src/jvm/org/apache/storm/messaging/netty/SaslStormServerHandler.java index d4e127359f9..0de2e18d07d 100644 --- a/storm-client/src/jvm/org/apache/storm/messaging/netty/SaslStormServerHandler.java +++ b/storm-client/src/jvm/org/apache/storm/messaging/netty/SaslStormServerHandler.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -48,7 +54,8 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception // initialize server-side SASL functionality, if we haven't yet // (in which case we are looking at the first SASL message from the // client). - SaslNettyServer saslNettyServer = channel.attr(SaslNettyServerState.SASL_NETTY_SERVER).get(); + SaslNettyServer saslNettyServer = channel.attr(SaslNettyServerState.SASL_NETTY_SERVER) + .get(); if (saslNettyServer == null) { LOG.debug("No saslNettyServer for " + channel + " yet; creating now, with topology token: " + topologyName); @@ -61,7 +68,8 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception + channel.localAddress() + " for client " + channel.remoteAddress()); - throw new IllegalStateException("Failed to set SaslNettyServerState.SASL_NETTY_SERVER"); + throw new IllegalStateException("Failed to set " + + "SaslNettyServerState.SASL_NETTY_SERVER"); } channel.attr(SaslNettyServerState.SASL_NETTY_SERVER).set(saslNettyServer); @@ -88,7 +96,8 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception // initialize server-side SASL functionality, if we haven't yet // (in which case we are looking at the first SASL message from the // client). - SaslNettyServer saslNettyServer = channel.attr(SaslNettyServerState.SASL_NETTY_SERVER).get(); + SaslNettyServer saslNettyServer = channel.attr(SaslNettyServerState.SASL_NETTY_SERVER) + .get(); if (saslNettyServer == null) { throw new Exception("saslNettyServer was unexpectedly " + "null for channel: " + channel); diff --git a/storm-client/src/jvm/org/apache/storm/messaging/netty/SaslUtils.java b/storm-client/src/jvm/org/apache/storm/messaging/netty/SaslUtils.java index 7a7e2e8f71a..db08e3054c2 100644 --- a/storm-client/src/jvm/org/apache/storm/messaging/netty/SaslUtils.java +++ b/storm-client/src/jvm/org/apache/storm/messaging/netty/SaslUtils.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -53,7 +59,8 @@ static String encodeIdentifier(byte[] identifier) { } static String getSecretKey(Map conf) { - return conf == null || conf.isEmpty() ? null : (String) conf.get(Config.STORM_ZOOKEEPER_TOPOLOGY_AUTH_PAYLOAD); + return conf == null || conf.isEmpty() ? null : (String) conf + .get(Config.STORM_ZOOKEEPER_TOPOLOGY_AUTH_PAYLOAD); } } diff --git a/storm-client/src/jvm/org/apache/storm/messaging/netty/Server.java b/storm-client/src/jvm/org/apache/storm/messaging/netty/Server.java index 3d54cab0d7d..d64e79628d4 100644 --- a/storm-client/src/jvm/org/apache/storm/messaging/netty/Server.java +++ b/storm-client/src/jvm/org/apache/storm/messaging/netty/Server.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -56,12 +62,14 @@ class Server extends ConnectionWithStatus implements IStatefulObject, ISaslServe private final EventLoopGroup bossEventLoopGroup; private final EventLoopGroup workerEventLoopGroup; private final ServerBootstrap bootstrap; - private final ConcurrentHashMap messagesEnqueued = new ConcurrentHashMap<>(); + private final ConcurrentHashMap messagesEnqueued = + new ConcurrentHashMap<>(); private final AtomicInteger messagesDequeued = new AtomicInteger(0); private final int boundPort; private final Map topoConf; private final int port; - private final ChannelGroup allChannels = new DefaultChannelGroup("storm-server", GlobalEventExecutor.INSTANCE); + private final ChannelGroup allChannels = new DefaultChannelGroup("storm-server", + GlobalEventExecutor.INSTANCE); private final KryoValuesSerializer ser; private final IConnectionCallback cb; private final Supplier newConnectionResponse; @@ -70,23 +78,29 @@ class Server extends ConnectionWithStatus implements IStatefulObject, ISaslServe /** * Starts Netty at the given port. + * * @param topoConf The topology config * @param port The port to start Netty at * @param cb The callback to deliver incoming messages to - * @param newConnectionResponse The response to send to clients when they connect. Can be null. If authentication + * @param newConnectionResponse The response to send to clients when they connect. Can be null. + * If authentication * is required, the message will be sent after authentication is complete. */ - Server(Map topoConf, int port, IConnectionCallback cb, Supplier newConnectionResponse) { + Server(Map topoConf, int port, IConnectionCallback cb, + Supplier newConnectionResponse) { this.topoConf = topoConf; - this.isNettyAuthRequired = (Boolean) topoConf.get(Config.STORM_MESSAGING_NETTY_AUTHENTICATION); + this.isNettyAuthRequired = (Boolean) topoConf + .get(Config.STORM_MESSAGING_NETTY_AUTHENTICATION); this.port = port; ser = new KryoValuesSerializer(topoConf); this.cb = cb; this.newConnectionResponse = newConnectionResponse; // Configure the server. - int bufferSize = ObjectReader.getInt(topoConf.get(Config.STORM_MESSAGING_NETTY_BUFFER_SIZE)); - int maxWorkers = ObjectReader.getInt(topoConf.get(Config.STORM_MESSAGING_NETTY_SERVER_WORKER_THREADS)); + int bufferSize = ObjectReader.getInt(topoConf + .get(Config.STORM_MESSAGING_NETTY_BUFFER_SIZE)); + int maxWorkers = ObjectReader.getInt(topoConf + .get(Config.STORM_MESSAGING_NETTY_SERVER_WORKER_THREADS)); ThreadFactory bossFactory = new NettyRenameThreadFactory(netty_name() + "-boss"); ThreadFactory workerFactory = new NettyRenameThreadFactory(netty_name() + "-worker"); @@ -94,13 +108,16 @@ class Server extends ConnectionWithStatus implements IStatefulObject, ISaslServe bossEventLoopGroup = new NioEventLoopGroup(1, bossFactory); // 0 means DEFAULT_EVENT_LOOP_THREADS // https://github.com/netty/netty/blob/netty-4.1.24.Final/transport/src/main/java/io/netty/channel/MultithreadEventLoopGroup.java#L40 - this.workerEventLoopGroup = new NioEventLoopGroup(maxWorkers > 0 ? maxWorkers : 0, workerFactory); + this.workerEventLoopGroup = new NioEventLoopGroup(maxWorkers > 0 ? maxWorkers : 0, + workerFactory); - LOG.info("Create Netty Server " + netty_name() + ", buffer_size: " + bufferSize + ", maxWorkers: " + maxWorkers); + LOG.info("Create Netty Server " + netty_name() + ", buffer_size: " + bufferSize + + ", maxWorkers: " + maxWorkers); SslContext sslContext = NettyTlsUtils.createSslContext(topoConf, true); - int backlog = ObjectReader.getInt(topoConf.get(Config.STORM_MESSAGING_NETTY_SOCKET_BACKLOG), 500); + int backlog = ObjectReader.getInt(topoConf.get(Config.STORM_MESSAGING_NETTY_SOCKET_BACKLOG), + 500); bootstrap = new ServerBootstrap() .group(bossEventLoopGroup, workerEventLoopGroup) .channel(NioServerSocketChannel.class) @@ -124,7 +141,7 @@ class Server extends ConnectionWithStatus implements IStatefulObject, ISaslServe } private void addReceiveCount(String from, int amount) { - //This is possibly lossy in the case where a value is deleted + // This is possibly lossy in the case where a value is deleted // because it has received no messages over the metrics collection // period and new messages are starting to come in. This is // because I don't want the overhead of a synchronize just to have @@ -142,7 +159,7 @@ private void addReceiveCount(String from, int amount) { } /** - * enqueue a received message. + * Enqueue a received message. */ protected void enqueue(List msgs, String from) throws InterruptedException { if (null == msgs || msgs.isEmpty() || closing) { @@ -158,7 +175,7 @@ public int getPort() { } /** - * close all channels, and release resources. + * Close all channels, and release resources. */ @Override public void close() { @@ -171,7 +188,8 @@ public void close() { public void sendLoadMetrics(Map taskToLoad) { MessageBatch mb = new MessageBatch(1); synchronized (ser) { - mb.add(new TaskMessage(LOAD_METRICS_TASK_ID, ser.serialize(Collections.singletonList((Object) taskToLoad)))); + mb.add(new TaskMessage(LOAD_METRICS_TASK_ID, ser.serialize(Collections + .singletonList((Object) taskToLoad)))); } allChannels.writeAndFlush(mb); } @@ -179,7 +197,8 @@ public void sendLoadMetrics(Map taskToLoad) { // this method expected to be thread safe @Override public void sendBackPressureStatus(BackPressureStatus bpStatus) { - LOG.info("Sending BackPressure status update to connected workers. BPStatus = {}", bpStatus); + LOG.info("Sending BackPressure status update to connected workers. BPStatus = {}", + bpStatus); allChannels.writeAndFlush(bpStatus); } @@ -232,7 +251,8 @@ public Object getState() { Iterator> it = messagesEnqueued.entrySet().iterator(); while (it.hasNext()) { Map.Entry ent = it.next(); - //Yes we can delete something that is not 0 because of races, but that is OK for metrics + // Yes we can delete something that is not 0 because of races, but that is OK for + // metrics AtomicInteger i = ent.getValue(); if (i.get() == 0) { it.remove(); @@ -264,20 +284,22 @@ public Object getState() { @Override public void channelActive(Channel c) { if (!isNettyAuthRequired) { - //if authentication is not required, treat it as authenticated. + // if authentication is not required, treat it as authenticated. authenticated(c); } allChannels.add(c); } @Override - public void received(Object message, String remote, Channel channel) throws InterruptedException { + public void received(Object message, String remote, + Channel channel) throws InterruptedException { List msgs; try { msgs = (List) message; } catch (ClassCastException e) { - LOG.error("Worker netty server received message other than the expected class List from remote: {}. Ignored.", + LOG.error("Worker netty server received message other than the expected class " + + "List from remote: {}. Ignored.", remote, e); return; } diff --git a/storm-client/src/jvm/org/apache/storm/messaging/netty/StormClientHandler.java b/storm-client/src/jvm/org/apache/storm/messaging/netty/StormClientHandler.java index a3837a8dc8c..bb5dea277f1 100644 --- a/storm-client/src/jvm/org/apache/storm/messaging/netty/StormClientHandler.java +++ b/storm-client/src/jvm/org/apache/storm/messaging/netty/StormClientHandler.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -38,7 +44,7 @@ public class StormClientHandler extends ChannelInboundHandlerAdapter { @Override public void channelRead(ChannelHandlerContext ctx, Object message) throws Exception { - //examine the response message from server + // examine the response message from server if (message instanceof ControlMessage) { ControlMessage msg = (ControlMessage) message; if (msg == ControlMessage.FAILURE_RESPONSE) { @@ -51,7 +57,7 @@ public void channelRead(ChannelHandlerContext ctx, Object message) throws Except try { remoteBpStatus[bpTask].set(true); } catch (ArrayIndexOutOfBoundsException e) { - //Just in case we get something we are confused about + // Just in case we get something we are confused about // we can continue processing the rest of the tasks LOG.error("BP index out of bounds {}", e); } @@ -62,7 +68,7 @@ public void channelRead(ChannelHandlerContext ctx, Object message) throws Except try { remoteBpStatus[bpTask].set(false); } catch (ArrayIndexOutOfBoundsException e) { - //Just in case we get something we are confused about + // Just in case we get something we are confused about // we can continue processing the rest of the tasks LOG.error("BP index out of bounds {}", e); } @@ -70,22 +76,27 @@ public void channelRead(ChannelHandlerContext ctx, Object message) throws Except } LOG.debug("Received BackPressure status update : {}", status); } else if (message instanceof List) { - //This should be the load metrics. - //There will usually only be one message, but if there are multiple we only process the latest one. + // This should be the load metrics. + // There will usually only be one message, but if there are multiple we only process the + // latest one. List list = (List) message; if (list.size() < 1) { - throw new RuntimeException("Didn't see enough load metrics (" + client.getDstAddress() + ") " + list); + throw new RuntimeException("Didn't see enough load metrics (" + client + .getDstAddress() + ") " + list); } TaskMessage tm = list.get(list.size() - 1); if (tm.task() != Server.LOAD_METRICS_TASK_ID) { - throw new RuntimeException("Metrics messages are sent to the system task (" + client.getDstAddress() + ") " + tm); + throw new RuntimeException("Metrics messages are sent to the system task (" + client + .getDstAddress() + ") " + tm); } List metrics = des.deserialize(tm.message()); if (metrics.size() < 1) { - throw new RuntimeException("No metrics data in the metrics message (" + client.getDstAddress() + ") " + metrics); + throw new RuntimeException("No metrics data in the metrics message (" + client + .getDstAddress() + ") " + metrics); } if (!(metrics.get(0) instanceof Map)) { - throw new RuntimeException("The metrics did not have a map in the first slot (" + client.getDstAddress() + ") " + metrics); + throw new RuntimeException("The metrics did not have a map in the first slot (" + + client.getDstAddress() + ") " + metrics); } client.setLoadMetrics((Map) metrics.get(0)); } else { diff --git a/storm-client/src/jvm/org/apache/storm/messaging/netty/StormClientPipelineFactory.java b/storm-client/src/jvm/org/apache/storm/messaging/netty/StormClientPipelineFactory.java index 58c5f8bd170..b0c60c84c24 100644 --- a/storm-client/src/jvm/org/apache/storm/messaging/netty/StormClientPipelineFactory.java +++ b/storm-client/src/jvm/org/apache/storm/messaging/netty/StormClientPipelineFactory.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -27,7 +33,8 @@ class StormClientPipelineFactory extends ChannelInitializer { private static final String ENDPOINT_IDENTIFICATION_ALGORITHM = "HTTPS"; - // An empty algorithm turns the check off. A null one must not be used here, the JDK engine ignores it and keeps + // An empty algorithm turns the check off. A null one must not be used here, the JDK engine + // ignores it and keeps // whatever algorithm it already had. private static final String NO_ENDPOINT_IDENTIFICATION = ""; @@ -38,7 +45,8 @@ class StormClientPipelineFactory extends ChannelInitializer { private final String dstHost; private final int dstPort; - StormClientPipelineFactory(Client client, AtomicBoolean[] remoteBpStatus, Map conf, + StormClientPipelineFactory(Client client, AtomicBoolean[] remoteBpStatus, Map conf, SslContext sslContext, String dstHost, int dstPort) { this.client = client; this.remoteBpStatus = remoteBpStatus; diff --git a/storm-client/src/jvm/org/apache/storm/messaging/netty/StormServerHandler.java b/storm-client/src/jvm/org/apache/storm/messaging/netty/StormServerHandler.java index 0088395d179..ebbcdc7a715 100644 --- a/storm-client/src/jvm/org/apache/storm/messaging/netty/StormServerHandler.java +++ b/storm-client/src/jvm/org/apache/storm/messaging/netty/StormServerHandler.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -25,7 +31,8 @@ public class StormServerHandler extends ChannelInboundHandlerAdapter { private static final Logger LOG = LoggerFactory.getLogger(StormServerHandler.class); - private static final Set> ALLOWED_EXCEPTIONS = new HashSet<>(Arrays.asList(new Class[]{ IOException.class })); + private static final Set> ALLOWED_EXCEPTIONS = new HashSet<>(Arrays + .asList(new Class[]{ IOException.class })); private final IServer server; public StormServerHandler(IServer server) { @@ -54,9 +61,11 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception @Override public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { try { - LOG.error("server errors in handling the request from {}", ctx.channel().remoteAddress().toString(), cause); + LOG.error("server errors in handling the request from {}", ctx.channel().remoteAddress() + .toString(), cause); } catch (Throwable err) { - // Doing nothing (probably due to an oom issue) and hoping Utils.handleUncaughtException will handle it + // Doing nothing (probably due to an oom issue) and hoping Utils.handleUncaughtException + // will handle it } try { Utils.handleUncaughtException(cause, ALLOWED_EXCEPTIONS, false); diff --git a/storm-client/src/jvm/org/apache/storm/messaging/netty/StormServerPipelineFactory.java b/storm-client/src/jvm/org/apache/storm/messaging/netty/StormServerPipelineFactory.java index 02f838142a4..4cc66ec72ec 100644 --- a/storm-client/src/jvm/org/apache/storm/messaging/netty/StormServerPipelineFactory.java +++ b/storm-client/src/jvm/org/apache/storm/messaging/netty/StormServerPipelineFactory.java @@ -1,13 +1,19 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -48,10 +54,12 @@ protected void initChannel(Channel ch) throws Exception { .get(Config.STORM_MESSAGING_NETTY_AUTHENTICATION); // Decoder - pipeline.addLast("decoder", new MessageDecoder(new KryoValuesDeserializer(topoConf), isNettyAuth)); + pipeline.addLast("decoder", new MessageDecoder(new KryoValuesDeserializer(topoConf), + isNettyAuth)); // Encoders pipeline.addLast("netty-serializable-encoder", NettySerializableMessageEncoder.INSTANCE); - pipeline.addLast("backpressure-encoder", new BackPressureStatusEncoder(new KryoValuesSerializer(topoConf))); + pipeline.addLast("backpressure-encoder", + new BackPressureStatusEncoder(new KryoValuesSerializer(topoConf))); if (isNettyAuth) { // Authenticate: Removed after authentication completes diff --git a/storm-client/src/jvm/org/apache/storm/metric/EventLoggerBolt.java b/storm-client/src/jvm/org/apache/storm/metric/EventLoggerBolt.java index 97768487140..b3e97f8c0a1 100644 --- a/storm-client/src/jvm/org/apache/storm/metric/EventLoggerBolt.java +++ b/storm-client/src/jvm/org/apache/storm/metric/EventLoggerBolt.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -40,11 +46,13 @@ public class EventLoggerBolt implements IBolt { private List eventLoggers; @Override - public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) { + public void prepare(Map topoConf, TopologyContext context, + OutputCollector collector) { LOG.info("EventLoggerBolt prepare called"); eventLoggers = new ArrayList<>(); - List> registerInfo = (List>) topoConf.get(Config.TOPOLOGY_EVENT_LOGGER_REGISTER); + List> registerInfo = (List>) topoConf + .get(Config.TOPOLOGY_EVENT_LOGGER_REGISTER); if (registerInfo != null && !registerInfo.isEmpty()) { initializeEventLoggers(topoConf, context, registerInfo); } else { @@ -54,11 +62,14 @@ public void prepare(Map topoConf, TopologyContext context, Outpu @Override public void execute(Tuple input) { - LOG.debug("** EventLoggerBolt got tuple from sourceComponent {}, with values {}", input.getSourceComponent(), input.getValues()); + LOG.debug("** EventLoggerBolt got tuple from sourceComponent {}, with values {}", input + .getSourceComponent(), input.getValues()); Object msgId = input.getValueByField(FIELD_MESSAGE_ID); - EventInfo eventInfo = new EventInfo(input.getLongByField(FIELD_TS), input.getSourceComponent(), - input.getSourceTask(), msgId, (List) input.getValueByField(FIELD_VALUES)); + EventInfo eventInfo = new EventInfo(input.getLongByField(FIELD_TS), input + .getSourceComponent(), + input.getSourceTask(), msgId, (List) input + .getValueByField(FIELD_VALUES)); for (IEventLogger eventLogger : eventLoggers) { eventLogger.log(eventInfo); @@ -72,17 +83,21 @@ public void cleanup() { } } - private void initializeEventLoggers(Map topoConf, TopologyContext context, List> registerInfo) { + private void initializeEventLoggers(Map topoConf, TopologyContext context, + List> registerInfo) { for (Map info : registerInfo) { String className = (String) info.get(TOPOLOGY_EVENT_LOGGER_CLASS); - Map arguments = (Map) info.get(TOPOLOGY_EVENT_LOGGER_ARGUMENTS); + Map arguments = (Map) info + .get(TOPOLOGY_EVENT_LOGGER_ARGUMENTS); IEventLogger eventLogger; try { eventLogger = (IEventLogger) Class.forName(className).newInstance(); } catch (Exception e) { - throw new RuntimeException("Could not instantiate a class listed in config under section " - + Config.TOPOLOGY_EVENT_LOGGER_REGISTER + " with fully qualified name " + className, e); + throw new RuntimeException("Could not instantiate a class listed in config under " + + "section " + + Config.TOPOLOGY_EVENT_LOGGER_REGISTER + + " with fully qualified name " + className, e); } eventLogger.prepare(topoConf, arguments, context); @@ -90,7 +105,8 @@ private void initializeEventLoggers(Map topoConf, TopologyContex } } - private void initializeDefaultEventLogger(Map topoConf, TopologyContext context) { + private void initializeDefaultEventLogger(Map topoConf, + TopologyContext context) { FileBasedEventLogger eventLogger = new FileBasedEventLogger(); eventLogger.prepare(topoConf, null, context); eventLoggers.add(eventLogger); diff --git a/storm-client/src/jvm/org/apache/storm/metric/FileBasedEventLogger.java b/storm-client/src/jvm/org/apache/storm/metric/FileBasedEventLogger.java index f626b839642..be067fa561b 100644 --- a/storm-client/src/jvm/org/apache/storm/metric/FileBasedEventLogger.java +++ b/storm-client/src/jvm/org/apache/storm/metric/FileBasedEventLogger.java @@ -65,7 +65,8 @@ private void initLogWriter(Path logFilePath) { currentFileSize = Files.exists(eventLogPath) ? Files.size(eventLogPath) : 0L; - eventLogWriter = Files.newBufferedWriter(eventLogPath, StandardCharsets.UTF_8, StandardOpenOption.CREATE, + eventLogWriter = Files.newBufferedWriter(eventLogPath, StandardCharsets.UTF_8, + StandardOpenOption.CREATE, StandardOpenOption.WRITE, StandardOpenOption.APPEND); } catch (IOException e) { LOG.error("Error setting up FileBasedEventLogger.", e); @@ -101,14 +102,17 @@ public void run() { } @Override - public void prepare(Map conf, Map arguments, TopologyContext context) { + public void prepare(Map conf, Map arguments, + TopologyContext context) { String stormId = context.getStormId(); int port = context.getThisWorkerPort(); - int rotationSizeMb = ObjectReader.getInt(conf.get(Config.TOPOLOGY_EVENTLOGGER_ROTATION_SIZE_MB), + int rotationSizeMb = ObjectReader.getInt(conf + .get(Config.TOPOLOGY_EVENTLOGGER_ROTATION_SIZE_MB), DEFAULT_ROTATION_SIZE_MB); this.maxFileSize = rotationSizeMb * BYTES_PER_MB; - this.maxRetainedFiles = ObjectReader.getInt(conf.get(Config.TOPOLOGY_EVENTLOGGER_MAX_RETAINED_FILES), + this.maxRetainedFiles = ObjectReader.getInt(conf + .get(Config.TOPOLOGY_EVENTLOGGER_MAX_RETAINED_FILES), DEFAULT_MAX_RETAINED_FILES); /* diff --git a/storm-client/src/jvm/org/apache/storm/metric/IEventLogger.java b/storm-client/src/jvm/org/apache/storm/metric/IEventLogger.java index 7a8af678e20..0fc900f49aa 100644 --- a/storm-client/src/jvm/org/apache/storm/metric/IEventLogger.java +++ b/storm-client/src/jvm/org/apache/storm/metric/IEventLogger.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -18,14 +24,16 @@ import org.apache.storm.task.TopologyContext; /** - * EventLogger interface for logging the event info to a sink like log file or db for inspecting the events via UI for debugging. + * EventLogger interface for logging the event info to a sink like log file or db for inspecting the + * events via UI for debugging. */ public interface IEventLogger { void prepare(Map conf, Map arguments, TopologyContext context); /** - * This method would be invoked when the {@link EventLoggerBolt} receives a tuple from the spouts or bolts that has event logging + * This method would be invoked when the {@link EventLoggerBolt} receives a tuple from the + * spouts or bolts that has event logging * enabled. * * @param e the event @@ -44,7 +52,8 @@ class EventInfo { private Object messageId; private List values; - public EventInfo(long ts, String component, int task, Object messageId, List values) { + public EventInfo(long ts, String component, int task, Object messageId, + List values) { this.ts = ts; this.component = component; this.task = task; diff --git a/storm-client/src/jvm/org/apache/storm/metric/LoggingMetricsConsumer.java b/storm-client/src/jvm/org/apache/storm/metric/LoggingMetricsConsumer.java index 769163e256f..c4c361e98fc 100644 --- a/storm-client/src/jvm/org/apache/storm/metric/LoggingMetricsConsumer.java +++ b/storm-client/src/jvm/org/apache/storm/metric/LoggingMetricsConsumer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -21,22 +27,25 @@ import org.slf4j.LoggerFactory; /** - * Listens for all metrics, dumps them to log + * Listens for all metrics, dumps them to log. * *

    To use, add this to your topology's configuration: * - *

    ```java conf.registerMetricsConsumer(org.apache.storm.metrics.LoggingMetricsConsumer.class, 1); ``` + *

    ```java conf.registerMetricsConsumer(org.apache.storm.metrics.LoggingMetricsConsumer.class, + * 1); ``` * *

    Or edit the storm.yaml config file: * - *

    ```yaml topology.metrics.consumer.register: - class: "org.apache.storm.metrics.LoggingMetricsConsumer" parallelism.hint: 1 ``` + *

    ```yaml topology.metrics.consumer.register: - class: + * "org.apache.storm.metrics.LoggingMetricsConsumer" parallelism.hint: 1 ``` */ public class LoggingMetricsConsumer implements IMetricsConsumer { public static final Logger LOG = LoggerFactory.getLogger(LoggingMetricsConsumer.class); private static String padding = " "; @Override - public void prepare(Map topoConf, Object registrationArgument, TopologyContext context, IErrorReporter errorReporter) { + public void prepare(Map topoConf, Object registrationArgument, + TopologyContext context, IErrorReporter errorReporter) { } @Override diff --git a/storm-client/src/jvm/org/apache/storm/metric/MetricsConsumerBolt.java b/storm-client/src/jvm/org/apache/storm/metric/MetricsConsumerBolt.java index 311055f0ff3..fb161c7ed67 100644 --- a/storm-client/src/jvm/org/apache/storm/metric/MetricsConsumerBolt.java +++ b/storm-client/src/jvm/org/apache/storm/metric/MetricsConsumerBolt.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -43,7 +49,8 @@ public class MetricsConsumerBolt implements IBolt { private Thread taskExecuteThread; private volatile boolean running = true; - public MetricsConsumerBolt(String consumerClassName, Object registrationArgument, int maxRetainMetricTuples, + public MetricsConsumerBolt(String consumerClassName, Object registrationArgument, + int maxRetainMetricTuples, Predicate filterPredicate, DataPointExpander expander) { this.consumerClassName = consumerClassName; @@ -60,11 +67,13 @@ public MetricsConsumerBolt(String consumerClassName, Object registrationArgument } @Override - public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) { + public void prepare(Map topoConf, TopologyContext context, + OutputCollector collector) { try { metricsConsumer = (IMetricsConsumer) Class.forName(consumerClassName).newInstance(); } catch (Exception e) { - throw new RuntimeException("Could not instantiate a class listed in config under section " + throw new RuntimeException("Could not instantiate a class listed in config under " + + "section " + Config.TOPOLOGY_METRICS_CONSUMER_REGISTER + " with fully qualified name " + consumerClassName, @@ -81,8 +90,10 @@ public void prepare(Map topoConf, TopologyContext context, Outpu public void execute(Tuple input) { IMetricsConsumer.TaskInfo taskInfo = (IMetricsConsumer.TaskInfo) input.getValue(0); Collection dataPoints = (Collection) input.getValue(1); - Collection expandedDataPoints = expander.expandDataPoints(dataPoints); - List filteredDataPoints = getFilteredDataPoints(expandedDataPoints); + Collection expandedDataPoints = expander + .expandDataPoints(dataPoints); + List filteredDataPoints = + getFilteredDataPoints(expandedDataPoints); MetricsTask metricsTask = new MetricsTask(taskInfo, filteredDataPoints); while (!taskQueue.offer(metricsTask)) { @@ -107,7 +118,8 @@ static class MetricsTask { private IMetricsConsumer.TaskInfo taskInfo; private Collection dataPoints; - MetricsTask(IMetricsConsumer.TaskInfo taskInfo, Collection dataPoints) { + MetricsTask(IMetricsConsumer.TaskInfo taskInfo, + Collection dataPoints) { this.taskInfo = taskInfo; this.dataPoints = dataPoints; } diff --git a/storm-client/src/jvm/org/apache/storm/metric/SystemBolt.java b/storm-client/src/jvm/org/apache/storm/metric/SystemBolt.java index e6bc50e4a8c..660cecf3ea7 100644 --- a/storm-client/src/jvm/org/apache/storm/metric/SystemBolt.java +++ b/storm-client/src/jvm/org/apache/storm/metric/SystemBolt.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -22,7 +28,6 @@ import java.util.HashMap; import java.util.Map; import java.util.concurrent.TimeUnit; - import org.apache.storm.Config; import org.apache.storm.metric.api.IMetric; import org.apache.storm.metrics2.PerReporterGauge; @@ -44,7 +49,8 @@ public class SystemBolt implements IBolt { @SuppressWarnings({ "unchecked" }) @Override - public void prepare(final Map topoConf, TopologyContext context, OutputCollector collector) { + public void prepare(final Map topoConf, TopologyContext context, + OutputCollector collector) { if (prepareWasCalled && !"local".equals(topoConf.get(Config.STORM_CLUSTER_MODE))) { throw new RuntimeException("A single worker should have 1 SystemBolt instance."); } @@ -74,9 +80,12 @@ public Long getValue() { context.registerGauge("newWorkerEvent", new NewWorkerGauge()); context.registerGauge("workerCpuUsage", new WorkerCpuMetric()); - int bucketSize = ObjectReader.getInt(topoConf.get(Config.TOPOLOGY_BUILTIN_METRICS_BUCKET_SIZE_SECS)); - registerMetrics(context, (Map) topoConf.get(Config.WORKER_METRICS), bucketSize, topoConf); - registerMetrics(context, (Map) topoConf.get(Config.TOPOLOGY_WORKER_METRICS), bucketSize, topoConf); + int bucketSize = ObjectReader.getInt(topoConf + .get(Config.TOPOLOGY_BUILTIN_METRICS_BUCKET_SIZE_SECS)); + registerMetrics(context, (Map) topoConf.get(Config.WORKER_METRICS), + bucketSize, topoConf); + registerMetrics(context, (Map) topoConf.get(Config.TOPOLOGY_WORKER_METRICS), + bucketSize, topoConf); } private class WorkerCpuMetric implements Gauge { @@ -137,7 +146,8 @@ public int getValueAndReset() { } } - // allow reporting new worker metric for multiple reporters if they support getValueForReporter(). + // allow reporting new worker metric for multiple reporters if they support + // getValueForReporter(). private class NewWorkerGauge extends PerReporterGauge { private final NewWorkerMetric defaultValue = new NewWorkerMetric(); private final Map reporterValues = new HashMap<>(); @@ -150,11 +160,13 @@ public Integer getValue() { @Override public Integer getValueForReporter(Object reporter) { - return (Integer) reporterValues.computeIfAbsent(reporter, (rep) -> new NewWorkerMetric()).getValueAndReset(); + return (Integer) reporterValues.computeIfAbsent(reporter, + (rep) -> new NewWorkerMetric()).getValueAndReset(); } } - private void registerMetrics(TopologyContext context, Map metrics, int bucketSize, Map conf) { + private void registerMetrics(TopologyContext context, Map metrics, + int bucketSize, Map conf) { if (metrics == null) { return; } diff --git a/storm-client/src/jvm/org/apache/storm/metric/api/AssignableMetric.java b/storm-client/src/jvm/org/apache/storm/metric/api/AssignableMetric.java index 8bcc40b43da..bc8920a6352 100644 --- a/storm-client/src/jvm/org/apache/storm/metric/api/AssignableMetric.java +++ b/storm-client/src/jvm/org/apache/storm/metric/api/AssignableMetric.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/metric/api/CombinedMetric.java b/storm-client/src/jvm/org/apache/storm/metric/api/CombinedMetric.java index 9b84a989d2c..6e83afda18e 100644 --- a/storm-client/src/jvm/org/apache/storm/metric/api/CombinedMetric.java +++ b/storm-client/src/jvm/org/apache/storm/metric/api/CombinedMetric.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/metric/api/CountMetric.java b/storm-client/src/jvm/org/apache/storm/metric/api/CountMetric.java index 36df8f950c5..f591088fd0a 100644 --- a/storm-client/src/jvm/org/apache/storm/metric/api/CountMetric.java +++ b/storm-client/src/jvm/org/apache/storm/metric/api/CountMetric.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/metric/api/ICombiner.java b/storm-client/src/jvm/org/apache/storm/metric/api/ICombiner.java index 26dcb82a330..ff0902a4c8d 100644 --- a/storm-client/src/jvm/org/apache/storm/metric/api/ICombiner.java +++ b/storm-client/src/jvm/org/apache/storm/metric/api/ICombiner.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/metric/api/IMetric.java b/storm-client/src/jvm/org/apache/storm/metric/api/IMetric.java index 3ba0c187c0e..95b59c5032c 100644 --- a/storm-client/src/jvm/org/apache/storm/metric/api/IMetric.java +++ b/storm-client/src/jvm/org/apache/storm/metric/api/IMetric.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -18,7 +24,8 @@ /** * Produces metrics. * Usually, metric is a measurement identified by a name string. - * Dimensions are a collection of additional key-value metadata map containing extra information of this measurement. + * Dimensions are a collection of additional key-value metadata map containing extra information of + * this measurement. * It is optional when customizing your metric by implementing this interface */ public interface IMetric { @@ -26,7 +33,9 @@ public interface IMetric { * Get value and reset. * * @return an object that will be sent to - * {@link IMetricsConsumer#handleDataPoints(org.apache.storm.metric.api.IMetricsConsumer.TaskInfo, java.util.Collection)}. + * {@link + * IMetricsConsumer#handleDataPoints(org.apache.storm.metric.api.IMetricsConsumer.TaskInfo, + * java.util.Collection)}. * If {@code null} is returned nothing will be sent. If this value can be reset, like with a counter, a side effect * of calling this should be that the value is reset. */ @@ -34,6 +43,7 @@ public interface IMetric { /** * Get dimension map. An empty map will be returned if metric is not dimensional. + * * @return a K-V map of the additional metadata. */ default Map getDimensions() { diff --git a/storm-client/src/jvm/org/apache/storm/metric/api/IMetricsConsumer.java b/storm-client/src/jvm/org/apache/storm/metric/api/IMetricsConsumer.java index 68c64126933..5278f64d624 100644 --- a/storm-client/src/jvm/org/apache/storm/metric/api/IMetricsConsumer.java +++ b/storm-client/src/jvm/org/apache/storm/metric/api/IMetricsConsumer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -20,7 +26,8 @@ import org.apache.storm.task.TopologyContext; public interface IMetricsConsumer { - void prepare(Map topoConf, Object registrationArgument, TopologyContext context, IErrorReporter errorReporter); + void prepare(Map topoConf, Object registrationArgument, TopologyContext context, + IErrorReporter errorReporter); void handleDataPoints(TaskInfo taskInfo, Collection dataPoints); @@ -37,7 +44,8 @@ class TaskInfo { public TaskInfo() { } - public TaskInfo(String srcWorkerHost, int srcWorkerPort, String srcComponentId, int srcTaskId, long timestamp, + public TaskInfo(String srcWorkerHost, int srcWorkerPort, String srcComponentId, + int srcTaskId, long timestamp, int updateIntervalSecs) { this.srcWorkerHost = srcWorkerHost; this.srcWorkerPort = srcWorkerPort; diff --git a/storm-client/src/jvm/org/apache/storm/metric/api/IMetricsRegistrant.java b/storm-client/src/jvm/org/apache/storm/metric/api/IMetricsRegistrant.java index 54707f364a3..256dfeecb36 100644 --- a/storm-client/src/jvm/org/apache/storm/metric/api/IMetricsRegistrant.java +++ b/storm-client/src/jvm/org/apache/storm/metric/api/IMetricsRegistrant.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/metric/api/IReducer.java b/storm-client/src/jvm/org/apache/storm/metric/api/IReducer.java index 8a2da52d004..59a16af6716 100644 --- a/storm-client/src/jvm/org/apache/storm/metric/api/IReducer.java +++ b/storm-client/src/jvm/org/apache/storm/metric/api/IReducer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/metric/api/IStatefulObject.java b/storm-client/src/jvm/org/apache/storm/metric/api/IStatefulObject.java index eabf3cdf012..06dda383a41 100644 --- a/storm-client/src/jvm/org/apache/storm/metric/api/IStatefulObject.java +++ b/storm-client/src/jvm/org/apache/storm/metric/api/IStatefulObject.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/metric/api/MultiCountMetric.java b/storm-client/src/jvm/org/apache/storm/metric/api/MultiCountMetric.java index 21d89402e13..b801f2a2ce7 100644 --- a/storm-client/src/jvm/org/apache/storm/metric/api/MultiCountMetric.java +++ b/storm-client/src/jvm/org/apache/storm/metric/api/MultiCountMetric.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/metric/api/MultiReducedMetric.java b/storm-client/src/jvm/org/apache/storm/metric/api/MultiReducedMetric.java index 9b0efe0a2bf..31c3f7ed9e4 100644 --- a/storm-client/src/jvm/org/apache/storm/metric/api/MultiReducedMetric.java +++ b/storm-client/src/jvm/org/apache/storm/metric/api/MultiReducedMetric.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/metric/api/ReducedMetric.java b/storm-client/src/jvm/org/apache/storm/metric/api/ReducedMetric.java index 718d34fa6f1..39c5c310935 100644 --- a/storm-client/src/jvm/org/apache/storm/metric/api/ReducedMetric.java +++ b/storm-client/src/jvm/org/apache/storm/metric/api/ReducedMetric.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/metric/api/StateMetric.java b/storm-client/src/jvm/org/apache/storm/metric/api/StateMetric.java index cde20b4c7ee..e4d1aff28b4 100644 --- a/storm-client/src/jvm/org/apache/storm/metric/api/StateMetric.java +++ b/storm-client/src/jvm/org/apache/storm/metric/api/StateMetric.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/metric/api/rpc/AssignableShellMetric.java b/storm-client/src/jvm/org/apache/storm/metric/api/rpc/AssignableShellMetric.java index ec74f719f68..d6d50fbef42 100644 --- a/storm-client/src/jvm/org/apache/storm/metric/api/rpc/AssignableShellMetric.java +++ b/storm-client/src/jvm/org/apache/storm/metric/api/rpc/AssignableShellMetric.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/metric/api/rpc/CombinedShellMetric.java b/storm-client/src/jvm/org/apache/storm/metric/api/rpc/CombinedShellMetric.java index bffd1e73cdb..13cf88c4b82 100644 --- a/storm-client/src/jvm/org/apache/storm/metric/api/rpc/CombinedShellMetric.java +++ b/storm-client/src/jvm/org/apache/storm/metric/api/rpc/CombinedShellMetric.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/metric/api/rpc/CountShellMetric.java b/storm-client/src/jvm/org/apache/storm/metric/api/rpc/CountShellMetric.java index 7fb4b711b8a..579f54e7918 100644 --- a/storm-client/src/jvm/org/apache/storm/metric/api/rpc/CountShellMetric.java +++ b/storm-client/src/jvm/org/apache/storm/metric/api/rpc/CountShellMetric.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -30,7 +36,8 @@ public void updateMetricFromRPC(Object value) { } else if (value instanceof Number) { incrBy(((Number) value).longValue()); } else { - throw new RuntimeException("CountShellMetric updateMetricFromRPC params should be null or number."); + throw new RuntimeException("CountShellMetric updateMetricFromRPC params should be " + + "null or number."); } } } diff --git a/storm-client/src/jvm/org/apache/storm/metric/api/rpc/IShellMetric.java b/storm-client/src/jvm/org/apache/storm/metric/api/rpc/IShellMetric.java index 2b90a3eae79..1d07e551236 100644 --- a/storm-client/src/jvm/org/apache/storm/metric/api/rpc/IShellMetric.java +++ b/storm-client/src/jvm/org/apache/storm/metric/api/rpc/IShellMetric.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/metric/api/rpc/ReducedShellMetric.java b/storm-client/src/jvm/org/apache/storm/metric/api/rpc/ReducedShellMetric.java index f7abddbab1f..ab750849c09 100644 --- a/storm-client/src/jvm/org/apache/storm/metric/api/rpc/ReducedShellMetric.java +++ b/storm-client/src/jvm/org/apache/storm/metric/api/rpc/ReducedShellMetric.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/metric/cgroup/CGroupCpu.java b/storm-client/src/jvm/org/apache/storm/metric/cgroup/CGroupCpu.java index aef62eebe94..ea1f74b4ff7 100644 --- a/storm-client/src/jvm/org/apache/storm/metric/cgroup/CGroupCpu.java +++ b/storm-client/src/jvm/org/apache/storm/metric/cgroup/CGroupCpu.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -20,8 +26,8 @@ import java.util.Map; import org.apache.storm.container.cgroup.SubSystemType; import org.apache.storm.container.cgroup.core.CgroupCore; -import org.apache.storm.container.cgroup.core.CpuacctCore; import org.apache.storm.container.cgroup.core.CpuacctCore.StatType; +import org.apache.storm.container.cgroup.core.CpuacctCore; /** * Report CPU used in the cgroup. @@ -41,7 +47,8 @@ public synchronized int getUserHZ() throws IOException { if (userHz < 0) { ProcessBuilder pb = new ProcessBuilder("getconf", "CLK_TCK"); Process p = pb.start(); - BufferedReader in = new BufferedReader(new InputStreamReader(p.getInputStream(), StandardCharsets.UTF_8)); + BufferedReader in = new BufferedReader(new InputStreamReader(p.getInputStream(), + StandardCharsets.UTF_8)); String line = in.readLine().trim(); userHz = Integer.valueOf(line); } @@ -61,7 +68,7 @@ public Map getDataFrom(CgroupCore core) throws IOException { previousSystem = systemHz; long hz = getUserHZ(); HashMap ret = new HashMap<>(); - ret.put("user-ms", user * 1000 / hz); //Convert to millis + ret.put("user-ms", user * 1000 / hz); // Convert to millis ret.put("sys-ms", sys * 1000 / hz); return ret; } diff --git a/storm-client/src/jvm/org/apache/storm/metric/cgroup/CGroupCpuGuarantee.java b/storm-client/src/jvm/org/apache/storm/metric/cgroup/CGroupCpuGuarantee.java index 5fa1e0277d0..25a4619c76d 100644 --- a/storm-client/src/jvm/org/apache/storm/metric/cgroup/CGroupCpuGuarantee.java +++ b/storm-client/src/jvm/org/apache/storm/metric/cgroup/CGroupCpuGuarantee.java @@ -1,12 +1,17 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -21,7 +26,8 @@ /** * Report the guaranteed number of ms this worker has requested. * It gets the result from cpu.shares. - * Use this when org.apache.storm.container.cgroup.CgroupManager is used as the storm.resource.isolation.plugin. + * Use this when org.apache.storm.container.cgroup.CgroupManager is used as the + * storm.resource.isolation.plugin. */ @Deprecated public class CGroupCpuGuarantee extends CGroupMetricsBase { @@ -38,7 +44,7 @@ public Long getDataFrom(CgroupCore core) throws IOException { long now = System.currentTimeMillis(); if (previousTime > 0) { long shares = cpu.getCpuShares(); - //By convention each share corresponds to 1% of a CPU core + // By convention each share corresponds to 1% of a CPU core // or 100 = 1 core full time. So the guaranteed number of ms // (approximately) should be ... msGuarantee = (shares * (now - previousTime)) / 100; diff --git a/storm-client/src/jvm/org/apache/storm/metric/cgroup/CGroupCpuGuaranteeByCfsQuota.java b/storm-client/src/jvm/org/apache/storm/metric/cgroup/CGroupCpuGuaranteeByCfsQuota.java index 36dce34f96f..9c7444dacaf 100644 --- a/storm-client/src/jvm/org/apache/storm/metric/cgroup/CGroupCpuGuaranteeByCfsQuota.java +++ b/storm-client/src/jvm/org/apache/storm/metric/cgroup/CGroupCpuGuaranteeByCfsQuota.java @@ -1,12 +1,17 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -21,7 +26,8 @@ /** * Report the guaranteed number of ms this worker has requested. * It gets the result from cpu.cfs_period_us and cpu.cfs_quota_us. - * Use this when org.apache.storm.container.docker.DockerManager is used as the storm.resource.isolation.plugin. + * Use this when org.apache.storm.container.docker.DockerManager is used as the + * storm.resource.isolation.plugin. */ @Deprecated public class CGroupCpuGuaranteeByCfsQuota extends CGroupMetricsBase { @@ -39,7 +45,8 @@ public Long getDataFrom(CgroupCore core) throws IOException { if (previousTime > 0) { long cpuCfsQuotaUs = cpu.getCpuCfsQuotaUs(); if (cpuCfsQuotaUs == -1) { - //cpu.cfs_quota_us = -1 indicates that the cgroup does not adhere to any CPU time restrictions. + // cpu.cfs_quota_us = -1 indicates that the cgroup does not adhere to any CPU time + // restrictions. msGuarantee = -1L; } else { long cpuCfsPeriodUs = cpu.getCpuCfsPeriodUs(); diff --git a/storm-client/src/jvm/org/apache/storm/metric/cgroup/CGroupMemoryLimit.java b/storm-client/src/jvm/org/apache/storm/metric/cgroup/CGroupMemoryLimit.java index e306a1865e4..528bbe1700a 100644 --- a/storm-client/src/jvm/org/apache/storm/metric/cgroup/CGroupMemoryLimit.java +++ b/storm-client/src/jvm/org/apache/storm/metric/cgroup/CGroupMemoryLimit.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -30,7 +36,7 @@ public class CGroupMemoryLimit extends CGroupMetricsBase { public CGroupMemoryLimit(Map conf) { super(conf, SubSystemType.memory); - //In some cases we might be limiting memory in the supervisor and not in the cgroups + // In some cases we might be limiting memory in the supervisor and not in the cgroups long limit = -1; try { limit = Long.valueOf(System.getProperty("worker.memory_limit_mb", "-1")); diff --git a/storm-client/src/jvm/org/apache/storm/metric/cgroup/CGroupMemoryUsage.java b/storm-client/src/jvm/org/apache/storm/metric/cgroup/CGroupMemoryUsage.java index 99edccb6368..ba389fc1c89 100644 --- a/storm-client/src/jvm/org/apache/storm/metric/cgroup/CGroupMemoryUsage.java +++ b/storm-client/src/jvm/org/apache/storm/metric/cgroup/CGroupMemoryUsage.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/metric/cgroup/CGroupMetricsBase.java b/storm-client/src/jvm/org/apache/storm/metric/cgroup/CGroupMetricsBase.java index 16c5d73ef19..9b9f0bb1067 100644 --- a/storm-client/src/jvm/org/apache/storm/metric/cgroup/CGroupMetricsBase.java +++ b/storm-client/src/jvm/org/apache/storm/metric/cgroup/CGroupMetricsBase.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -41,7 +47,8 @@ public CGroupMetricsBase(Map conf, SubSystemType type) { enabled = false; CgroupCenter center = CgroupCenter.getInstance(); if (center == null) { - LOG.warn("{} is disabled. cgroups do not appear to be enabled on this system", simpleName); + LOG.warn("{} is disabled. cgroups do not appear to be enabled on this system", + simpleName); return; } if (!center.isSubSystemEnabled(type)) { @@ -49,54 +56,59 @@ public CGroupMetricsBase(Map conf, SubSystemType type) { return; } - //Check to see if the CGroup is mounted at all + // Check to see if the CGroup is mounted at all if (null == center.getHierarchyWithSubSystem(type)) { LOG.warn("{} is disabled. {} is not a mounted subsystem", simpleName, type); return; } - //Good so far, check if we are in a CGroup + // Good so far, check if we are in a CGroup File cgroupFile = new File("/proc/self/cgroup"); if (!cgroupFile.exists()) { - LOG.warn("{} is disabled we do not appear to be a part of a CGroup", getClass().getSimpleName()); + LOG.warn("{} is disabled we do not appear to be a part of a CGroup", getClass() + .getSimpleName()); return; } String cgroupPath; try (BufferedReader reader = new BufferedReader(new FileReader(cgroupFile))) { - //There can be more then one line if cgroups are mounted in more then one place, but we assume the first is good enough + // There can be more then one line if cgroups are mounted in more then one place, but we + // assume the first is good enough String line = reader.readLine(); - //hierarchy-ID:controller-list:cgroup-path + // hierarchy-ID:controller-list:cgroup-path String[] parts = line.split(":"); - //parts[0] == 0 for CGroup V2, else maps to hierarchy in /proc/cgroups - //parts[1] is empty for CGroups V2 else what is mapped that we are looking for + // parts[0] == 0 for CGroup V2, else maps to hierarchy in /proc/cgroups + // parts[1] is empty for CGroups V2 else what is mapped that we are looking for cgroupPath = parts[2]; } catch (Exception e) { LOG.warn("{} is disabled error trying to read or parse {}", simpleName, cgroupFile); return; } - //Storm on Rhel6 and Rhel7 use different cgroup settings. - //On Rhel6, the cgroup of the worker is under + // Storm on Rhel6 and Rhel7 use different cgroup settings. + // On Rhel6, the cgroup of the worker is under // "Config.STORM_CGROUP_HIERARCHY_DIR/DaemonConfig.STORM_SUPERVISOR_CGROUP_ROOTDIR/" - //On Rhel7, the cgroup of the worker is under + // On Rhel7, the cgroup of the worker is under // "Config.STORM_OCI_CGROUP_ROOT//DaemonConfig.STORM_OCI_CGROUP_PARENT/" // This block of code is a workaround for the CGroupMetrics to work on both system String hierarchyDir = (String) conf.get(Config.STORM_CGROUP_HIERARCHY_DIR); if (StringUtils.isEmpty(hierarchyDir) || !new File(hierarchyDir, cgroupPath).exists()) { - LOG.info("{} is not set or does not exist. checking {}", Config.STORM_CGROUP_HIERARCHY_DIR, + LOG.info("{} is not set or does not exist. checking {}", + Config.STORM_CGROUP_HIERARCHY_DIR, Config.STORM_OCI_CGROUP_ROOT); String ociCgroupRoot = (String) conf.get(Config.STORM_OCI_CGROUP_ROOT); hierarchyDir = ociCgroupRoot + File.separator + type; - if (StringUtils.isEmpty(ociCgroupRoot) || !new File(hierarchyDir, cgroupPath).exists()) { + if (StringUtils.isEmpty(ociCgroupRoot) || !new File(hierarchyDir, cgroupPath) + .exists()) { LOG.info("{} is not set or does not exist", Config.STORM_OCI_CGROUP_ROOT); LOG.warn("{} is disabled", simpleName); return; } } - core = CgroupCoreFactory.getInstance(type, new File(hierarchyDir, cgroupPath).getAbsolutePath()); + core = CgroupCoreFactory.getInstance(type, new File(hierarchyDir, cgroupPath) + .getAbsolutePath()); enabled = true; LOG.info("{} is ENABLED {} exists...", simpleName, hierarchyDir); @@ -111,7 +123,7 @@ public Object getValueAndReset() { return getDataFrom(core); } catch (FileNotFoundException e) { LOG.warn("Exception trying to read a file {}", e); - //Something happened and we couldn't find the file, so ignore it for now. + // Something happened and we couldn't find the file, so ignore it for now. return null; } catch (Exception e) { throw new RuntimeException(e); diff --git a/storm-client/src/jvm/org/apache/storm/metric/filter/FilterByMetricName.java b/storm-client/src/jvm/org/apache/storm/metric/filter/FilterByMetricName.java index a278a9519cf..50cf91f8e66 100644 --- a/storm-client/src/jvm/org/apache/storm/metric/filter/FilterByMetricName.java +++ b/storm-client/src/jvm/org/apache/storm/metric/filter/FilterByMetricName.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -46,7 +52,8 @@ public FilterByMetricName(List whitelistPattern, List blacklistP if (this.whitelistPattern.isEmpty() && this.blacklistPattern.isEmpty()) { noneSpecified = true; } else if (!this.whitelistPattern.isEmpty() && !this.blacklistPattern.isEmpty()) { - throw new IllegalArgumentException("You have to specify either includes or excludes, or none."); + throw new IllegalArgumentException("You have to specify either includes or excludes, " + + "or none."); } filterCache = CacheBuilder.newBuilder() @@ -73,7 +80,8 @@ public boolean apply(IMetricsConsumer.DataPoint dataPoint) { } private ArrayList convertPatternStringsToPatternInstances(List patterns) { - return Lists.newArrayList(Iterators.transform(patterns.iterator(), s -> Pattern.compile(s))); + return Lists.newArrayList(Iterators.transform(patterns.iterator(), s -> Pattern + .compile(s))); } private boolean isFilteredIn(String metricName) { @@ -86,7 +94,8 @@ private boolean isFilteredIn(String metricName) { throw new IllegalStateException("Shouldn't reach here"); } - private boolean checkMatching(String metricName, List patterns, boolean valueWhenMatched) { + private boolean checkMatching(String metricName, List patterns, + boolean valueWhenMatched) { for (Pattern pattern : patterns) { if (pattern.matcher(metricName).find()) { return valueWhenMatched; diff --git a/storm-client/src/jvm/org/apache/storm/metric/filter/MetricsFilter.java b/storm-client/src/jvm/org/apache/storm/metric/filter/MetricsFilter.java index c1165760a51..133d0968e6d 100644 --- a/storm-client/src/jvm/org/apache/storm/metric/filter/MetricsFilter.java +++ b/storm-client/src/jvm/org/apache/storm/metric/filter/MetricsFilter.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/metric/internal/CountStat.java b/storm-client/src/jvm/org/apache/storm/metric/internal/CountStat.java index 31ce455f66c..63fbc6d07e8 100644 --- a/storm-client/src/jvm/org/apache/storm/metric/internal/CountStat.java +++ b/storm-client/src/jvm/org/apache/storm/metric/internal/CountStat.java @@ -1,12 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -22,15 +27,15 @@ */ public class CountStat { private final AtomicLong currentBucket; - //10 min values + // 10 min values private final int tmSize; private final long[] tmBuckets; private final long[] tmTime; - //3 hour values + // 3 hour values private final int thSize; private final long[] thBuckets; private final long[] thTime; - //1 day values + // 1 day values private final int odSize; private final long[] odBuckets; private final long[] odTime; @@ -38,9 +43,9 @@ public class CountStat { // All internal state except for the count of the current bucket are // protected using a lock on this counter private long bucketStart; - //exact variable time, that is added to the current bucket + // exact variable time, that is added to the current bucket private long exactExtra; - //all time + // all time private long allTime; /** @@ -60,7 +65,7 @@ public CountStat(int numBuckets) { */ CountStat(int numBuckets, long startTime) { numBuckets = Math.max(numBuckets, 2); - //We want to capture the full time range, so the target size is as + // We want to capture the full time range, so the target size is as // if we had one bucket less, then we do tmSize = 10 * 60 * 1000 / (numBuckets - 1); thSize = 3 * 60 * 60 * 1000 / (numBuckets - 1); @@ -111,7 +116,8 @@ synchronized void rotateBuckets(long value, long timeSpent) { allTime += value; } - private synchronized void rotate(long value, long timeSpent, long targetSize, long[] times, long[] buckets) { + private synchronized void rotate(long value, long timeSpent, long targetSize, long[] times, + long[] buckets) { times[0] += timeSpent; buckets[0] += value; @@ -132,7 +138,9 @@ private synchronized void rotate(long value, long timeSpent, long targetSize, lo /** * Get time counts. - * @return a map of time window to count. Keys are "600" for last 10 mins "10800" for the last 3 hours "86400" for the last day + * + * @return a map of time window to count. Keys are "600" for last 10 mins "10800" for the last 3 + * hours "86400" for the last day * ":all-time" for all time */ public synchronized Map getTimeCounts() { @@ -150,7 +158,8 @@ synchronized Map getTimeCounts(long now) { return ret; } - long readApproximateTime(long value, long timeSpent, long[] bucketTime, long[] buckets, long desiredTime) { + long readApproximateTime(long value, long timeSpent, long[] bucketTime, long[] buckets, + long desiredTime) { long timeNeeded = desiredTime - timeSpent; long total = value; for (int i = 0; i < bucketTime.length; i++) { diff --git a/storm-client/src/jvm/org/apache/storm/metric/internal/LatencyStat.java b/storm-client/src/jvm/org/apache/storm/metric/internal/LatencyStat.java index a1d385956fe..9bf429379ae 100644 --- a/storm-client/src/jvm/org/apache/storm/metric/internal/LatencyStat.java +++ b/storm-client/src/jvm/org/apache/storm/metric/internal/LatencyStat.java @@ -27,22 +27,22 @@ * Keeps track of approximate latency for the last 10 mins, 3 hours, 1 day, and all time. */ public class LatencyStat { - //The current lat and count buckets are protected by a different lock + // The current lat and count buckets are protected by a different lock // from the other buckets. This is to reduce the lock contention // When doing complex calculations. Never grab the instance object lock // while holding currentLock to avoid deadlocks private final Object currentLock = new byte[0]; - //10 min values + // 10 min values private final int tmSize; private final long[] tmLatBuckets; private final long[] tmCountBuckets; private final long[] tmTime; - //3 hour values + // 3 hour values private final int thSize; private final long[] thLatBuckets; private final long[] thCountBuckets; private final long[] thTime; - //1 day values + // 1 day values private final int odSize; private final long[] odLatBuckets; private final long[] odCountBuckets; @@ -53,10 +53,10 @@ public class LatencyStat { // All internal state except for the current buckets are // protected using the Object Lock private long bucketStart; - //exact variable time, that is added to the current bucket + // exact variable time, that is added to the current bucket private long exactExtraLat; private long exactExtraCount; - //all time + // all time private long allTimeLat; private long allTimeCount; @@ -77,7 +77,7 @@ public LatencyStat(int numBuckets) { */ LatencyStat(int numBuckets, long startTime) { numBuckets = Math.max(numBuckets, 2); - //We want to capture the full time range, so the target size is as + // We want to capture the full time range, so the target size is as // if we had one bucket less, then we do tmSize = 10 * 60 * 1000 / (numBuckets - 1); thSize = 3 * 60 * 60 * 1000 / (numBuckets - 1); @@ -200,7 +200,9 @@ private synchronized void rotate(long lat, long count, long timeSpent, long targ /** * Get time latency average. - * @return a map of time window to average latency. Keys are "600" for last 10 mins "10800" for the last 3 hours "86400" for the last + * + * @return a map of time window to average latency. Keys are "600" for last 10 mins "10800" for + * the last 3 hours "86400" for the last * day ":all-time" for all time */ public synchronized Map getTimeLatAvg() { @@ -216,9 +218,12 @@ synchronized Map getTimeLatAvg(long now) { count = currentCountBucket; } long timeSpent = now - bucketStart; - ret.put("600", readApproximateLatAvg(lat, count, timeSpent, tmTime, tmLatBuckets, tmCountBuckets, 600 * 1000)); - ret.put("10800", readApproximateLatAvg(lat, count, timeSpent, thTime, thLatBuckets, thCountBuckets, 10800 * 1000)); - ret.put("86400", readApproximateLatAvg(lat, count, timeSpent, odTime, odLatBuckets, odCountBuckets, 86400 * 1000)); + ret.put("600", readApproximateLatAvg(lat, count, timeSpent, tmTime, tmLatBuckets, + tmCountBuckets, 600 * 1000)); + ret.put("10800", readApproximateLatAvg(lat, count, timeSpent, thTime, thLatBuckets, + thCountBuckets, 10800 * 1000)); + ret.put("86400", readApproximateLatAvg(lat, count, timeSpent, odTime, odLatBuckets, + odCountBuckets, 86400 * 1000)); long allTimeCountSum = count + allTimeCount; ret.put(":all-time", Utils.zeroIfNaNOrInf( (double) lat + allTimeLat) / allTimeCountSum); @@ -231,7 +236,7 @@ synchronized Map getTimeLatAvg(long now) { long totalLat = lat; long totalCount = count; for (int i = 0; i < bucketTime.length && timeNeeded > 0; i++) { - //Don't pro-rate anything, it is all approximate so an extra bucket is not that bad. + // Don't pro-rate anything, it is all approximate so an extra bucket is not that bad. totalLat += latBuckets[i]; totalCount += countBuckets[i]; timeNeeded -= bucketTime[i]; diff --git a/storm-client/src/jvm/org/apache/storm/metric/internal/MetricStatTimer.java b/storm-client/src/jvm/org/apache/storm/metric/internal/MetricStatTimer.java index 41b87a5bf43..61284ca6bbe 100644 --- a/storm-client/src/jvm/org/apache/storm/metric/internal/MetricStatTimer.java +++ b/storm-client/src/jvm/org/apache/storm/metric/internal/MetricStatTimer.java @@ -1,12 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/metric/internal/MultiCountStat.java b/storm-client/src/jvm/org/apache/storm/metric/internal/MultiCountStat.java index 8bf18d56901..f0cc765b74b 100644 --- a/storm-client/src/jvm/org/apache/storm/metric/internal/MultiCountStat.java +++ b/storm-client/src/jvm/org/apache/storm/metric/internal/MultiCountStat.java @@ -1,12 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -18,7 +23,8 @@ import java.util.concurrent.ConcurrentHashMap; /** - * Acts as a MultiCount Stat, but keeps track of approximate counts for the last 10 mins, 3 hours, 1 day, and all time. for the same keys + * Acts as a MultiCount Stat, but keeps track of approximate counts for the last 10 mins, 3 hours, 1 + * day, and all time. for the same keys */ public class MultiCountStat { public static final int TEN_MIN_IN_SECONDS = 60 * 10; @@ -60,8 +66,8 @@ public void incBy(T key, long count) { protected String keyToString(T key) { if (key instanceof List) { - //This is a bit of a hack. If it is a list, then it is [component, stream] - //we want to format this as component:stream + // This is a bit of a hack. If it is a list, then it is [component, stream] + // we want to format this as component:stream List lk = (List) key; return lk.get(0) + ":" + lk.get(1); } diff --git a/storm-client/src/jvm/org/apache/storm/metric/internal/MultiLatencyStat.java b/storm-client/src/jvm/org/apache/storm/metric/internal/MultiLatencyStat.java index 83c4f7a22ba..c84974addca 100644 --- a/storm-client/src/jvm/org/apache/storm/metric/internal/MultiLatencyStat.java +++ b/storm-client/src/jvm/org/apache/storm/metric/internal/MultiLatencyStat.java @@ -1,12 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/metric/internal/RateTracker.java b/storm-client/src/jvm/org/apache/storm/metric/internal/RateTracker.java index 94f34335b79..b841f1ed172 100644 --- a/storm-client/src/jvm/org/apache/storm/metric/internal/RateTracker.java +++ b/storm-client/src/jvm/org/apache/storm/metric/internal/RateTracker.java @@ -1,12 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -21,7 +26,8 @@ */ public class RateTracker implements Closeable { private final int bucketSizeMillis; - //Old Buckets and their length are only touched when rotating or gathering the metrics, which should not be that frequent + // Old Buckets and their length are only touched when rotating or gathering the metrics, which + // should not be that frequent // As such all access to them should be protected by synchronizing with the RateTracker instance private final long[] bucketTime; private final long[] oldBuckets; @@ -34,8 +40,10 @@ public class RateTracker implements Closeable { /** * Constructor. * - * @param validTimeWindowInMils events that happened before validTimeWindowInMils are not considered when reporting the rate. - * @param numBuckets the number of time sildes to divide validTimeWindows. The more buckets, the smother the reported results + * @param validTimeWindowInMils events that happened before validTimeWindowInMils are not + * considered when reporting the rate. + * @param numBuckets the number of time sildes to divide validTimeWindows. The more buckets, the + * smother the reported results * will be. */ public RateTracker(int validTimeWindowInMils, int numBuckets) { @@ -45,8 +53,10 @@ public RateTracker(int validTimeWindowInMils, int numBuckets) { /** * Constructor. * - * @param validTimeWindowInMils events that happened before validTimeWindow are not considered when reporting the rate. - * @param numBuckets the number of time sildes to divide validTimeWindows. The more buckets, the smother the reported results + * @param validTimeWindowInMils events that happened before validTimeWindow are not considered + * when reporting the rate. + * @param numBuckets the number of time sildes to divide validTimeWindows. The more buckets, the + * smother the reported results * will be. * @param startTime if positive the simulated time to start the first bucket at. */ @@ -55,7 +65,8 @@ public RateTracker(int validTimeWindowInMils, int numBuckets) { bucketSizeMillis = validTimeWindowInMils / numBuckets; if (bucketSizeMillis < 1) { throw new IllegalArgumentException( - "validTimeWindowInMilis and numOfSildes cause each slide to have a window that is too small"); + "validTimeWindowInMilis and numOfSildes cause each slide to have a window that is " + + "too small"); } bucketTime = new long[numBuckets - 1]; oldBuckets = new long[numBuckets - 1]; @@ -81,6 +92,7 @@ public void notify(long count) { /** * Get report rate. + * * @return the approximate average rate per second. */ public synchronized double reportRate() { diff --git a/storm-client/src/jvm/org/apache/storm/metric/util/DataPointExpander.java b/storm-client/src/jvm/org/apache/storm/metric/util/DataPointExpander.java index e3f9947ce7a..d7f6c32f761 100644 --- a/storm-client/src/jvm/org/apache/storm/metric/util/DataPointExpander.java +++ b/storm-client/src/jvm/org/apache/storm/metric/util/DataPointExpander.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -60,8 +66,10 @@ public Collection expandDataPoint(IMetricsConsumer.D Map dataMap = (Map) dataPoint.value; for (Map.Entry entry : dataMap.entrySet()) { - String expandedDataPointName = dataPoint.name + metricNameSeparator + String.valueOf(entry.getKey()); - dataPoints.add(new IMetricsConsumer.DataPoint(expandedDataPointName, entry.getValue())); + String expandedDataPointName = dataPoint.name + metricNameSeparator + String + .valueOf(entry.getKey()); + dataPoints.add(new IMetricsConsumer.DataPoint(expandedDataPointName, entry + .getValue())); } } else { dataPoints.add(dataPoint); diff --git a/storm-client/src/jvm/org/apache/storm/metrics2/DimensionalReporter.java b/storm-client/src/jvm/org/apache/storm/metrics2/DimensionalReporter.java index fcf7fbef6d9..77c2634cc49 100644 --- a/storm-client/src/jvm/org/apache/storm/metrics2/DimensionalReporter.java +++ b/storm-client/src/jvm/org/apache/storm/metrics2/DimensionalReporter.java @@ -1,12 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -25,11 +30,16 @@ import java.util.concurrent.TimeUnit; /** - * Class that allows using a ScheduledReporter to report V2 task metrics with support for dimensions. + * Class that allows using a ScheduledReporter to report V2 task metrics with support for + * dimensions. + * *

    - * This reporter will be started and scheduled with the MetricRegistry. Once it is called on to report, - * it will query the StormMetricRegistry for various sets of task metrics with unique dimensions. The - * underlying ScheduledReporter will perform the actual reporting with the help of a DimensionHandler to + * This reporter will be started and scheduled with the MetricRegistry. Once it is called on to + * report, + * it will query the StormMetricRegistry for various sets of task metrics with unique dimensions. + * The + * underlying ScheduledReporter will perform the actual reporting with the help of a + * DimensionHandler to * deal with the dimensions. */ public class DimensionalReporter extends ScheduledReporter { @@ -42,14 +52,16 @@ public class DimensionalReporter extends ScheduledReporter { * Constructor. * * @param metricRegistryProvider MetricRegistryProvider tracking task-specific metrics. - * @param unstartedReporter ScheduledReporter to perform the actual reporting. It should NOT be started. + * @param unstartedReporter ScheduledReporter to perform the actual reporting. It should NOT be + * started. * @param dimensionHandler class to handle setting dimensions before reporting a set of metrics. * @param name the reporter's name. * @param filter the filter for which metrics to report. * @param rateUnit rate unit for the reporter. * @param durationUnit duration unit for the reporter. * @param executor the executor to use while scheduling reporting of metrics. - * @param shutdownExecutorOnStop if true, then executor will be stopped in same time with this reporter. + * @param shutdownExecutorOnStop if true, then executor will be stopped in same time with this + * reporter. */ public DimensionalReporter(MetricRegistryProvider metricRegistryProvider, ScheduledReporter unstartedReporter, @@ -60,7 +72,8 @@ public DimensionalReporter(MetricRegistryProvider metricRegistryProvider, TimeUnit durationUnit, ScheduledExecutorService executor, boolean shutdownExecutorOnStop) { - super(metricRegistryProvider.getRegistry(), name, filter, rateUnit, durationUnit, executor, shutdownExecutorOnStop); + super(metricRegistryProvider.getRegistry(), name, filter, rateUnit, durationUnit, executor, + shutdownExecutorOnStop); underlyingReporter = unstartedReporter; this.metricRegistryProvider = metricRegistryProvider; this.filter = filter; @@ -76,7 +89,8 @@ public void report(SortedMap gauges, SortedMap c @Override public void report() { - for (Map.Entry entry : metricRegistryProvider.getTaskMetrics().entrySet()) { + for (Map.Entry entry : metricRegistryProvider + .getTaskMetrics().entrySet()) { TaskMetricRepo repo = entry.getValue(); if (dimensionHandler != null) { TaskMetricDimensions dimensions = entry.getKey(); diff --git a/storm-client/src/jvm/org/apache/storm/metrics2/EwmaGauge.java b/storm-client/src/jvm/org/apache/storm/metrics2/EwmaGauge.java index 857e34215f0..3be246e1b65 100644 --- a/storm-client/src/jvm/org/apache/storm/metrics2/EwmaGauge.java +++ b/storm-client/src/jvm/org/apache/storm/metrics2/EwmaGauge.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -15,7 +21,6 @@ import static org.apache.storm.utils.ConfigUtils.RFC1889_ALPHA; import com.codahale.metrics.Gauge; - import java.util.concurrent.atomic.AtomicLong; /** diff --git a/storm-client/src/jvm/org/apache/storm/metrics2/MetricRegistryProvider.java b/storm-client/src/jvm/org/apache/storm/metrics2/MetricRegistryProvider.java index 889b236f8a7..588afaaaf36 100644 --- a/storm-client/src/jvm/org/apache/storm/metrics2/MetricRegistryProvider.java +++ b/storm-client/src/jvm/org/apache/storm/metrics2/MetricRegistryProvider.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/metrics2/PerReporterGauge.java b/storm-client/src/jvm/org/apache/storm/metrics2/PerReporterGauge.java index c5507cd2f37..0e45d9075e4 100644 --- a/storm-client/src/jvm/org/apache/storm/metrics2/PerReporterGauge.java +++ b/storm-client/src/jvm/org/apache/storm/metrics2/PerReporterGauge.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/metrics2/RateCounter.java b/storm-client/src/jvm/org/apache/storm/metrics2/RateCounter.java index 4c9b563f556..98fd66be895 100644 --- a/storm-client/src/jvm/org/apache/storm/metrics2/RateCounter.java +++ b/storm-client/src/jvm/org/apache/storm/metrics2/RateCounter.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -16,7 +22,8 @@ import com.codahale.metrics.Gauge; /** - * A Counter metric that also implements a Gauge to report the average rate of events per second over 1 minute. This class + * A Counter metric that also implements a Gauge to report the average rate of events per second + * over 1 minute. This class * was added as a compromise to using a Meter, which has a much larger performance impact. */ public class RateCounter implements Gauge { @@ -38,9 +45,11 @@ public class RateCounter implements Gauge { metricRegistry.gauge(metricName + ".m1_rate", this, componentId, taskId); } - this.timeSpanInSeconds = Math.max(60 - (60 % metricRegistry.getRateCounterUpdateIntervalSeconds()), + this.timeSpanInSeconds = Math.max(60 - (60 % metricRegistry + .getRateCounterUpdateIntervalSeconds()), metricRegistry.getRateCounterUpdateIntervalSeconds()); - this.values = new long[this.timeSpanInSeconds / metricRegistry.getRateCounterUpdateIntervalSeconds() + 1]; + this.values = new long[this.timeSpanInSeconds / metricRegistry + .getRateCounterUpdateIntervalSeconds() + 1]; } @@ -51,6 +60,7 @@ public class RateCounter implements Gauge { /** * Reports the the average rate of events per second over 1 minute for the metric. + * * @return the rate */ @Override @@ -68,7 +78,8 @@ public void inc(long n) { void update() { time = (time + 1) % values.length; values[time] = counter.getCount(); - currentRate = ((double) (values[time] - values[(time + 1) % values.length]) / timeSpanInSeconds); + currentRate = ((double) (values[time] - values[(time + + 1) % values.length]) / timeSpanInSeconds); } Counter getCounter() { diff --git a/storm-client/src/jvm/org/apache/storm/metrics2/RollingAverageGauge.java b/storm-client/src/jvm/org/apache/storm/metrics2/RollingAverageGauge.java index 9a50bb92f4f..57cfa04c9e0 100644 --- a/storm-client/src/jvm/org/apache/storm/metrics2/RollingAverageGauge.java +++ b/storm-client/src/jvm/org/apache/storm/metrics2/RollingAverageGauge.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

    - * http://www.apache.org/licenses/LICENSE-2.0 - *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

    http://www.apache.org/licenses/LICENSE-2.0 + * + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/metrics2/SimpleGauge.java b/storm-client/src/jvm/org/apache/storm/metrics2/SimpleGauge.java index 16d71190649..51c68f26fc2 100644 --- a/storm-client/src/jvm/org/apache/storm/metrics2/SimpleGauge.java +++ b/storm-client/src/jvm/org/apache/storm/metrics2/SimpleGauge.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/metrics2/StormMetricRegistry.java b/storm-client/src/jvm/org/apache/storm/metrics2/StormMetricRegistry.java index 7a836b04f64..3ba0394f1a7 100644 --- a/storm-client/src/jvm/org/apache/storm/metrics2/StormMetricRegistry.java +++ b/storm-client/src/jvm/org/apache/storm/metrics2/StormMetricRegistry.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -50,12 +56,18 @@ public class StormMetricRegistry implements MetricRegistryProvider { private final MetricRegistry registry = new MetricRegistry(); private final List reporters = new ArrayList<>(); - private final ConcurrentMap> taskIdGauges = new ConcurrentHashMap<>(); - private final ConcurrentMap> taskIdMeters = new ConcurrentHashMap<>(); - private final ConcurrentMap> taskIdCounters = new ConcurrentHashMap<>(); - private final ConcurrentMap> taskIdTimers = new ConcurrentHashMap<>(); - private final ConcurrentMap> taskIdHistograms = new ConcurrentHashMap<>(); - private final ConcurrentMap taskMetrics = new ConcurrentHashMap<>(); + private final ConcurrentMap> taskIdGauges = + new ConcurrentHashMap<>(); + private final ConcurrentMap> taskIdMeters = + new ConcurrentHashMap<>(); + private final ConcurrentMap> taskIdCounters = + new ConcurrentHashMap<>(); + private final ConcurrentMap> taskIdTimers = + new ConcurrentHashMap<>(); + private final ConcurrentMap> taskIdHistograms = + new ConcurrentHashMap<>(); + private final ConcurrentMap taskMetrics = + new ConcurrentHashMap<>(); private String hostName = null; private int port = -1; private String topologyId = null; @@ -88,13 +100,15 @@ public SimpleGauge gauge( public Gauge gauge(String name, Gauge gauge, TopologyContext context) { MetricNames metricNames = topologyMetricName(name, context); - gauge = registerGauge(metricNames, gauge, context.getThisTaskId(), context.getThisComponentId(), null); + gauge = registerGauge(metricNames, gauge, context.getThisTaskId(), context + .getThisComponentId(), null); saveMetricTaskIdMapping(context.getThisTaskId(), metricNames, gauge, taskIdGauges); return gauge; } @Deprecated - public Gauge gauge(String name, Gauge gauge, String topologyId, String componentId, Integer taskId, Integer port) { + public Gauge gauge(String name, Gauge gauge, String topologyId, String componentId, + Integer taskId, Integer port) { MetricNames metricNames = workerMetricName(name, topologyId, componentId, taskId, port); gauge = registerGauge(metricNames, gauge, taskId, componentId, null); saveMetricTaskIdMapping(taskId, metricNames, gauge, taskIdGauges); @@ -110,21 +124,26 @@ public Gauge gauge(String name, Gauge gauge, String componentId, Integ public Gauge gauge(String name, Gauge gauge, String topologyId, String componentId, String streamId, Integer taskId, Integer port) { - MetricNames metricNames = workerMetricName(name, topologyId, componentId, streamId, taskId, port); + MetricNames metricNames = workerMetricName(name, topologyId, componentId, streamId, taskId, + port); gauge = registerGauge(metricNames, gauge, taskId, componentId, streamId); saveMetricTaskIdMapping(taskId, metricNames, gauge, taskIdGauges); return gauge; } - public Meter meter(String name, WorkerTopologyContext context, String componentId, Integer taskId, String streamId) { - MetricNames metricNames = workerMetricName(name, context.getStormId(), componentId, streamId, taskId, context.getThisWorkerPort()); + public Meter meter(String name, WorkerTopologyContext context, String componentId, + Integer taskId, String streamId) { + MetricNames metricNames = workerMetricName(name, context.getStormId(), componentId, + streamId, taskId, context.getThisWorkerPort()); Meter meter = registerMeter(metricNames, new Meter(), taskId, componentId, streamId); saveMetricTaskIdMapping(taskId, metricNames, meter, taskIdMeters); return meter; } - public Meter meter(String name, WorkerTopologyContext context, String componentId, Integer taskId) { - MetricNames metricNames = workerMetricName(name, context.getStormId(), componentId, taskId, context.getThisWorkerPort()); + public Meter meter(String name, WorkerTopologyContext context, String componentId, + Integer taskId) { + MetricNames metricNames = workerMetricName(name, context.getStormId(), componentId, taskId, + context.getThisWorkerPort()); Meter meter = registerMeter(metricNames, new Meter(), taskId, componentId, null); saveMetricTaskIdMapping(taskId, metricNames, meter, taskIdMeters); return meter; @@ -132,28 +151,36 @@ public Meter meter(String name, WorkerTopologyContext context, String componentI public Meter meter(String name, TopologyContext context) { MetricNames metricNames = topologyMetricName(name, context); - Meter meter = registerMeter(metricNames, new Meter(), context.getThisTaskId(), context.getThisComponentId(), null); + Meter meter = registerMeter(metricNames, new Meter(), context.getThisTaskId(), context + .getThisComponentId(), null); saveMetricTaskIdMapping(context.getThisTaskId(), metricNames, meter, taskIdMeters); return meter; } - public Counter counter(String name, WorkerTopologyContext context, String componentId, Integer taskId, String streamId) { - MetricNames metricNames = workerMetricName(name, context.getStormId(), componentId, streamId, taskId, context.getThisWorkerPort()); - Counter counter = registerCounter(metricNames, new Counter(), taskId, componentId, streamId); + public Counter counter(String name, WorkerTopologyContext context, String componentId, + Integer taskId, String streamId) { + MetricNames metricNames = workerMetricName(name, context.getStormId(), componentId, + streamId, taskId, context.getThisWorkerPort()); + Counter counter = registerCounter(metricNames, new Counter(), taskId, componentId, + streamId); saveMetricTaskIdMapping(taskId, metricNames, counter, taskIdCounters); return counter; } - public Counter counter(String name, String topologyId, String componentId, Integer taskId, Integer workerPort, String streamId) { - MetricNames metricNames = workerMetricName(name, topologyId, componentId, streamId, taskId, workerPort); - Counter counter = registerCounter(metricNames, new Counter(), taskId, componentId, streamId); + public Counter counter(String name, String topologyId, String componentId, Integer taskId, + Integer workerPort, String streamId) { + MetricNames metricNames = workerMetricName(name, topologyId, componentId, streamId, taskId, + workerPort); + Counter counter = registerCounter(metricNames, new Counter(), taskId, componentId, + streamId); saveMetricTaskIdMapping(taskId, metricNames, counter, taskIdCounters); return counter; } public Counter counter(String name, TopologyContext context) { MetricNames metricNames = topologyMetricName(name, context); - Counter counter = registerCounter(metricNames, new Counter(), context.getThisTaskId(), context.getThisComponentId(), null); + Counter counter = registerCounter(metricNames, new Counter(), context.getThisTaskId(), + context.getThisComponentId(), null); saveMetricTaskIdMapping(context.getThisTaskId(), metricNames, counter, taskIdCounters); return counter; } @@ -167,14 +194,16 @@ public Counter counter(String name, String componentId, Integer taskId) { public Timer timer(String name, TopologyContext context) { MetricNames metricNames = topologyMetricName(name, context); - Timer timer = registerTimer(metricNames, new Timer(), context.getThisTaskId(), context.getThisComponentId(), null); + Timer timer = registerTimer(metricNames, new Timer(), context.getThisTaskId(), context + .getThisComponentId(), null); saveMetricTaskIdMapping(context.getThisTaskId(), metricNames, timer, taskIdTimers); return timer; } public Histogram histogram(String name, TopologyContext context) { MetricNames metricNames = topologyMetricName(name, context); - Histogram histogram = registerHistogram(metricNames, new Histogram(new ExponentiallyDecayingReservoir()), + Histogram histogram = registerHistogram(metricNames, + new Histogram(new ExponentiallyDecayingReservoir()), context.getThisTaskId(), context.getThisComponentId(), null); saveMetricTaskIdMapping(context.getThisTaskId(), metricNames, histogram, taskIdHistograms); return histogram; @@ -188,46 +217,62 @@ public void metricSet(String prefix, MetricSet set, TopologyContext context) { MetricNames metricNames = topologyMetricName(prefix + "." + entry.getKey(), context); Metric metric = entry.getValue(); if (metric instanceof Gauge) { - registerGauge(metricNames, (Gauge) metric, context.getThisTaskId(), context.getThisComponentId(), null); - saveMetricTaskIdMapping(context.getThisTaskId(), metricNames, (Gauge) metric, taskIdGauges); + registerGauge(metricNames, (Gauge) metric, context.getThisTaskId(), context + .getThisComponentId(), null); + saveMetricTaskIdMapping(context.getThisTaskId(), metricNames, (Gauge) metric, + taskIdGauges); } else if (metric instanceof Meter) { - registerMeter(metricNames, (Meter) metric, context.getThisTaskId(), context.getThisComponentId(), null); - saveMetricTaskIdMapping(context.getThisTaskId(), metricNames, (Meter) metric, taskIdMeters); + registerMeter(metricNames, (Meter) metric, context.getThisTaskId(), context + .getThisComponentId(), null); + saveMetricTaskIdMapping(context.getThisTaskId(), metricNames, (Meter) metric, + taskIdMeters); } else if (metric instanceof Counter) { registerCounter(metricNames, (Counter) metric, context.getThisTaskId(), context.getThisComponentId(), null); - saveMetricTaskIdMapping(context.getThisTaskId(), metricNames, (Counter) metric, taskIdCounters); + saveMetricTaskIdMapping(context.getThisTaskId(), metricNames, (Counter) metric, + taskIdCounters); } else if (metric instanceof Timer) { - registerTimer(metricNames, (Timer) metric, context.getThisTaskId(), context.getThisComponentId(), null); - saveMetricTaskIdMapping(context.getThisTaskId(), metricNames, (Timer) metric, taskIdTimers); + registerTimer(metricNames, (Timer) metric, context.getThisTaskId(), context + .getThisComponentId(), null); + saveMetricTaskIdMapping(context.getThisTaskId(), metricNames, (Timer) metric, + taskIdTimers); } else if (metric instanceof Histogram) { registerHistogram(metricNames, (Histogram) metric, context.getThisTaskId(), context.getThisComponentId(), null); - saveMetricTaskIdMapping(context.getThisTaskId(), metricNames, (Histogram) metric, taskIdHistograms); + saveMetricTaskIdMapping(context.getThisTaskId(), metricNames, (Histogram) metric, + taskIdHistograms); } else { - LOG.error("Unable to save taskId mapping for metric {} named {}", metric, metricNames.getLongName()); + LOG.error("Unable to save taskId mapping for metric {} named {}", metric, + metricNames.getLongName()); } } } - private static void saveMetricTaskIdMapping(Integer taskId, MetricNames names, T metric, Map void saveMetricTaskIdMapping(Integer taskId, + MetricNames names, T metric, Map> taskIdMetrics) { - Map metrics = taskIdMetrics.computeIfAbsent(taskId, (tid) -> new ConcurrentHashMap<>()); + Map metrics = taskIdMetrics.computeIfAbsent(taskId, + (tid) -> new ConcurrentHashMap<>()); metrics.put(names.getShortName(), metric); } private Gauge registerGauge(MetricNames metricNames, Gauge gauge, int taskId, String componentId, String streamId) { - TaskMetricDimensions taskMetricDimensions = new TaskMetricDimensions(taskId, componentId, streamId, this); - TaskMetricRepo repo = taskMetrics.computeIfAbsent(taskMetricDimensions, (k) -> new TaskMetricRepo()); + TaskMetricDimensions taskMetricDimensions = new TaskMetricDimensions(taskId, componentId, + streamId, this); + TaskMetricRepo repo = taskMetrics.computeIfAbsent(taskMetricDimensions, + (k) -> new TaskMetricRepo()); repo.addGauge(metricNames.getShortName(), gauge); gauge = registry.register(metricNames.getLongName(), gauge); return gauge; } - private Meter registerMeter(MetricNames metricNames, Meter meter, int taskId, String componentId, String streamId) { - TaskMetricDimensions taskMetricDimensions = new TaskMetricDimensions(taskId, componentId, streamId, this); - TaskMetricRepo repo = taskMetrics.computeIfAbsent(taskMetricDimensions, (k) -> new TaskMetricRepo()); + private Meter registerMeter(MetricNames metricNames, Meter meter, int taskId, + String componentId, String streamId) { + TaskMetricDimensions taskMetricDimensions = new TaskMetricDimensions(taskId, componentId, + streamId, this); + TaskMetricRepo repo = taskMetrics.computeIfAbsent(taskMetricDimensions, + (k) -> new TaskMetricRepo()); repo.addMeter(metricNames.getShortName(), meter); meter = registry.register(metricNames.getLongName(), meter); return meter; @@ -235,16 +280,21 @@ private Meter registerMeter(MetricNames metricNames, Meter meter, int taskId, St private Counter registerCounter(MetricNames metricNames, Counter counter, int taskId, String componentId, String streamId) { - TaskMetricDimensions taskMetricDimensions = new TaskMetricDimensions(taskId, componentId, streamId, this); - TaskMetricRepo repo = taskMetrics.computeIfAbsent(taskMetricDimensions, (k) -> new TaskMetricRepo()); + TaskMetricDimensions taskMetricDimensions = new TaskMetricDimensions(taskId, componentId, + streamId, this); + TaskMetricRepo repo = taskMetrics.computeIfAbsent(taskMetricDimensions, + (k) -> new TaskMetricRepo()); repo.addCounter(metricNames.getShortName(), counter); counter = registry.register(metricNames.getLongName(), counter); return counter; } - private Timer registerTimer(MetricNames metricNames, Timer timer, int taskId, String componentId, String streamId) { - TaskMetricDimensions taskMetricDimensions = new TaskMetricDimensions(taskId, componentId, streamId, this); - TaskMetricRepo repo = taskMetrics.computeIfAbsent(taskMetricDimensions, (k) -> new TaskMetricRepo()); + private Timer registerTimer(MetricNames metricNames, Timer timer, int taskId, + String componentId, String streamId) { + TaskMetricDimensions taskMetricDimensions = new TaskMetricDimensions(taskId, componentId, + streamId, this); + TaskMetricRepo repo = taskMetrics.computeIfAbsent(taskMetricDimensions, + (k) -> new TaskMetricRepo()); repo.addTimer(metricNames.getShortName(), timer); timer = registry.register(metricNames.getLongName(), timer); return timer; @@ -252,8 +302,10 @@ private Timer registerTimer(MetricNames metricNames, Timer timer, int taskId, St private Histogram registerHistogram(MetricNames metricNames, Histogram histogram, int taskId, String componentId, String streamId) { - TaskMetricDimensions taskMetricDimensions = new TaskMetricDimensions(taskId, componentId, streamId, this); - TaskMetricRepo repo = taskMetrics.computeIfAbsent(taskMetricDimensions, (k) -> new TaskMetricRepo()); + TaskMetricDimensions taskMetricDimensions = new TaskMetricDimensions(taskId, componentId, + streamId, this); + TaskMetricRepo repo = taskMetrics.computeIfAbsent(taskMetricDimensions, + (k) -> new TaskMetricRepo()); repo.addHistogram(metricNames.getShortName(), histogram); histogram = registry.register(metricNames.getLongName(), histogram); return histogram; @@ -267,7 +319,8 @@ public void deregister(Set toRemove) { registry.removeMatching(metricFilter); } - private Map getMetricNameMap(int taskId, Map> taskIdMetrics) { + private Map getMetricNameMap(int taskId, Map> taskIdMetrics) { Map ret = new HashMap<>(); Map taskMetrics = taskIdMetrics.getOrDefault(taskId, Collections.emptyMap()); ret.putAll(taskMetrics); @@ -298,7 +351,8 @@ public void start(Map topoConf, int port) { try { hostName = dotToUnderScore(Utils.localHostname()); } catch (UnknownHostException e) { - LOG.warn("Unable to determine hostname while starting the metrics system. Hostname will be reported" + LOG.warn("Unable to determine hostname while starting the metrics system. Hostname " + + "will be reported" + " as 'localhost'."); } @@ -313,7 +367,8 @@ public void start(Map topoConf, int port) { RATE_COUNTER_UPDATE_INTERVAL_SECONDS, new RateCounterUpdater()); LOG.info("Starting metrics reporters..."); - List> reporterList = (List>) topoConf.get(Config.TOPOLOGY_METRICS_REPORTERS); + List> reporterList = (List>) topoConf + .get(Config.TOPOLOGY_METRICS_REPORTERS); if (reporterList != null && reporterList.size() > 0) { for (Map reporterConfig : reporterList) { @@ -371,7 +426,8 @@ Integer getPort() { return port; } - private MetricNames workerMetricName(String name, String stormId, String componentId, String streamId, + private MetricNames workerMetricName(String name, String stormId, String componentId, + String streamId, Integer taskId, Integer workerPort) { StringBuilder sb = new StringBuilder(WORKER_METRIC_PREFIX); sb.append(stormId); @@ -392,7 +448,8 @@ private MetricNames workerMetricName(String name, String stormId, String compone return names; } - private MetricNames workerMetricName(String name, String stormId, String componentId, Integer taskId, Integer workerPort) { + private MetricNames workerMetricName(String name, String stormId, String componentId, + Integer taskId, Integer workerPort) { StringBuilder sb = new StringBuilder(WORKER_METRIC_PREFIX); sb.append(stormId); sb.append("."); @@ -443,6 +500,7 @@ private static class MetricNames { /** * Returns the full metric name to be used for registering with the metrics registry. + * * @return The full metric name. */ String getLongName() { @@ -451,6 +509,7 @@ String getLongName() { /** * Returns the short metric name (without dimensions). + * * @return The short metric name. */ String getShortName() { @@ -473,7 +532,8 @@ private static class RemoveMetricFilter implements MetricFilter { RemoveMetricFilter(Set toRemove) { this.metrics.addAll(toRemove); for (Metric metric : toRemove) { - // RateCounters are gauges, but also have internal Counters that should also be removed + // RateCounters are gauges, but also have internal Counters that should also be + // removed if (metric instanceof RateCounter) { RateCounter rateCounter = (RateCounter) metric; this.metrics.add(rateCounter.getCounter()); diff --git a/storm-client/src/jvm/org/apache/storm/metrics2/TaskMetricDimensions.java b/storm-client/src/jvm/org/apache/storm/metrics2/TaskMetricDimensions.java index 05d824493d6..cc63cc9ca5c 100644 --- a/storm-client/src/jvm/org/apache/storm/metrics2/TaskMetricDimensions.java +++ b/storm-client/src/jvm/org/apache/storm/metrics2/TaskMetricDimensions.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -24,7 +30,8 @@ public class TaskMetricDimensions { private String streamId; private Map dimensions = new HashMap<>(); - public TaskMetricDimensions(int taskId, String componentId, String streamId, StormMetricRegistry metricRegistry) { + public TaskMetricDimensions(int taskId, String componentId, String streamId, + StormMetricRegistry metricRegistry) { this.taskId = taskId; dimensions.put("taskid", Integer.toString(this.taskId)); diff --git a/storm-client/src/jvm/org/apache/storm/metrics2/TaskMetricRepo.java b/storm-client/src/jvm/org/apache/storm/metrics2/TaskMetricRepo.java index 00f9c7fff6a..b80bcf1f83f 100644 --- a/storm-client/src/jvm/org/apache/storm/metrics2/TaskMetricRepo.java +++ b/storm-client/src/jvm/org/apache/storm/metrics2/TaskMetricRepo.java @@ -1,12 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -90,13 +95,16 @@ public void report(ScheduledReporter reporter, MetricFilter filter) { filteredTimers.put(entry.getKey(), entry.getValue()); } } - reporter.report(filteredGauges, filteredCounters, filteredHistograms, filteredMeters, filteredTimers); + reporter.report(filteredGauges, filteredCounters, filteredHistograms, filteredMeters, + filteredTimers); } void degister(MetricFilter metricFilter) { gauges.entrySet().removeIf(entry -> metricFilter.matches(entry.getKey(), entry.getValue())); - counters.entrySet().removeIf(entry -> metricFilter.matches(entry.getKey(), entry.getValue())); - histograms.entrySet().removeIf(entry -> metricFilter.matches(entry.getKey(), entry.getValue())); + counters.entrySet().removeIf(entry -> metricFilter.matches(entry.getKey(), entry + .getValue())); + histograms.entrySet().removeIf(entry -> metricFilter.matches(entry.getKey(), entry + .getValue())); meters.entrySet().removeIf(entry -> metricFilter.matches(entry.getKey(), entry.getValue())); timers.entrySet().removeIf(entry -> metricFilter.matches(entry.getKey(), entry.getValue())); } diff --git a/storm-client/src/jvm/org/apache/storm/metrics2/TaskMetrics.java b/storm-client/src/jvm/org/apache/storm/metrics2/TaskMetrics.java index 059109b0bd8..e1d3c524eab 100644 --- a/storm-client/src/jvm/org/apache/storm/metrics2/TaskMetrics.java +++ b/storm-client/src/jvm/org/apache/storm/metrics2/TaskMetrics.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -17,7 +23,6 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.function.Supplier; - import org.apache.storm.task.WorkerTopologyContext; import org.apache.storm.utils.ConfigUtils; import org.apache.storm.utils.Utils; @@ -50,7 +55,6 @@ public class TaskMetrics { private final int samplingRate; private final boolean ewmaEnable; - public TaskMetrics(WorkerTopologyContext context, String componentId, Integer taskid, StormMetricRegistry metricRegistry, Map topoConf) { this.metricRegistry = metricRegistry; @@ -68,7 +72,8 @@ public TaskMetrics(WorkerTopologyContext context, String componentId, Integer ta public void setCapacity(double capacity) { String metricName = METRIC_NAME_CAPACITY; // capacity is over all streams, will report using the default streamId - RollingAverageGauge gauge = this.getRollingAverageGauge(metricName, Utils.DEFAULT_STREAM_ID); + RollingAverageGauge gauge = this.getRollingAverageGauge(metricName, + Utils.DEFAULT_STREAM_ID); gauge.addValue(capacity); } @@ -83,7 +88,8 @@ public void spoutAckedTuple(String streamId, long latencyMs) { if (this.ewmaEnable) { metricName = METRIC_NAME_COMPLETE_JITTER + "-" + streamId; - EwmaGauge ewmaGauge = this.getExponentialWeightedMovingAverageGauge(metricName, streamId); + EwmaGauge ewmaGauge = this.getExponentialWeightedMovingAverageGauge(metricName, + streamId); ewmaGauge.addValue(latencyMs); } } @@ -100,7 +106,8 @@ public void boltAckedTuple(String sourceComponentId, String sourceStreamId, long if (this.ewmaEnable) { metricName = METRIC_NAME_PROCESS_JITTER + "-" + key; - EwmaGauge ewmaGauge = this.getExponentialWeightedMovingAverageGauge(metricName, sourceStreamId); + EwmaGauge ewmaGauge = this.getExponentialWeightedMovingAverageGauge(metricName, + sourceStreamId); ewmaGauge.addValue(latencyMs); } } @@ -145,7 +152,8 @@ public void boltExecuteTuple(String sourceComponentId, String sourceStreamId, lo if (this.ewmaEnable) { metricName = METRIC_NAME_EXECUTE_JITTER + "-" + key; - EwmaGauge ewmaGauge = this.getExponentialWeightedMovingAverageGauge(metricName, sourceStreamId); + EwmaGauge ewmaGauge = this.getExponentialWeightedMovingAverageGauge(metricName, + sourceStreamId); ewmaGauge.addValue(latencyMs); } } @@ -166,7 +174,8 @@ private RateCounter getRateCounter(String metricName, String streamId) { } private RollingAverageGauge getRollingAverageGauge(String metricName, String streamId) { - return getOrCreateGauge(metricName, streamId, RollingAverageGauge.class, this.rollingAverageGaugeFactory); + return getOrCreateGauge(metricName, streamId, RollingAverageGauge.class, + this.rollingAverageGaugeFactory); } private EwmaGauge getExponentialWeightedMovingAverageGauge(String metricName, String streamId) { diff --git a/storm-client/src/jvm/org/apache/storm/metrics2/WorkerMetricRegistrant.java b/storm-client/src/jvm/org/apache/storm/metrics2/WorkerMetricRegistrant.java index ee0f3c51831..c995f83e89d 100644 --- a/storm-client/src/jvm/org/apache/storm/metrics2/WorkerMetricRegistrant.java +++ b/storm-client/src/jvm/org/apache/storm/metrics2/WorkerMetricRegistrant.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/metrics2/cgroup/CGroupCpu.java b/storm-client/src/jvm/org/apache/storm/metrics2/cgroup/CGroupCpu.java index 4630a937258..e5ac83553ab 100644 --- a/storm-client/src/jvm/org/apache/storm/metrics2/cgroup/CGroupCpu.java +++ b/storm-client/src/jvm/org/apache/storm/metrics2/cgroup/CGroupCpu.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/metrics2/cgroup/CGroupCpuGuarantee.java b/storm-client/src/jvm/org/apache/storm/metrics2/cgroup/CGroupCpuGuarantee.java index d6a22269a0e..5161eb4375c 100644 --- a/storm-client/src/jvm/org/apache/storm/metrics2/cgroup/CGroupCpuGuarantee.java +++ b/storm-client/src/jvm/org/apache/storm/metrics2/cgroup/CGroupCpuGuarantee.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -23,7 +29,8 @@ /** * Report the guaranteed number of cpu percentage this worker has requested. * It gets the result from cpu.shares. - * Use this when org.apache.storm.container.cgroup.CgroupManager is used as the storm.resource.isolation.plugin. + * Use this when org.apache.storm.container.cgroup.CgroupManager is used as the + * storm.resource.isolation.plugin. */ public class CGroupCpuGuarantee extends CGroupMetricsBase implements WorkerMetricRegistrant { private long shares = -1L; diff --git a/storm-client/src/jvm/org/apache/storm/metrics2/cgroup/CGroupCpuGuaranteeByCfsQuota.java b/storm-client/src/jvm/org/apache/storm/metrics2/cgroup/CGroupCpuGuaranteeByCfsQuota.java index d68bac4378c..4d840aed9f3 100644 --- a/storm-client/src/jvm/org/apache/storm/metrics2/cgroup/CGroupCpuGuaranteeByCfsQuota.java +++ b/storm-client/src/jvm/org/apache/storm/metrics2/cgroup/CGroupCpuGuaranteeByCfsQuota.java @@ -1,12 +1,17 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -23,7 +28,8 @@ /** * Report the percentage of the cpu guaranteed for the worker. * It gets the result from cpu.cfs_period_us and cpu.cfs_quota_us. - * Use this when org.apache.storm.container.docker.DockerManager or org.apache.storm.container.oci.RuncLibContainerManager + * Use this when org.apache.storm.container.docker.DockerManager or + * org.apache.storm.container.oci.RuncLibContainerManager * is used as the storm.resource.isolation.plugin. */ public class CGroupCpuGuaranteeByCfsQuota extends CGroupMetricsBase implements WorkerMetricRegistrant { @@ -44,7 +50,8 @@ public Long getValue() { try { long cpuCfsQuotaUs = cpu.getCpuCfsQuotaUs(); if (cpuCfsQuotaUs == -1) { - //cpu.cfs_quota_us = -1 indicates that the cgroup does not adhere to any CPU time restrictions. + // cpu.cfs_quota_us = -1 indicates that the cgroup does not adhere + // to any CPU time restrictions. guarantee = -1L; } else { long cpuCfsPeriodUs = cpu.getCpuCfsPeriodUs(); diff --git a/storm-client/src/jvm/org/apache/storm/metrics2/cgroup/CGroupCpuStat.java b/storm-client/src/jvm/org/apache/storm/metrics2/cgroup/CGroupCpuStat.java index a26e3e0852c..090a80f2c5a 100644 --- a/storm-client/src/jvm/org/apache/storm/metrics2/cgroup/CGroupCpuStat.java +++ b/storm-client/src/jvm/org/apache/storm/metrics2/cgroup/CGroupCpuStat.java @@ -1,19 +1,24 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.metrics2.cgroup; import com.codahale.metrics.Gauge; - import java.io.IOException; import java.util.Map; import org.apache.storm.container.cgroup.SubSystemType; @@ -54,17 +59,18 @@ public Long getValue() { } }); - topologyContext.registerGauge("CGroupCpuStat.nr.throttled-percentage", new Gauge() { - @Override + topologyContext.registerGauge("CGroupCpuStat.nr.throttled-percentage", + new Gauge() { + @Override public Long getValue() { - try { - CpuCore.Stat stat = ((CpuCore) core).getCpuStat(); - return (long) (stat.nrThrottled * 100.0 / stat.nrPeriods); - } catch (IOException e) { - throw new RuntimeException(e); + try { + CpuCore.Stat stat = ((CpuCore) core).getCpuStat(); + return (long) (stat.nrThrottled * 100.0 / stat.nrPeriods); + } catch (IOException e) { + throw new RuntimeException(e); + } } - } - }); + }); topologyContext.registerGauge("CGroupCpuStat.throttled.time-ms", new Gauge() { @Override diff --git a/storm-client/src/jvm/org/apache/storm/metrics2/cgroup/CGroupMemoryLimit.java b/storm-client/src/jvm/org/apache/storm/metrics2/cgroup/CGroupMemoryLimit.java index fa6d203120f..3e0abb3f95b 100644 --- a/storm-client/src/jvm/org/apache/storm/metrics2/cgroup/CGroupMemoryLimit.java +++ b/storm-client/src/jvm/org/apache/storm/metrics2/cgroup/CGroupMemoryLimit.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -32,7 +38,7 @@ public class CGroupMemoryLimit extends CGroupMetricsBase implements WorkerMetric public CGroupMemoryLimit(Map conf) { super(conf, SubSystemType.memory); - //In some cases we might be limiting memory in the supervisor and not in the cgroups + // In some cases we might be limiting memory in the supervisor and not in the cgroups long limit = -1; try { limit = Long.valueOf(System.getProperty("worker.memory_limit_mb", "-1")); diff --git a/storm-client/src/jvm/org/apache/storm/metrics2/cgroup/CGroupMemoryUsage.java b/storm-client/src/jvm/org/apache/storm/metrics2/cgroup/CGroupMemoryUsage.java index 32dade10ed9..e86b324b762 100644 --- a/storm-client/src/jvm/org/apache/storm/metrics2/cgroup/CGroupMemoryUsage.java +++ b/storm-client/src/jvm/org/apache/storm/metrics2/cgroup/CGroupMemoryUsage.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/metrics2/cgroup/CGroupMetricsBase.java b/storm-client/src/jvm/org/apache/storm/metrics2/cgroup/CGroupMetricsBase.java index 4ae9efa3fc4..6b576504fd2 100644 --- a/storm-client/src/jvm/org/apache/storm/metrics2/cgroup/CGroupMetricsBase.java +++ b/storm-client/src/jvm/org/apache/storm/metrics2/cgroup/CGroupMetricsBase.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -38,7 +44,8 @@ public CGroupMetricsBase(Map conf, SubSystemType type) { enabled = false; CgroupCenter center = CgroupCenter.getInstance(); if (center == null) { - LOG.warn("{} is disabled. cgroups do not appear to be enabled on this system", simpleName); + LOG.warn("{} is disabled. cgroups do not appear to be enabled on this system", + simpleName); return; } if (!center.isSubSystemEnabled(type)) { @@ -46,54 +53,59 @@ public CGroupMetricsBase(Map conf, SubSystemType type) { return; } - //Check to see if the CGroup is mounted at all + // Check to see if the CGroup is mounted at all if (null == center.getHierarchyWithSubSystem(type)) { LOG.warn("{} is disabled. {} is not a mounted subsystem", simpleName, type); return; } - //Good so far, check if we are in a CGroup + // Good so far, check if we are in a CGroup File cgroupFile = new File("/proc/self/cgroup"); if (!cgroupFile.exists()) { - LOG.warn("{} is disabled we do not appear to be a part of a CGroup", getClass().getSimpleName()); + LOG.warn("{} is disabled we do not appear to be a part of a CGroup", getClass() + .getSimpleName()); return; } String cgroupPath; try (BufferedReader reader = new BufferedReader(new FileReader(cgroupFile))) { - //There can be more then one line if cgroups are mounted in more then one place, but we assume the first is good enough + // There can be more then one line if cgroups are mounted in more then one place, but we + // assume the first is good enough String line = reader.readLine(); - //hierarchy-ID:controller-list:cgroup-path + // hierarchy-ID:controller-list:cgroup-path String[] parts = line.split(":"); - //parts[0] == 0 for CGroup V2, else maps to hierarchy in /proc/cgroups - //parts[1] is empty for CGroups V2 else what is mapped that we are looking for + // parts[0] == 0 for CGroup V2, else maps to hierarchy in /proc/cgroups + // parts[1] is empty for CGroups V2 else what is mapped that we are looking for cgroupPath = parts[2]; } catch (Exception e) { LOG.warn("{} is disabled error trying to read or parse {}", simpleName, cgroupFile); return; } - //Storm on Rhel6 and Rhel7 use different cgroup settings. - //On Rhel6, the cgroup of the worker is under + // Storm on Rhel6 and Rhel7 use different cgroup settings. + // On Rhel6, the cgroup of the worker is under // "Config.STORM_CGROUP_HIERARCHY_DIR/DaemonConfig.STORM_SUPERVISOR_CGROUP_ROOTDIR/" - //On Rhel7, the cgroup of the worker is under + // On Rhel7, the cgroup of the worker is under // "Config.STORM_OCI_CGROUP_ROOT//DaemonConfig.STORM_OCI_CGROUP_PARENT/" // This block of code is a workaround for the CGroupMetrics to work on both system String hierarchyDir = (String) conf.get(Config.STORM_CGROUP_HIERARCHY_DIR); if (StringUtils.isEmpty(hierarchyDir) || !new File(hierarchyDir, cgroupPath).exists()) { - LOG.info("{} is not set or does not exist. checking {}", Config.STORM_CGROUP_HIERARCHY_DIR, + LOG.info("{} is not set or does not exist. checking {}", + Config.STORM_CGROUP_HIERARCHY_DIR, Config.STORM_OCI_CGROUP_ROOT); String ociCgroupRoot = (String) conf.get(Config.STORM_OCI_CGROUP_ROOT); hierarchyDir = ociCgroupRoot + File.separator + type; - if (StringUtils.isEmpty(ociCgroupRoot) || !new File(hierarchyDir, cgroupPath).exists()) { + if (StringUtils.isEmpty(ociCgroupRoot) || !new File(hierarchyDir, cgroupPath) + .exists()) { LOG.info("{} is not set or does not exist", Config.STORM_OCI_CGROUP_ROOT); LOG.warn("{} is disabled", simpleName); return; } } - core = CgroupCoreFactory.getInstance(type, new File(hierarchyDir, cgroupPath).getAbsolutePath()); + core = CgroupCoreFactory.getInstance(type, new File(hierarchyDir, cgroupPath) + .getAbsolutePath()); enabled = true; LOG.info("Metric {} is ENABLED and directory {} exists...", simpleName, hierarchyDir); } diff --git a/storm-client/src/jvm/org/apache/storm/metrics2/filters/RegexFilter.java b/storm-client/src/jvm/org/apache/storm/metrics2/filters/RegexFilter.java index ee8ccd0e780..949b22fa084 100644 --- a/storm-client/src/jvm/org/apache/storm/metrics2/filters/RegexFilter.java +++ b/storm-client/src/jvm/org/apache/storm/metrics2/filters/RegexFilter.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -21,7 +27,6 @@ public class RegexFilter implements StormMetricsFilter { private Pattern pattern; - @Override public void prepare(Map config) { String expression = (String) config.get("expression"); diff --git a/storm-client/src/jvm/org/apache/storm/metrics2/filters/StormMetricsFilter.java b/storm-client/src/jvm/org/apache/storm/metrics2/filters/StormMetricsFilter.java index 64612557009..1a2e4ce4274 100644 --- a/storm-client/src/jvm/org/apache/storm/metrics2/filters/StormMetricsFilter.java +++ b/storm-client/src/jvm/org/apache/storm/metrics2/filters/StormMetricsFilter.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -20,7 +26,8 @@ public interface StormMetricsFilter extends MetricFilter { /** * Called after the filter is instantiated. * - * @param config A map of the properties from the 'filter' section of the reporter configuration. + * @param config A map of the properties from the 'filter' section of the reporter + * configuration. */ void prepare(Map config); diff --git a/storm-client/src/jvm/org/apache/storm/metrics2/reporters/ConsoleStormReporter.java b/storm-client/src/jvm/org/apache/storm/metrics2/reporters/ConsoleStormReporter.java index 7786586a22a..67489db7eb1 100644 --- a/storm-client/src/jvm/org/apache/storm/metrics2/reporters/ConsoleStormReporter.java +++ b/storm-client/src/jvm/org/apache/storm/metrics2/reporters/ConsoleStormReporter.java @@ -1,12 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -15,7 +20,6 @@ import com.codahale.metrics.ConsoleReporter; import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.ScheduledReporter; - import java.io.IOException; import java.util.Locale; import java.util.Map; @@ -31,7 +35,8 @@ public class ConsoleStormReporter extends ScheduledStormReporter implements Dime private static final Logger LOG = LoggerFactory.getLogger(ConsoleStormReporter.class); @Override - public void prepare(MetricRegistry registry, Map topoConf, Map reporterConf) { + public void prepare(MetricRegistry registry, Map topoConf, Map reporterConf) { init(registry, null, reporterConf); } @@ -41,7 +46,8 @@ public void prepare(MetricRegistryProvider metricRegistryProvider, Map reporterConf) { + private void init(MetricRegistry registry, MetricRegistryProvider metricRegistryProvider, + Map reporterConf) { LOG.debug("Preparing ConsoleReporter"); ConsoleReporter.Builder builder = ConsoleReporter.forRegistry(registry); @@ -66,10 +72,10 @@ private void init(MetricRegistry registry, MetricRegistryProvider metricRegistry builder.filter(filter); } - //defaults to 10 + // defaults to 10 reportingPeriod = getReportPeriod(reporterConf); - //defaults to seconds + // defaults to seconds reportingPeriodUnit = getReportPeriodUnit(reporterConf); ScheduledReporter consoleReporter = builder.build(); @@ -77,7 +83,8 @@ private void init(MetricRegistry registry, MetricRegistryProvider metricRegistry boolean reportDimensions = isReportDimensionsEnabled(reporterConf); if (reportDimensions) { if (metricRegistryProvider == null) { - throw new RuntimeException("MetricRegistryProvider is required to enable reporting dimensions"); + throw new RuntimeException("MetricRegistryProvider is required to enable " + + "reporting dimensions"); } if (rateUnit == null) { rateUnit = TimeUnit.SECONDS; @@ -85,7 +92,8 @@ private void init(MetricRegistry registry, MetricRegistryProvider metricRegistry if (durationUnit == null) { durationUnit = TimeUnit.MILLISECONDS; } - DimensionalReporter dimensionalReporter = new DimensionalReporter(metricRegistryProvider, consoleReporter, this, + DimensionalReporter dimensionalReporter = + new DimensionalReporter(metricRegistryProvider, consoleReporter, this, "ConsoleDimensionalReporter", filter, rateUnit, durationUnit, null, true); reporter = dimensionalReporter; diff --git a/storm-client/src/jvm/org/apache/storm/metrics2/reporters/CsvStormReporter.java b/storm-client/src/jvm/org/apache/storm/metrics2/reporters/CsvStormReporter.java index a64ba127595..fb9941851d4 100644 --- a/storm-client/src/jvm/org/apache/storm/metrics2/reporters/CsvStormReporter.java +++ b/storm-client/src/jvm/org/apache/storm/metrics2/reporters/CsvStormReporter.java @@ -1,12 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -14,7 +19,6 @@ import com.codahale.metrics.CsvReporter; import com.codahale.metrics.MetricRegistry; - import java.io.File; import java.io.IOException; import java.util.Locale; @@ -35,7 +39,8 @@ private static File getCsvLogDir(Map stormConf, Map reporterConf) { String csvMetricsLogDirectory = ObjectReader.getString(reporterConf.get(CSV_LOG_DIR), null); if (csvMetricsLogDirectory == null) { csvMetricsLogDirectory = ConfigUtils.absoluteStormLocalDir(stormConf); - csvMetricsLogDirectory = csvMetricsLogDirectory + ConfigUtils.FILE_SEPARATOR + "csvmetrics"; + csvMetricsLogDirectory = csvMetricsLogDirectory + ConfigUtils.FILE_SEPARATOR + + "csvmetrics"; } File csvMetricsDir = new File(csvMetricsLogDirectory); validateCreateOutputDir(csvMetricsDir); @@ -55,7 +60,8 @@ private static void validateCreateOutputDir(File dir) { } @Override - public void prepare(MetricRegistry metricsRegistry, Map topoConf, Map reporterConf) { + public void prepare(MetricRegistry metricsRegistry, Map topoConf, Map reporterConf) { LOG.debug("Preparing..."); CsvReporter.Builder builder = CsvReporter.forRegistry(metricsRegistry); @@ -79,10 +85,10 @@ public void prepare(MetricRegistry metricsRegistry, Map topoConf builder.filter(filter); } - //defaults to 10 + // defaults to 10 reportingPeriod = getReportPeriod(reporterConf); - //defaults to seconds + // defaults to seconds reportingPeriodUnit = getReportPeriodUnit(reporterConf); File csvMetricsDir = getCsvLogDir(topoConf, reporterConf); diff --git a/storm-client/src/jvm/org/apache/storm/metrics2/reporters/GraphiteStormReporter.java b/storm-client/src/jvm/org/apache/storm/metrics2/reporters/GraphiteStormReporter.java index 647832ee2ce..1aaed243f87 100644 --- a/storm-client/src/jvm/org/apache/storm/metrics2/reporters/GraphiteStormReporter.java +++ b/storm-client/src/jvm/org/apache/storm/metrics2/reporters/GraphiteStormReporter.java @@ -1,12 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -17,7 +22,6 @@ import com.codahale.metrics.graphite.GraphiteReporter; import com.codahale.metrics.graphite.GraphiteSender; import com.codahale.metrics.graphite.GraphiteUDP; - import java.io.IOException; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -51,7 +55,8 @@ private static String getMetricsTargetTransport(Map reporterConf } @Override - public void prepare(MetricRegistry metricsRegistry, Map topoConf, Map reporterConf) { + public void prepare(MetricRegistry metricsRegistry, Map topoConf, Map reporterConf) { LOG.debug("Preparing..."); GraphiteReporter.Builder builder = GraphiteReporter.forRegistry(metricsRegistry); @@ -74,10 +79,10 @@ public void prepare(MetricRegistry metricsRegistry, Map topoConf builder.prefixedWith(prefix); } - //defaults to 10 + // defaults to 10 reportingPeriod = getReportPeriod(reporterConf); - //defaults to seconds + // defaults to seconds reportingPeriodUnit = getReportPeriodUnit(reporterConf); // Not exposed: diff --git a/storm-client/src/jvm/org/apache/storm/metrics2/reporters/JmxStormReporter.java b/storm-client/src/jvm/org/apache/storm/metrics2/reporters/JmxStormReporter.java index cd9da98360f..04de622cf10 100644 --- a/storm-client/src/jvm/org/apache/storm/metrics2/reporters/JmxStormReporter.java +++ b/storm-client/src/jvm/org/apache/storm/metrics2/reporters/JmxStormReporter.java @@ -1,12 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -14,7 +19,6 @@ import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.jmx.JmxReporter; - import java.io.IOException; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -34,7 +38,8 @@ public static String getMetricsJmxDomain(Map reporterConf) { } @Override - public void prepare(MetricRegistry metricsRegistry, Map topoConf, Map reporterConf) { + public void prepare(MetricRegistry metricsRegistry, Map topoConf, Map reporterConf) { LOG.info("Preparing..."); JmxReporter.Builder builder = JmxReporter.forRegistry(metricsRegistry); @@ -72,7 +77,8 @@ public void start() { LOG.debug("Starting..."); reporter.start(); } else { - throw new IllegalStateException("Attempt to start without preparing " + getClass().getSimpleName()); + throw new IllegalStateException("Attempt to start without preparing " + getClass() + .getSimpleName()); } } @@ -82,7 +88,8 @@ public void stop() { LOG.debug("Stopping..."); reporter.stop(); } else { - throw new IllegalStateException("Attempt to stop without preparing " + getClass().getSimpleName()); + throw new IllegalStateException("Attempt to stop without preparing " + getClass() + .getSimpleName()); } } diff --git a/storm-client/src/jvm/org/apache/storm/metrics2/reporters/ScheduledStormReporter.java b/storm-client/src/jvm/org/apache/storm/metrics2/reporters/ScheduledStormReporter.java index 97220d0e370..237e4c0530c 100644 --- a/storm-client/src/jvm/org/apache/storm/metrics2/reporters/ScheduledStormReporter.java +++ b/storm-client/src/jvm/org/apache/storm/metrics2/reporters/ScheduledStormReporter.java @@ -1,19 +1,24 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.metrics2.reporters; import com.codahale.metrics.ScheduledReporter; - import java.io.IOException; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -62,7 +67,8 @@ public void start() { LOG.debug("Starting..."); reporter.start(reportingPeriod, reportingPeriodUnit); } else { - throw new IllegalStateException("Attempt to start without preparing " + getClass().getSimpleName()); + throw new IllegalStateException("Attempt to start without preparing " + getClass() + .getSimpleName()); } } @@ -72,7 +78,8 @@ public void stop() { LOG.debug("Stopping..."); reporter.stop(); } else { - throw new IllegalStateException("Attempt to stop without preparing " + getClass().getSimpleName()); + throw new IllegalStateException("Attempt to stop without preparing " + getClass() + .getSimpleName()); } } diff --git a/storm-client/src/jvm/org/apache/storm/metrics2/reporters/StormReporter.java b/storm-client/src/jvm/org/apache/storm/metrics2/reporters/StormReporter.java index 7791bafb7a3..dd5603f40fa 100644 --- a/storm-client/src/jvm/org/apache/storm/metrics2/reporters/StormReporter.java +++ b/storm-client/src/jvm/org/apache/storm/metrics2/reporters/StormReporter.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -17,16 +23,17 @@ import java.util.Map; import org.apache.storm.metrics2.MetricRegistryProvider; - public interface StormReporter extends Reporter { String REPORT_PERIOD = "report.period"; String REPORT_PERIOD_UNITS = "report.period.units"; String REPORT_DIMENSIONS_ENABLED = "report.dimensions.enabled"; @Deprecated - void prepare(MetricRegistry metricsRegistry, Map topoConf, Map reporterConf); + void prepare(MetricRegistry metricsRegistry, Map topoConf, Map reporterConf); - default void prepare(MetricRegistryProvider metricRegistryProvider, Map topoConf, + default void prepare(MetricRegistryProvider metricRegistryProvider, Map topoConf, Map reporterConf) { prepare(metricRegistryProvider.getRegistry(), topoConf, reporterConf); } diff --git a/storm-client/src/jvm/org/apache/storm/multilang/BoltMsg.java b/storm-client/src/jvm/org/apache/storm/multilang/BoltMsg.java index 956780fd90c..171f769db92 100644 --- a/storm-client/src/jvm/org/apache/storm/multilang/BoltMsg.java +++ b/storm-client/src/jvm/org/apache/storm/multilang/BoltMsg.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -15,11 +21,14 @@ import java.util.List; /** - * BoltMsg is an object that represents the data sent from a shell component to a bolt process that implements a multi-language protocol. It + * BoltMsg is an object that represents the data sent from a shell component to a bolt process that + * implements a multi-language protocol. It * is the union of all data types that a bolt can receive from Storm. * - *

    BoltMsgs are objects sent to the ISerializer interface, for serialization according to the wire protocol - * implemented by the serializer. The BoltMsg class allows for a decoupling between the serialized representation of the + *

    BoltMsgs are objects sent to the ISerializer interface, for serialization according to the + * wire protocol + * implemented by the serializer. The BoltMsg class allows for a decoupling between the serialized + * representation of the * data and the data itself. */ public class BoltMsg { diff --git a/storm-client/src/jvm/org/apache/storm/multilang/ISerializer.java b/storm-client/src/jvm/org/apache/storm/multilang/ISerializer.java index bc7c3acebc2..f02830ab9fe 100644 --- a/storm-client/src/jvm/org/apache/storm/multilang/ISerializer.java +++ b/storm-client/src/jvm/org/apache/storm/multilang/ISerializer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -21,7 +27,8 @@ import org.apache.storm.task.TopologyContext; /** - * The ISerializer interface describes the methods that an object should implement to provide serialization and de-serialization + * The ISerializer interface describes the methods that an object should implement to provide + * serialization and de-serialization * capabilities to non-JVM language components. */ public interface ISerializer extends Serializable { diff --git a/storm-client/src/jvm/org/apache/storm/multilang/JsonSerializer.java b/storm-client/src/jvm/org/apache/storm/multilang/JsonSerializer.java index be023a9b8ae..409c12cbe3a 100644 --- a/storm-client/src/jvm/org/apache/storm/multilang/JsonSerializer.java +++ b/storm-client/src/jvm/org/apache/storm/multilang/JsonSerializer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -34,7 +40,7 @@ */ public class JsonSerializer implements ISerializer { public static final String DEFAULT_CHARSET = "UTF-8"; - //ANY CHANGE TO THIS CODE MUST BE SERIALIZABLE COMPATIBLE OR THERE WILL BE PROBLEMS + // ANY CHANGE TO THIS CODE MUST BE SERIALIZABLE COMPATIBLE OR THERE WILL BE PROBLEMS private static final long serialVersionUID = 2548814660410474022L; private transient BufferedWriter processIn; private transient BufferedReader processOut; @@ -43,7 +49,8 @@ public class JsonSerializer implements ISerializer { public void initialize(OutputStream processIn, InputStream processOut) { try { this.processIn = new BufferedWriter(new OutputStreamWriter(processIn, DEFAULT_CHARSET)); - this.processOut = new BufferedReader(new InputStreamReader(processOut, DEFAULT_CHARSET)); + this.processOut = new BufferedReader(new InputStreamReader(processOut, + DEFAULT_CHARSET)); } catch (UnsupportedEncodingException e) { throw new RuntimeException(e); } diff --git a/storm-client/src/jvm/org/apache/storm/multilang/NoOutputException.java b/storm-client/src/jvm/org/apache/storm/multilang/NoOutputException.java index f733c553ddc..77bbd710c36 100644 --- a/storm-client/src/jvm/org/apache/storm/multilang/NoOutputException.java +++ b/storm-client/src/jvm/org/apache/storm/multilang/NoOutputException.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/multilang/ShellMsg.java b/storm-client/src/jvm/org/apache/storm/multilang/ShellMsg.java index 0695d868f97..d1193af7a89 100644 --- a/storm-client/src/jvm/org/apache/storm/multilang/ShellMsg.java +++ b/storm-client/src/jvm/org/apache/storm/multilang/ShellMsg.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -16,11 +22,14 @@ import java.util.List; /** - * ShellMsg is an object that represents the data sent to a shell component from a process that implements a multi-language protocol. It is + * ShellMsg is an object that represents the data sent to a shell component from a process that + * implements a multi-language protocol. It is * the union of all data types that a component can send to Storm. * - *

    ShellMsgs are objects received from the ISerializer interface, after the serializer has deserialized the data from the underlying wire - * protocol. The ShellMsg class allows for a decoupling between the serialized representation of the data and the data itself. + *

    ShellMsgs are objects received from the ISerializer interface, after the serializer has + * deserialized the data from the underlying wire + * protocol. The ShellMsg class allows for a decoupling between the serialized representation of the + * data and the data itself. */ public class ShellMsg { private String command; @@ -32,7 +41,7 @@ public class ShellMsg { private List tuple; private boolean needTaskIds; - //metrics rpc + // metrics rpc private String metricName; private Object metricParams; private ShellLogLevel logLevel = ShellLogLevel.INFO; @@ -156,7 +165,7 @@ public String toString() { + '}'; } - //logLevel + // logLevel public enum ShellLogLevel { TRACE, DEBUG, INFO, WARN, ERROR; diff --git a/storm-client/src/jvm/org/apache/storm/multilang/SpoutMsg.java b/storm-client/src/jvm/org/apache/storm/multilang/SpoutMsg.java index 870ed03c1f3..aa89351acf1 100644 --- a/storm-client/src/jvm/org/apache/storm/multilang/SpoutMsg.java +++ b/storm-client/src/jvm/org/apache/storm/multilang/SpoutMsg.java @@ -1,23 +1,32 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.multilang; /** - * SpoutMsg is an object that represents the data sent from a shell spout to a process that implements a multi-language spout. The SpoutMsg + * SpoutMsg is an object that represents the data sent from a shell spout to a process that + * implements a multi-language spout. The SpoutMsg * is used to send a "next", "ack" or "fail" message to a spout. * - *

    Spout messages are objects sent to the ISerializer interface, for serialization according to the wire protocol implemented by the - * serializer. The SpoutMsg class allows for a decoupling between the serialized representation of the data and the data itself. + *

    Spout messages are objects sent to the ISerializer interface, for serialization according to + * the wire protocol implemented by the + * serializer. The SpoutMsg class allows for a decoupling between the serialized representation of + * the data and the data itself. */ public class SpoutMsg { private String command; diff --git a/storm-client/src/jvm/org/apache/storm/networktopography/AbstractDNSToSwitchMapping.java b/storm-client/src/jvm/org/apache/storm/networktopography/AbstractDNSToSwitchMapping.java index 480e32e3902..6f9f26b639f 100644 --- a/storm-client/src/jvm/org/apache/storm/networktopography/AbstractDNSToSwitchMapping.java +++ b/storm-client/src/jvm/org/apache/storm/networktopography/AbstractDNSToSwitchMapping.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

    - * http://www.apache.org/licenses/LICENSE-2.0 - *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

    http://www.apache.org/licenses/LICENSE-2.0 + * + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -19,8 +25,10 @@ /** * This is a base class for DNS to Switch mappings. * - *

    It is not mandatory to derive {@link DNSToSwitchMapping} implementations from it, but it is strongly recommended, - * as it makes it easy for the developers to add new methods to this base class that are automatically picked up by all + *

    It is not mandatory to derive {@link DNSToSwitchMapping} implementations from it, but it is + * strongly recommended, + * as it makes it easy for the developers to add new methods to this base class that are + * automatically picked up by all * implementations. */ @SuppressWarnings("checkstyle:AbbreviationAsWordInName") @@ -34,8 +42,10 @@ protected AbstractDNSToSwitchMapping() { } /** - * Predicate that indicates that the switch mapping is known to be single-switch. The base class returns false: it assumes all mappings - * are multi-rack. Subclasses may override this with methods that are more aware of their topologies. + * Predicate that indicates that the switch mapping is known to be single-switch. The base class + * returns false: it assumes all mappings + * are multi-rack. Subclasses may override this with methods that are more aware of their + * topologies. * * @return true if the mapping thinks that it is on a single switch */ @@ -53,7 +63,8 @@ public Map getSwitchMap() { } /** - * Generate a string listing the switch mapping implementation, the mapping for every known node and the number of nodes and unique + * Generate a string listing the switch mapping implementation, the mapping for every known node + * and the number of nodes and unique * switches known about -each entry to a separate line. * * @return a string that can be presented to the ops team or used in debug messages. diff --git a/storm-client/src/jvm/org/apache/storm/networktopography/DNSToSwitchMapping.java b/storm-client/src/jvm/org/apache/storm/networktopography/DNSToSwitchMapping.java index e2cf14bf3c7..cb76814299a 100644 --- a/storm-client/src/jvm/org/apache/storm/networktopography/DNSToSwitchMapping.java +++ b/storm-client/src/jvm/org/apache/storm/networktopography/DNSToSwitchMapping.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

    - * http://www.apache.org/licenses/LICENSE-2.0 - *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

    http://www.apache.org/licenses/LICENSE-2.0 + * + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -23,17 +29,23 @@ public interface DNSToSwitchMapping { String DEFAULT_RACK = "/default-rack"; /** - * Resolves a list of DNS-names/IP-address and returns back a map of DNS-name->switch information ( network paths). - * Consider an element in the argument list - x.y.com. The switch information that is returned must be a network - * path of the form /foo/rack, where / is the root, and 'foo' is the switch where 'rack' is connected. Note the - * hostname/ip-address is not part of the returned path. The network topology of the cluster would determine the + * Resolves a list of DNS-names/IP-address and returns back a map of DNS-name->switch + * information ( network paths). + * Consider an element in the argument list - x.y.com. The switch information that is returned + * must be a network + * path of the form /foo/rack, where / is the root, and 'foo' is the switch where 'rack' is + * connected. Note the + * hostname/ip-address is not part of the returned path. The network topology of the cluster + * would determine the * number of components in the network path. * - *

    If a name cannot be resolved to a rack, the implementation should return {DEFAULT_RACK}. This is what the + *

    If a name cannot be resolved to a rack, the implementation should return {DEFAULT_RACK}. + * This is what the * bundled implementations do, though it is not a formal requirement. * * @param names the list of hosts to resolve (can be empty) - * @return Map of hosts to resolved network paths. If names is empty, then return empty Map + * @return Map of hosts to resolved network paths. If names is empty, then return empty + * Map */ Map resolve(List names); } diff --git a/storm-client/src/jvm/org/apache/storm/networktopography/DefaultRackDNSToSwitchMapping.java b/storm-client/src/jvm/org/apache/storm/networktopography/DefaultRackDNSToSwitchMapping.java index 94535135b95..5c48ac62527 100644 --- a/storm-client/src/jvm/org/apache/storm/networktopography/DefaultRackDNSToSwitchMapping.java +++ b/storm-client/src/jvm/org/apache/storm/networktopography/DefaultRackDNSToSwitchMapping.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -18,7 +24,8 @@ import java.util.concurrent.ConcurrentHashMap; /** - * This class implements the {@link DNSToSwitchMapping} interface It returns the DEFAULT_RACK for every host. + * This class implements the {@link DNSToSwitchMapping} interface It returns the DEFAULT_RACK for + * every host. */ @SuppressWarnings("checkstyle:AbbreviationAsWordInName") public final class DefaultRackDNSToSwitchMapping extends AbstractDNSToSwitchMapping { @@ -30,7 +37,7 @@ public Map resolve(List names) { Map m = new HashMap<>(); if (names.isEmpty()) { - //name list is empty, return an empty map + // name list is empty, return an empty map return m; } for (String name : names) { @@ -42,6 +49,7 @@ public Map resolve(List names) { @Override public String toString() { - return "DefaultRackDNSToSwitchMapping (" + mappingCache.size() + " mappings cached)" + dumpTopology(); + return "DefaultRackDNSToSwitchMapping (" + mappingCache.size() + " mappings cached)" + + dumpTopology(); } } diff --git a/storm-client/src/jvm/org/apache/storm/nimbus/ILeaderElector.java b/storm-client/src/jvm/org/apache/storm/nimbus/ILeaderElector.java index f0d877f7b8f..434d58e9a9d 100644 --- a/storm-client/src/jvm/org/apache/storm/nimbus/ILeaderElector.java +++ b/storm-client/src/jvm/org/apache/storm/nimbus/ILeaderElector.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -24,39 +30,46 @@ public interface ILeaderElector extends AutoCloseable { /** * Method guaranteed to be called as part of initialization of leader elector instance. + * * @param conf configuration */ void prepare(Map conf); /** - * queue up for leadership lock. The call returns immediately and the caller must + * Queue up for leadership lock. The call returns immediately and the caller must * check isLeader() to perform any leadership action. This method can be called * multiple times so it needs to be idempotent. */ void addToLeaderLockQueue() throws Exception; /** - * Removes the caller from leadership election, relinquishing leadership if acquired, then requeues for leadership after the specified + * Removes the caller from leadership election, relinquishing leadership if acquired, then + * requeues for leadership after the specified * delay. + * * @param delayMs The delay to wait before re-entering the election */ void quitElectionFor(int delayMs) throws Exception; /** * Decide if the caller currently has the leader lock. + * * @return true if the caller currently has the leader lock. */ boolean isLeader() throws Exception; /** * Get the current leader's address. + * * @return the current leader's address, may return null if no one has the lock. */ NimbusInfo getLeader(); /** - * Wait for the caller to gain leadership. This should only be used in single-Nimbus clusters, and is only useful to allow testing - * code to wait for a LocalCluster's Nimbus to gain leadership before trying to submit topologies. + * Wait for the caller to gain leadership. This should only be used in single-Nimbus clusters, + * and is only useful to allow testing + * code to wait for a LocalCluster's Nimbus to gain leadership before trying to submit + * topologies. * * @return true is leadership was acquired, false otherwise */ @@ -65,6 +78,7 @@ public interface ILeaderElector extends AutoCloseable { /** * Get list of current nimbus addresses. + * * @return list of current nimbus addresses, includes leader. */ List getAllNimbuses() throws Exception; diff --git a/storm-client/src/jvm/org/apache/storm/nimbus/NimbusInfo.java b/storm-client/src/jvm/org/apache/storm/nimbus/NimbusInfo.java index b5e7bd22292..2be2920f8c4 100644 --- a/storm-client/src/jvm/org/apache/storm/nimbus/NimbusInfo.java +++ b/storm-client/src/jvm/org/apache/storm/nimbus/NimbusInfo.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -26,8 +32,10 @@ public class NimbusInfo implements Serializable { private static final long serialVersionUID = 2161446155116099333L; private static final Logger LOG = LoggerFactory.getLogger(NimbusInfo.class); - private static final Pattern NON_TLS_HOST_PORT_PATTERN_FALLBACK = Pattern.compile("^(.*):([0-9]+)$"); - private static final Pattern TLS_HOST_PORT_PATTERN = Pattern.compile("^(.*):([0-9]+):([0-9]+)$"); + private static final Pattern NON_TLS_HOST_PORT_PATTERN_FALLBACK = Pattern + .compile("^(.*):([0-9]+)$"); + private static final Pattern TLS_HOST_PORT_PATTERN = Pattern + .compile("^(.*):([0-9]+):([0-9]+)$"); private String host; private int port; private int tlsPort; @@ -53,14 +61,17 @@ public NimbusInfo(String host, int port, int tlsPort, boolean isLeader) { public static NimbusInfo parse(String nimbusInfo) { Matcher m = TLS_HOST_PORT_PATTERN.matcher(nimbusInfo); if (m.matches()) { - return new NimbusInfo(m.group(1), Integer.parseInt(m.group(2)), Integer.parseInt(m.group(3)), false); + return new NimbusInfo(m.group(1), Integer.parseInt(m.group(2)), Integer.parseInt(m + .group(3)), false); } else { - LOG.info("nimbusInfo {} doesn't match the format of host:port:tlsPort; fall back to the non-tls format host:port", nimbusInfo); + LOG.info("nimbusInfo {} doesn't match the format of host:port:tlsPort; fall back to " + + "the non-tls format host:port", nimbusInfo); m = NON_TLS_HOST_PORT_PATTERN_FALLBACK.matcher(nimbusInfo); if (m.matches()) { return new NimbusInfo(m.group(1), Integer.parseInt(m.group(2)), false); } else { - throw new RuntimeException("nimbusInfo should have format of host:port:tlsPort or host:port, invalid string " + nimbusInfo); + throw new RuntimeException("nimbusInfo should have format of host:port:tlsPort or " + + "host:port, invalid string " + nimbusInfo); } } } @@ -80,7 +91,8 @@ public static NimbusInfo fromConf(Map conf) { return new NimbusInfo(host, port, tlsPort, false); } catch (UnknownHostException e) { - throw new RuntimeException("Something wrong with network/dns config, host cant figure out its name", e); + throw new RuntimeException("Something wrong with network/dns config, host cant figure " + + "out its name", e); } } diff --git a/storm-client/src/jvm/org/apache/storm/pacemaker/PacemakerClient.java b/storm-client/src/jvm/org/apache/storm/pacemaker/PacemakerClient.java index b4a4b66b3ef..fc01961a24b 100644 --- a/storm-client/src/jvm/org/apache/storm/pacemaker/PacemakerClient.java +++ b/storm-client/src/jvm/org/apache/storm/pacemaker/PacemakerClient.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -42,8 +48,10 @@ /** * Netty client that sends heartbeat requests to a single Pacemaker server. * - * @deprecated Pacemaker is deprecated and only kept for backward compatibility; it will be removed in a future release. - * Use the default heartbeat path instead: workers heartbeat to their supervisor, which reports them to Nimbus over + * @deprecated Pacemaker is deprecated and only kept for backward compatibility; it will be removed + * in a future release. + * Use the default heartbeat path instead: workers heartbeat to their supervisor, which reports + * them to Nimbus over * Thrift, with the default ZooKeeper-based cluster state store ({@code org.apache.storm.cluster.ZKStateStorageFactory}). */ @Deprecated @@ -66,7 +74,8 @@ public class PacemakerClient implements ISaslClient { private static final int maxRetries = 10; private String host; - private StormBoundedExponentialBackoffRetry backoff = new StormBoundedExponentialBackoffRetry(100, 5000, 20); + private StormBoundedExponentialBackoffRetry backoff = + new StormBoundedExponentialBackoffRetry(100, 5000, 20); private int retryTimes = 0; public PacemakerClient(Map config, String host) { @@ -82,10 +91,12 @@ public PacemakerClient(Map config, String host) { case "DIGEST": authMethod = ThriftNettyClientCodec.AuthMethod.DIGEST; - secret = ClientAuthUtils.makeDigestPayload(config, ClientAuthUtils.LOGIN_CONTEXT_PACEMAKER_DIGEST); + secret = ClientAuthUtils.makeDigestPayload(config, + ClientAuthUtils.LOGIN_CONTEXT_PACEMAKER_DIGEST); if (secret == null) { LOG.error("Can't start pacemaker server without digest secret."); - throw new RuntimeException("Can't start pacemaker server without digest secret."); + throw new RuntimeException("Can't start pacemaker server without digest " + + "secret."); } break; @@ -112,7 +123,8 @@ public PacemakerClient(Map config, String host) { // 0 means DEFAULT_EVENT_LOOP_THREADS // https://github.com/netty/netty/blob/netty-4.1.24.Final/transport/src/main/java/io/netty/channel/MultithreadEventLoopGroup.java#L40 int maxWorkers = (int) config.get(Config.PACEMAKER_CLIENT_MAX_THREADS); - this.workerEventLoopGroup = new NioEventLoopGroup(maxWorkers > 0 ? maxWorkers : 0, workerFactory); + this.workerEventLoopGroup = new NioEventLoopGroup(maxWorkers > 0 ? maxWorkers : 0, + workerFactory); int thriftMessageMaxSize = (Integer) config.get(Config.PACEMAKER_THRIFT_MESSAGE_SIZE_MAX); bootstrap = new Bootstrap() .group(workerEventLoopGroup) @@ -120,9 +132,11 @@ public PacemakerClient(Map config, String host) { .option(ChannelOption.TCP_NODELAY, true) .option(ChannelOption.SO_SNDBUF, 5242880) .option(ChannelOption.SO_KEEPALIVE, true) - .option(ChannelOption.WRITE_BUFFER_WATER_MARK, new WriteBufferWaterMark(8 * 1024, 32 * 1024)) + .option(ChannelOption.WRITE_BUFFER_WATER_MARK, new WriteBufferWaterMark(8 * 1024, + 32 * 1024)) .option(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT) - .handler(new ThriftNettyClientCodec(this, config, authMethod, host, thriftMessageMaxSize)); + .handler(new ThriftNettyClientCodec(this, config, authMethod, host, + thriftMessageMaxSize)); int port = (int) config.get(Config.PACEMAKER_PORT); remoteAddr = new InetSocketAddress(host, port); @@ -180,7 +194,8 @@ public HBMessage send(HBMessage m) throws PacemakerConnectionException, Interrup m.wait(1000); } if (messages[next] != m && messages[next] != null) { - // messages[next] == null can happen if we lost the connection and subsequently reconnected or timed out. + // messages[next] == null can happen if we lost the connection and + // subsequently reconnected or timed out. HBMessage ret = messages[next]; messages[next] = null; LOG.debug("Got Response: {}", ret); @@ -190,13 +205,16 @@ public HBMessage send(HBMessage m) throws PacemakerConnectionException, Interrup if (retry <= 0) { throw e; } - LOG.error("Error attempting to write to a channel to host {} - {}", host, e.getMessage()); + LOG.error("Error attempting to write to a channel to host {} - {}", host, e + .getMessage()); } if (retry <= 0) { - throw new PacemakerConnectionException("couldn't get response after " + maxRetries + " attempts."); + throw new PacemakerConnectionException("couldn't get response after " + + maxRetries + " attempts."); } retry--; - LOG.warn("Not getting response or getting null response. Making {} more attempts for {}.", retry, host); + LOG.warn("Not getting response or getting null response. Making {} more attempts " + + "for {}.", retry, host); } } } @@ -209,7 +227,8 @@ private void waitUntilReady() throws PacemakerConnectionException, InterruptedEx LOG.debug("Waiting for netty channel to be ready."); this.wait(1000); if (!ready.get() || channelRef.get() == null) { - throw new PacemakerConnectionException("Timed out waiting for channel ready."); + throw new PacemakerConnectionException("Timed out waiting for channel " + + "ready."); } } } diff --git a/storm-client/src/jvm/org/apache/storm/pacemaker/PacemakerClientHandler.java b/storm-client/src/jvm/org/apache/storm/pacemaker/PacemakerClientHandler.java index c3e183cd8e4..d0d5d9acad0 100644 --- a/storm-client/src/jvm/org/apache/storm/pacemaker/PacemakerClientHandler.java +++ b/storm-client/src/jvm/org/apache/storm/pacemaker/PacemakerClientHandler.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -24,8 +30,10 @@ /** * Inbound handler of the Pacemaker client pipeline. * - * @deprecated Pacemaker is deprecated and only kept for backward compatibility; it will be removed in a future release. - * Use the default heartbeat path instead: workers heartbeat to their supervisor, which reports them to Nimbus over + * @deprecated Pacemaker is deprecated and only kept for backward compatibility; it will be removed + * in a future release. + * Use the default heartbeat path instead: workers heartbeat to their supervisor, which reports + * them to Nimbus over * Thrift, with the default ZooKeeper-based cluster state store ({@code org.apache.storm.cluster.ZKStateStorageFactory}). */ @Deprecated diff --git a/storm-client/src/jvm/org/apache/storm/pacemaker/PacemakerClientPool.java b/storm-client/src/jvm/org/apache/storm/pacemaker/PacemakerClientPool.java index 43c6c1f7c4c..b26d258e133 100644 --- a/storm-client/src/jvm/org/apache/storm/pacemaker/PacemakerClientPool.java +++ b/storm-client/src/jvm/org/apache/storm/pacemaker/PacemakerClientPool.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -26,8 +32,10 @@ /** * Pool of clients for the configured Pacemaker servers. * - * @deprecated Pacemaker is deprecated and only kept for backward compatibility; it will be removed in a future release. - * Use the default heartbeat path instead: workers heartbeat to their supervisor, which reports them to Nimbus over + * @deprecated Pacemaker is deprecated and only kept for backward compatibility; it will be removed + * in a future release. + * Use the default heartbeat path instead: workers heartbeat to their supervisor, which reports + * them to Nimbus over * Thrift, with the default ZooKeeper-based cluster state store ({@code org.apache.storm.cluster.ZKStateStorageFactory}). */ @Deprecated @@ -72,7 +80,8 @@ public List sendAll(HBMessage m) throws PacemakerConnectionException, HBMessage response = getClientForServer(s).send(m); responses.add(response); } catch (PacemakerConnectionException e) { - LOG.warn("Failed to connect to the pacemaker server {}, attempting to reconnect", s); + LOG.warn("Failed to connect to the pacemaker server {}, attempting to reconnect", + s); getClientForServer(s).reconnect(); } } diff --git a/storm-client/src/jvm/org/apache/storm/pacemaker/PacemakerConnectionException.java b/storm-client/src/jvm/org/apache/storm/pacemaker/PacemakerConnectionException.java index 3390cc5a897..2320e531be4 100644 --- a/storm-client/src/jvm/org/apache/storm/pacemaker/PacemakerConnectionException.java +++ b/storm-client/src/jvm/org/apache/storm/pacemaker/PacemakerConnectionException.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -15,8 +21,10 @@ /** * Thrown when no connection to a Pacemaker server is available. * - * @deprecated Pacemaker is deprecated and only kept for backward compatibility; it will be removed in a future release. - * Use the default heartbeat path instead: workers heartbeat to their supervisor, which reports them to Nimbus over + * @deprecated Pacemaker is deprecated and only kept for backward compatibility; it will be removed + * in a future release. + * Use the default heartbeat path instead: workers heartbeat to their supervisor, which reports + * them to Nimbus over * Thrift, with the default ZooKeeper-based cluster state store ({@code org.apache.storm.cluster.ZKStateStorageFactory}). */ @Deprecated diff --git a/storm-client/src/jvm/org/apache/storm/pacemaker/codec/ThriftDecoder.java b/storm-client/src/jvm/org/apache/storm/pacemaker/codec/ThriftDecoder.java index b66420e5c3e..007bd983683 100644 --- a/storm-client/src/jvm/org/apache/storm/pacemaker/codec/ThriftDecoder.java +++ b/storm-client/src/jvm/org/apache/storm/pacemaker/codec/ThriftDecoder.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -28,8 +34,10 @@ /** * Decodes length-prefixed thrift {@link HBMessage} frames of the Pacemaker protocol. * - * @deprecated Pacemaker is deprecated and only kept for backward compatibility; it will be removed in a future release. - * Use the default heartbeat path instead: workers heartbeat to their supervisor, which reports them to Nimbus over + * @deprecated Pacemaker is deprecated and only kept for backward compatibility; it will be removed + * in a future release. + * Use the default heartbeat path instead: workers heartbeat to their supervisor, which reports + * them to Nimbus over * Thrift, with the default ZooKeeper-based cluster state store ({@code org.apache.storm.cluster.ZKStateStorageFactory}). */ @Deprecated @@ -44,8 +52,10 @@ public class ThriftDecoder extends ByteToMessageDecoder { private final int maxLength; /** - * Whether this decoder sits in a Pacemaker server pipeline. A server only accepts the control message a client - * sends to start the SASL handshake; any other control frame is dropped and the connection closed. + * Whether this decoder sits in a Pacemaker server pipeline. A server only accepts the control + * message a client + * sends to start the SASL handshake; any other control frame is dropped and the connection + * closed. */ private final boolean serverSide; @@ -60,7 +70,8 @@ public ThriftDecoder(final int maxLengthBytes) { * Instantiate a ThriftDecoder that accepts serialized messages of at most maxLength bytes. * * @param maxLengthBytes the maximum length of a serialized thrift message - * @param serverSide true if the decoder is used by a Pacemaker server, which restricts the control messages it accepts + * @param serverSide true if the decoder is used by a Pacemaker server, which restricts the + * control messages it accepts */ public ThriftDecoder(final int maxLengthBytes, final boolean serverSide) { maxLength = maxLengthBytes; @@ -68,7 +79,8 @@ public ThriftDecoder(final int maxLengthBytes, final boolean serverSide) { } @Override - protected void decode(ChannelHandlerContext channelHandlerContext, ByteBuf buf, List out) throws Exception { + protected void decode(ChannelHandlerContext channelHandlerContext, ByteBuf buf, + List out) throws Exception { long available = buf.readableBytes(); if (available < INTEGER_SIZE) { return; diff --git a/storm-client/src/jvm/org/apache/storm/pacemaker/codec/ThriftEncoder.java b/storm-client/src/jvm/org/apache/storm/pacemaker/codec/ThriftEncoder.java index d029888f176..eac43367e4d 100644 --- a/storm-client/src/jvm/org/apache/storm/pacemaker/codec/ThriftEncoder.java +++ b/storm-client/src/jvm/org/apache/storm/pacemaker/codec/ThriftEncoder.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -31,8 +37,10 @@ /** * Encodes messages as length-prefixed thrift {@link HBMessage} frames of the Pacemaker protocol. * - * @deprecated Pacemaker is deprecated and only kept for backward compatibility; it will be removed in a future release. - * Use the default heartbeat path instead: workers heartbeat to their supervisor, which reports them to Nimbus over + * @deprecated Pacemaker is deprecated and only kept for backward compatibility; it will be removed + * in a future release. + * Use the default heartbeat path instead: workers heartbeat to their supervisor, which reports + * them to Nimbus over * Thrift, with the default ZooKeeper-based cluster state store ({@code org.apache.storm.cluster.ZKStateStorageFactory}). */ @Deprecated @@ -62,7 +70,8 @@ private HBMessage encodeNettySerializable(ByteBufAllocator alloc, } @Override - protected void encode(ChannelHandlerContext channelHandlerContext, Object msg, List out) throws Exception { + protected void encode(ChannelHandlerContext channelHandlerContext, Object msg, + List out) throws Exception { if (msg == null) { return; } diff --git a/storm-client/src/jvm/org/apache/storm/pacemaker/codec/ThriftNettyClientCodec.java b/storm-client/src/jvm/org/apache/storm/pacemaker/codec/ThriftNettyClientCodec.java index 55f94d5daef..be6f181e9e7 100644 --- a/storm-client/src/jvm/org/apache/storm/pacemaker/codec/ThriftNettyClientCodec.java +++ b/storm-client/src/jvm/org/apache/storm/pacemaker/codec/ThriftNettyClientCodec.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -28,8 +34,10 @@ /** * Builds the Pacemaker client pipeline. * - * @deprecated Pacemaker is deprecated and only kept for backward compatibility; it will be removed in a future release. - * Use the default heartbeat path instead: workers heartbeat to their supervisor, which reports them to Nimbus over + * @deprecated Pacemaker is deprecated and only kept for backward compatibility; it will be removed + * in a future release. + * Use the default heartbeat path instead: workers heartbeat to their supervisor, which reports + * them to Nimbus over * Thrift, with the default ZooKeeper-based cluster state store ({@code org.apache.storm.cluster.ZKStateStorageFactory}). */ @Deprecated diff --git a/storm-client/src/jvm/org/apache/storm/policy/IWaitStrategy.java b/storm-client/src/jvm/org/apache/storm/policy/IWaitStrategy.java index 1934a02686a..88e7c408728 100644 --- a/storm-client/src/jvm/org/apache/storm/policy/IWaitStrategy.java +++ b/storm-client/src/jvm/org/apache/storm/policy/IWaitStrategy.java @@ -22,11 +22,11 @@ import org.apache.storm.Config; import org.apache.storm.utils.ReflectionUtils; - public interface IWaitStrategy { static IWaitStrategy createBackPressureWaitStrategy(Map topologyConf) { IWaitStrategy producerWaitStrategy = - ReflectionUtils.newInstance((String) topologyConf.get(Config.TOPOLOGY_BACKPRESSURE_WAIT_STRATEGY)); + ReflectionUtils.newInstance((String) topologyConf + .get(Config.TOPOLOGY_BACKPRESSURE_WAIT_STRATEGY)); producerWaitStrategy.prepare(topologyConf, WaitSituation.BACK_PRESSURE_WAIT); return producerWaitStrategy; } @@ -34,9 +34,11 @@ static IWaitStrategy createBackPressureWaitStrategy(Map topology void prepare(Map conf, WaitSituation waitSituation); /** - * Implementations of this method should be thread-safe (preferably no side-effects and lock-free). + * Implementations of this method should be thread-safe (preferably no side-effects and + * lock-free). * - *

    Supports static or dynamic backoff. Dynamic backoff relies on idleCounter to estimate how long caller has been idling. + *

    Supports static or dynamic backoff. Dynamic backoff relies on idleCounter to estimate how + * long caller has been idling. *

          * 
          *  int idleCounter = 0;
    diff --git a/storm-client/src/jvm/org/apache/storm/policy/WaitStrategyPark.java b/storm-client/src/jvm/org/apache/storm/policy/WaitStrategyPark.java
    index 702be124c7d..5622446bbb7 100644
    --- a/storm-client/src/jvm/org/apache/storm/policy/WaitStrategyPark.java
    +++ b/storm-client/src/jvm/org/apache/storm/policy/WaitStrategyPark.java
    @@ -37,11 +37,14 @@ public WaitStrategyPark(long microsec) {
         @Override
         public void prepare(Map conf, WaitSituation waitSituation) {
             if (waitSituation == WaitSituation.SPOUT_WAIT) {
    -            parkTimeNanoSec = 1_000 * ObjectReader.getLong(conf.get(Config.TOPOLOGY_SPOUT_WAIT_PARK_MICROSEC));
    +            parkTimeNanoSec = 1_000 * ObjectReader.getLong(conf
    +                    .get(Config.TOPOLOGY_SPOUT_WAIT_PARK_MICROSEC));
             } else if (waitSituation == WaitSituation.BOLT_WAIT) {
    -            parkTimeNanoSec = 1_000 * ObjectReader.getLong(conf.get(Config.TOPOLOGY_BOLT_WAIT_PARK_MICROSEC));
    +            parkTimeNanoSec = 1_000 * ObjectReader.getLong(conf
    +                    .get(Config.TOPOLOGY_BOLT_WAIT_PARK_MICROSEC));
             } else if (waitSituation == WaitSituation.BACK_PRESSURE_WAIT) {
    -            parkTimeNanoSec = 1_000 * ObjectReader.getLong(conf.get(Config.TOPOLOGY_BACKPRESSURE_WAIT_PARK_MICROSEC));
    +            parkTimeNanoSec = 1_000 * ObjectReader.getLong(conf
    +                    .get(Config.TOPOLOGY_BACKPRESSURE_WAIT_PARK_MICROSEC));
             } else {
                 throw new IllegalArgumentException("Unknown wait situation : " + waitSituation);
             }
    diff --git a/storm-client/src/jvm/org/apache/storm/policy/WaitStrategyProgressive.java b/storm-client/src/jvm/org/apache/storm/policy/WaitStrategyProgressive.java
    index a721db75829..c9525c04a5c 100644
    --- a/storm-client/src/jvm/org/apache/storm/policy/WaitStrategyProgressive.java
    +++ b/storm-client/src/jvm/org/apache/storm/policy/WaitStrategyProgressive.java
    @@ -26,12 +26,17 @@
     /**
      * A Progressive Wait Strategy.
      *
    - * 

    Has three levels of idling. Stays in each level for a configured number of iterations before entering the next level. - * Level 1 - No idling. Returns immediately. Stays in this level for `level1Count` iterations. Level 2 - Calls LockSupport.parkNanos(1). - * Stays in this level for `level2Count` iterations Level 3 - Calls Thread.sleep(). Stays in this level until wait situation changes. + *

    Has three levels of idling. Stays in each level for a configured number of iterations before + * entering the next level. + * Level 1 - No idling. Returns immediately. Stays in this level for `level1Count` iterations. Level + * 2 - Calls LockSupport.parkNanos(1). + * Stays in this level for `level2Count` iterations Level 3 - Calls Thread.sleep(). Stays in this + * level until wait situation changes. * - *

    The initial spin can be useful to prevent downstream bolt from repeatedly sleeping/parking when the upstream component is a bit - * relatively slower. Allows downstream bolt can enter deeper wait states only if the traffic to it appears to have reduced. + *

    The initial spin can be useful to prevent downstream bolt from repeatedly sleeping/parking + * when the upstream component is a bit + * relatively slower. Allows downstream bolt can enter deeper wait states only if the traffic to it + * appears to have reduced. */ public class WaitStrategyProgressive implements IWaitStrategy { private int level1Count; @@ -41,17 +46,26 @@ public class WaitStrategyProgressive implements IWaitStrategy { @Override public void prepare(Map conf, WaitSituation waitSituation) { if (waitSituation == WaitSituation.SPOUT_WAIT) { - level1Count = ObjectReader.getInt(conf.get(Config.TOPOLOGY_SPOUT_WAIT_PROGRESSIVE_LEVEL1_COUNT)); - level2Count = ObjectReader.getInt(conf.get(Config.TOPOLOGY_SPOUT_WAIT_PROGRESSIVE_LEVEL2_COUNT)); - level3SleepMs = ObjectReader.getLong(conf.get(Config.TOPOLOGY_SPOUT_WAIT_PROGRESSIVE_LEVEL3_SLEEP_MILLIS)); + level1Count = ObjectReader.getInt(conf + .get(Config.TOPOLOGY_SPOUT_WAIT_PROGRESSIVE_LEVEL1_COUNT)); + level2Count = ObjectReader.getInt(conf + .get(Config.TOPOLOGY_SPOUT_WAIT_PROGRESSIVE_LEVEL2_COUNT)); + level3SleepMs = ObjectReader.getLong(conf + .get(Config.TOPOLOGY_SPOUT_WAIT_PROGRESSIVE_LEVEL3_SLEEP_MILLIS)); } else if (waitSituation == WaitSituation.BOLT_WAIT) { - level1Count = ObjectReader.getInt(conf.get(Config.TOPOLOGY_BOLT_WAIT_PROGRESSIVE_LEVEL1_COUNT)); - level2Count = ObjectReader.getInt(conf.get(Config.TOPOLOGY_BOLT_WAIT_PROGRESSIVE_LEVEL2_COUNT)); - level3SleepMs = ObjectReader.getLong(conf.get(Config.TOPOLOGY_BOLT_WAIT_PROGRESSIVE_LEVEL3_SLEEP_MILLIS)); + level1Count = ObjectReader.getInt(conf + .get(Config.TOPOLOGY_BOLT_WAIT_PROGRESSIVE_LEVEL1_COUNT)); + level2Count = ObjectReader.getInt(conf + .get(Config.TOPOLOGY_BOLT_WAIT_PROGRESSIVE_LEVEL2_COUNT)); + level3SleepMs = ObjectReader.getLong(conf + .get(Config.TOPOLOGY_BOLT_WAIT_PROGRESSIVE_LEVEL3_SLEEP_MILLIS)); } else if (waitSituation == WaitSituation.BACK_PRESSURE_WAIT) { - level1Count = ObjectReader.getInt(conf.get(Config.TOPOLOGY_BACKPRESSURE_WAIT_PROGRESSIVE_LEVEL1_COUNT)); - level2Count = ObjectReader.getInt(conf.get(Config.TOPOLOGY_BACKPRESSURE_WAIT_PROGRESSIVE_LEVEL2_COUNT)); - level3SleepMs = ObjectReader.getLong(conf.get(Config.TOPOLOGY_BACKPRESSURE_WAIT_PROGRESSIVE_LEVEL3_SLEEP_MILLIS)); + level1Count = ObjectReader.getInt(conf + .get(Config.TOPOLOGY_BACKPRESSURE_WAIT_PROGRESSIVE_LEVEL1_COUNT)); + level2Count = ObjectReader.getInt(conf + .get(Config.TOPOLOGY_BACKPRESSURE_WAIT_PROGRESSIVE_LEVEL2_COUNT)); + level3SleepMs = ObjectReader.getLong(conf + .get(Config.TOPOLOGY_BACKPRESSURE_WAIT_PROGRESSIVE_LEVEL3_SLEEP_MILLIS)); } else { throw new IllegalArgumentException("Unknown wait situation : " + waitSituation); } diff --git a/storm-client/src/jvm/org/apache/storm/scheduler/WorkerSlot.java b/storm-client/src/jvm/org/apache/storm/scheduler/WorkerSlot.java index fa963d22f9a..1f201ce19b5 100644 --- a/storm-client/src/jvm/org/apache/storm/scheduler/WorkerSlot.java +++ b/storm-client/src/jvm/org/apache/storm/scheduler/WorkerSlot.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/security/INimbusCredentialPlugin.java b/storm-client/src/jvm/org/apache/storm/security/INimbusCredentialPlugin.java index 8f302665307..7aa20bc73ba 100644 --- a/storm-client/src/jvm/org/apache/storm/security/INimbusCredentialPlugin.java +++ b/storm-client/src/jvm/org/apache/storm/security/INimbusCredentialPlugin.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -16,7 +22,8 @@ import org.apache.storm.daemon.Shutdownable; /** - * Nimbus auto credential plugin that will be called on nimbus host during submit topology option. User can specify a list of implementation + * Nimbus auto credential plugin that will be called on nimbus host during submit topology option. + * User can specify a list of implementation * using config key nimbus.autocredential.plugins.classes. */ public interface INimbusCredentialPlugin extends Shutdownable { @@ -29,21 +36,27 @@ public interface INimbusCredentialPlugin extends Shutdownable { void prepare(Map conf); /** - * Method that will be called on nimbus as part of submit topology. This plugin will be called at least once during the submit Topology - * action. It will be not be called during activate instead the credentials return by this method will be merged with the other + * Method that will be called on nimbus as part of submit topology. This plugin will be called + * at least once during the submit Topology + * action. It will be not be called during activate instead the credentials return by this + * method will be merged with the other * credentials in the topology and stored in zookeeper. * * @param credentials credentials map where more credentials will be added. * @param topologyConf topology configuration */ @Deprecated - default void populateCredentials(Map credentials, Map topologyConf) { - throw new IllegalStateException("One of the populateCredentials methods must be overridden by " + this); + default void populateCredentials(Map credentials, Map topologyConf) { + throw new IllegalStateException("One of the populateCredentials methods must be " + + "overridden by " + this); } /** - * Method that will be called on nimbus as part of submit topology. This plugin will be called at least once during the submit Topology - * action. It will be not be called during activate instead the credentials return by this method will be merged with the other + * Method that will be called on nimbus as part of submit topology. This plugin will be called + * at least once during the submit Topology + * action. It will be not be called during activate instead the credentials return by this + * method will be merged with the other * credentials in the topology and stored in zookeeper. * * @param credentials credentials map where more credentials will be added. @@ -51,7 +64,8 @@ default void populateCredentials(Map credentials, Map credentials, Map topoConf, final String topologyOwnerPrincipal) { + default void populateCredentials(Map credentials, Map topoConf, + final String topologyOwnerPrincipal) { populateCredentials(credentials, topoConf); } } diff --git a/storm-client/src/jvm/org/apache/storm/security/auth/AutoSSL.java b/storm-client/src/jvm/org/apache/storm/security/auth/AutoSSL.java index 6cf1773e91f..ca0d7978ad1 100644 --- a/storm-client/src/jvm/org/apache/storm/security/auth/AutoSSL.java +++ b/storm-client/src/jvm/org/apache/storm/security/auth/AutoSSL.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -26,10 +32,14 @@ import org.slf4j.LoggerFactory; /** - * This plugin is intended to be used for user topologies to send SSL keystore/truststore files to the remote workers. On the client side, - * this takes the files specified in ssl.credential.files, reads the file contents, base64's it, converts it to a String, and adds it to the - * credentials map. The key in the credentials map is the name of the file. On the worker side it uses the filenames from the - * ssl.credential.files config to lookup the keys in the credentials map and decodes it and writes it back out as a file. + * This plugin is intended to be used for user topologies to send SSL keystore/truststore files to + * the remote workers. On the client side, + * this takes the files specified in ssl.credential.files, reads the file contents, base64's it, + * converts it to a String, and adds it to the + * credentials map. The key in the credentials map is the name of the file. On the worker side it + * uses the filenames from the + * ssl.credential.files config to lookup the keys in the credentials map and decodes it and writes + * it back out as a file. * *

    User is responsible for referencing them from the topology code as {@code filename}. */ @@ -103,7 +113,8 @@ protected String getSSLWriteDirFromConf(Map conf) { Collection getSSLFilesFromConf(Map conf) { Object sslConf = conf.get(SSL_FILES_CONF); if (sslConf == null) { - LOG.info("No ssl files requested, if you want to use SSL please set {} to the list of files", + LOG.info("No ssl files requested, if you want to use SSL please set {} to the list of " + + "files", SSL_FILES_CONF); return null; } diff --git a/storm-client/src/jvm/org/apache/storm/security/auth/ClientAuthUtils.java b/storm-client/src/jvm/org/apache/storm/security/auth/ClientAuthUtils.java index 34326ecf9aa..e7ed90c59bf 100644 --- a/storm-client/src/jvm/org/apache/storm/security/auth/ClientAuthUtils.java +++ b/storm-client/src/jvm/org/apache/storm/security/auth/ClientAuthUtils.java @@ -74,7 +74,7 @@ public static String getJaasConf(Map topoConf) { public static Configuration getConfiguration(Map topoConf) { Configuration loginConf = null; - //find login file configuration from Storm configuration + // find login file configuration from Storm configuration String loginConfigurationFile = getJaasConf(topoConf); if ((loginConfigurationFile != null) && (loginConfigurationFile.length() > 0)) { File configFile = new File(loginConfigurationFile); @@ -83,7 +83,8 @@ public static Configuration getConfiguration(Map topoConf) { } try { URI configUri = configFile.toURI(); - loginConf = Configuration.getInstance("JavaLoginConfig", new URIParameter(configUri)); + loginConf = Configuration.getInstance("JavaLoginConfig", + new URIParameter(configUri)); } catch (Exception ex) { throw new RuntimeException(ex); } @@ -105,7 +106,8 @@ public static AppConfigurationEntry[] getEntries(Configuration configuration, return null; } - AppConfigurationEntry[] configurationEntries = configuration.getAppConfigurationEntry(section); + AppConfigurationEntry[] configurationEntries = configuration + .getAppConfigurationEntry(section); if (configurationEntries == null) { String errorMessage = "Could not find a '" + section + "' entry in this configuration."; throw new IOException(errorMessage); @@ -123,7 +125,8 @@ public static AppConfigurationEntry[] getEntries(Configuration configuration, public static SortedMap pullConfig(Map topoConf, String section) throws IOException { Configuration configuration = ClientAuthUtils.getConfiguration(topoConf); - AppConfigurationEntry[] configurationEntries = ClientAuthUtils.getEntries(configuration, section); + AppConfigurationEntry[] configurationEntries = ClientAuthUtils.getEntries(configuration, + section); if (configurationEntries == null) { return null; @@ -149,13 +152,15 @@ public static AppConfigurationEntry[] getEntries(Configuration configuration, * @param key The key to look up inside of the section * @return Return a the String value of the configuration value */ - public static String get(Map topoConf, String section, String key) throws IOException { + public static String get(Map topoConf, String section, + String key) throws IOException { Configuration configuration = ClientAuthUtils.getConfiguration(topoConf); return get(configuration, section, key); } static String get(Configuration configuration, String section, String key) throws IOException { - AppConfigurationEntry[] configurationEntries = ClientAuthUtils.getEntries(configuration, section); + AppConfigurationEntry[] configurationEntries = ClientAuthUtils.getEntries(configuration, + section); if (configurationEntries == null) { return null; @@ -184,7 +189,7 @@ public static IPrincipalToLocal getPrincipalToLocalPlugin(Map to LOG.warn("No principal to local given {}", Config.STORM_PRINCIPAL_TO_LOCAL_PLUGIN); } else { ptol = ReflectionUtils.newInstance(ptolClassname); - //TODO: this can only ever be null if someone is doing something odd with mocking + // TODO: this can only ever be null if someone is doing something odd with mocking // We should really fix the mocking and remove this if (ptol != null) { ptol.prepare(topoConf); @@ -202,12 +207,15 @@ public static IPrincipalToLocal getPrincipalToLocalPlugin(Map to * @param conf daemon configuration * @return the plugin */ - public static IGroupMappingServiceProvider getGroupMappingServiceProviderPlugin(Map conf) { + public static IGroupMappingServiceProvider getGroupMappingServiceProviderPlugin(Map conf) { IGroupMappingServiceProvider gmsp = null; try { - String gmspClassName = (String) conf.get(Config.STORM_GROUP_MAPPING_SERVICE_PROVIDER_PLUGIN); + String gmspClassName = (String) conf + .get(Config.STORM_GROUP_MAPPING_SERVICE_PROVIDER_PLUGIN); if (gmspClassName == null) { - LOG.warn("No group mapper given {}", Config.STORM_GROUP_MAPPING_SERVICE_PROVIDER_PLUGIN); + LOG.warn("No group mapper given {}", + Config.STORM_GROUP_MAPPING_SERVICE_PROVIDER_PLUGIN); } else { gmsp = ReflectionUtils.newInstance(gmspClassName); if (gmsp != null) { @@ -229,7 +237,8 @@ public static IGroupMappingServiceProvider getGroupMappingServiceProviderPlugin( public static Collection getCredentialRenewers(Map conf) { try { Set ret = new HashSet<>(); - Collection clazzes = (Collection) conf.get(Config.NIMBUS_CREDENTIAL_RENEWERS); + Collection clazzes = (Collection) conf + .get(Config.NIMBUS_CREDENTIAL_RENEWERS); if (clazzes != null) { for (String clazz : clazzes) { ICredentialsRenewer inst = ReflectionUtils.newInstance(clazz); @@ -249,10 +258,12 @@ public static Collection getCredentialRenewers(Map getNimbusAutoCredPlugins(Map conf) { + public static Collection getNimbusAutoCredPlugins(Map conf) { try { Set ret = new HashSet<>(); - Collection clazzes = (Collection) conf.get(Config.NIMBUS_AUTO_CRED_PLUGINS); + Collection clazzes = (Collection) conf + .get(Config.NIMBUS_AUTO_CRED_PLUGINS); if (clazzes != null) { for (String clazz : clazzes) { INimbusCredentialPlugin inst = ReflectionUtils.newInstance(clazz); @@ -275,7 +286,8 @@ public static Collection getNimbusAutoCredPlugins(Map getAutoCredentials(Map topoConf) { try { Set autos = new HashSet<>(); - Collection clazzes = (Collection) topoConf.get(Config.TOPOLOGY_AUTO_CREDENTIALS); + Collection clazzes = (Collection) topoConf + .get(Config.TOPOLOGY_AUTO_CREDENTIALS); if (clazzes != null) { for (String clazz : clazzes) { IAutoCredentials a = ReflectionUtils.newInstance(clazz); @@ -307,7 +319,8 @@ public static String workerTokenCredentialsKey(WorkerTokenServiceType type) { * @param type the type of service we are looking for. * @return the deserialized WorkerToken or null if none could be found. */ - public static WorkerToken readWorkerToken(Map credentials, WorkerTokenServiceType type) { + public static WorkerToken readWorkerToken(Map credentials, + WorkerTokenServiceType type) { WorkerToken ret = null; String key = workerTokenCredentialsKey(type); String tokenStr = credentials.get(key); @@ -318,7 +331,8 @@ public static WorkerToken readWorkerToken(Map credentials, Worke } /** - * Store a worker token in some credentials. It can be pulled back out by calling readWorkerToken. + * Store a worker token in some credentials. It can be pulled back out by calling + * readWorkerToken. * * @param credentials the credentials map. * @param token the token you want to store. @@ -346,11 +360,13 @@ public static WorkerToken findWorkerToken(Subject subject, final WorkerTokenServ } private static boolean willWorkerTokensBeStoredSecurely(Map conf) { - boolean overrideZkAuth = ObjectReader.getBoolean(conf.get("TESTING.ONLY.ENABLE.INSECURE.WORKER.TOKENS"), false); + boolean overrideZkAuth = ObjectReader.getBoolean(conf + .get("TESTING.ONLY.ENABLE.INSECURE.WORKER.TOKENS"), false); if (Utils.isZkAuthenticationConfiguredStormServer(conf)) { return true; } else if (overrideZkAuth) { - LOG.error("\n\n\t\tYOU HAVE ENABLED INSECURE WORKER TOKENS. IF THIS IS NOT A UNIT TEST PLEASE STOP NOW!!!\n\n"); + LOG.error("\n\n\t\tYOU HAVE ENABLED INSECURE WORKER TOKENS. IF THIS IS NOT A UNIT " + + "TEST PLEASE STOP NOW!!!\n\n"); return true; } return false; @@ -359,12 +375,14 @@ private static boolean willWorkerTokensBeStoredSecurely(Map conf /** * Check if worker tokens should be enabled on the server side or not. * - * @param multiThriftServer a collection of Thrift servers to know if the transport support tokens or not. + * @param multiThriftServer a collection of Thrift servers to know if the transport support + * tokens or not. * No need to create a token if the transport does not support it. * @param conf the daemon configuration to be sure the tokens are secure. * @return true if we can enable them, else false. */ - public static boolean areWorkerTokensEnabledServer(MultiThriftServer multiThriftServer, Map conf) { + public static boolean areWorkerTokensEnabledServer(MultiThriftServer multiThriftServer, + Map conf) { return multiThriftServer.supportsWorkerTokens() && willWorkerTokensBeStoredSecurely(conf); } @@ -375,7 +393,8 @@ public static boolean areWorkerTokensEnabledServer(MultiThriftServer multiThr * @param conf the daemon configuration to be sure the tokens are secure. * @return true if we can enable them, else false. */ - public static boolean areWorkerTokensEnabledServer(ThriftConnectionType connectionType, Map conf) { + public static boolean areWorkerTokensEnabledServer(ThriftConnectionType connectionType, + Map conf) { return connectionType.getWtType() != null && willWorkerTokensBeStoredSecurely(conf); } @@ -399,7 +418,7 @@ public static WorkerTokenInfo getWorkerTokenInfo(WorkerToken wt) { return Utils.deserialize(wt.get_info(), WorkerTokenInfo.class); } - //Support for worker tokens Similar to an IAutoCredentials implementation + // Support for worker tokens Similar to an IAutoCredentials implementation private static Subject insertWorkerTokens(Subject subject, Map credentials) { if (credentials == null) { return subject; @@ -413,14 +432,16 @@ private static Subject insertWorkerTokens(Subject subject, Map c boolean notAlreadyContained = creds.add(token); if (notAlreadyContained) { if (previous != null) { - //this means token is not equal to previous so we should remove previous + // this means token is not equal to previous so we should remove + // previous creds.remove(previous); LOG.info("Replaced WorkerToken for service type {}", type); } else { LOG.info("Added new WorkerToken for service type {}", type); } } else { - LOG.info("The new WorkerToken for service type {} is the same as the previous token", type); + LOG.info("The new WorkerToken for service type {} is the same as the " + + "previous token", type); } } } @@ -436,7 +457,8 @@ private static Subject insertWorkerTokens(Subject subject, Map c * @param credentials the credentials to pull from * @return the populated subject. */ - public static Subject populateSubject(Subject subject, Collection autos, Map credentials) { + public static Subject populateSubject(Subject subject, Collection autos, + Map credentials) { try { if (subject == null) { subject = new Subject(); @@ -457,9 +479,11 @@ public static Subject populateSubject(Subject subject, Collection autos, Map credentials) { + public static void updateSubject(Subject subject, Collection autos, + Map credentials) { if (subject == null || autos == null) { - throw new RuntimeException("The subject or auto credentials cannot be null when updating a subject with credentials"); + throw new RuntimeException("The subject or auto credentials cannot be null when " + + "updating a subject with credentials"); } try { @@ -475,10 +499,12 @@ public static void updateSubject(Subject subject, Collection a /** * Construct a transport plugin per storm configuration. */ - public static ITransportPlugin getTransportPlugin(ThriftConnectionType type, Map topoConf) { + public static ITransportPlugin getTransportPlugin(ThriftConnectionType type, Map topoConf) { try { String transportPluginClassName = type.getTransportPlugin(topoConf); - ITransportPlugin transportPlugin = ReflectionUtils.newInstance(transportPluginClassName); + ITransportPlugin transportPlugin = ReflectionUtils + .newInstance(transportPluginClassName); transportPlugin.prepare(type, topoConf); return transportPlugin; } catch (Exception e) { @@ -511,8 +537,6 @@ public static String makeDigestPayload(Map topoConf, String conf } } - - public static byte[] serializeKerberosTicket(final KerberosTicket tgt) throws Exception { if (tgt == null) { throw new IllegalArgumentException("KerberosTicket must not be null"); diff --git a/storm-client/src/jvm/org/apache/storm/security/auth/DefaultPrincipalToLocal.java b/storm-client/src/jvm/org/apache/storm/security/auth/DefaultPrincipalToLocal.java index 40cf3a58c1e..fe1d9dab975 100644 --- a/storm-client/src/jvm/org/apache/storm/security/auth/DefaultPrincipalToLocal.java +++ b/storm-client/src/jvm/org/apache/storm/security/auth/DefaultPrincipalToLocal.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -15,7 +21,8 @@ import java.util.Map; /** - * Storm can be configured to launch worker processed as a given user. Some transports need to map the Principal to a local user name. + * Storm can be configured to launch worker processed as a given user. Some transports need to map + * the Principal to a local user name. */ public class DefaultPrincipalToLocal implements IPrincipalToLocal { /** diff --git a/storm-client/src/jvm/org/apache/storm/security/auth/FixedGroupsMapping.java b/storm-client/src/jvm/org/apache/storm/security/auth/FixedGroupsMapping.java index 7456cbdbfb9..b3b761a6cd8 100644 --- a/storm-client/src/jvm/org/apache/storm/security/auth/FixedGroupsMapping.java +++ b/storm-client/src/jvm/org/apache/storm/security/auth/FixedGroupsMapping.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

    - * http://www.apache.org/licenses/LICENSE-2.0 - *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

    http://www.apache.org/licenses/LICENSE-2.0 + * + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -21,7 +27,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; - public class FixedGroupsMapping implements IGroupMappingServiceProvider { public static final String STORM_FIXED_GROUP_MAPPING = "storm.fixed.group.mapping"; @@ -36,7 +41,8 @@ public class FixedGroupsMapping implements IGroupMappingServiceProvider { @Override public void prepare(Map stormConf) { Map params = (Map) stormConf.get(Config.STORM_GROUP_MAPPING_SERVICE_PARAMS); - Map> mapping = (Map>) params.get(STORM_FIXED_GROUP_MAPPING); + Map> mapping = (Map>) params + .get(STORM_FIXED_GROUP_MAPPING); if (mapping != null) { cachedGroups.putAll(mapping); } else { diff --git a/storm-client/src/jvm/org/apache/storm/security/auth/IAuthorizer.java b/storm-client/src/jvm/org/apache/storm/security/auth/IAuthorizer.java index fd244c0d396..45f66c76458 100644 --- a/storm-client/src/jvm/org/apache/storm/security/auth/IAuthorizer.java +++ b/storm-client/src/jvm/org/apache/storm/security/auth/IAuthorizer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -15,7 +21,8 @@ import java.util.Map; /** - * Nimbus could be configured with an authorization plugin. If not specified, all requests are authorized. + * Nimbus could be configured with an authorization plugin. If not specified, all requests are + * authorized. * *

    You could specify the authorization plugin via storm parameter. For example: * diff --git a/storm-client/src/jvm/org/apache/storm/security/auth/IAutoCredentials.java b/storm-client/src/jvm/org/apache/storm/security/auth/IAutoCredentials.java index 3497279b3bc..4a5cac9870f 100644 --- a/storm-client/src/jvm/org/apache/storm/security/auth/IAutoCredentials.java +++ b/storm-client/src/jvm/org/apache/storm/security/auth/IAutoCredentials.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -16,7 +22,8 @@ import javax.security.auth.Subject; /** - * Provides a way to automatically push credentials to a topology and to retrieve them in the worker. + * Provides a way to automatically push credentials to a topology and to retrieve them in the + * worker. */ public interface IAutoCredentials { @@ -37,9 +44,9 @@ public interface IAutoCredentials { */ void populateSubject(Subject subject, Map credentials); - /** - * Called to update the subject on the worker side when new credentials are received. This means that populateSubject has already been + * Called to update the subject on the worker side when new credentials are received. This means + * that populateSubject has already been * called on this subject. * * @param subject the subject to optionally put credentials in. diff --git a/storm-client/src/jvm/org/apache/storm/security/auth/ICredentialsRenewer.java b/storm-client/src/jvm/org/apache/storm/security/auth/ICredentialsRenewer.java index 09173b74d6e..db122c79b45 100644 --- a/storm-client/src/jvm/org/apache/storm/security/auth/ICredentialsRenewer.java +++ b/storm-client/src/jvm/org/apache/storm/security/auth/ICredentialsRenewer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -33,5 +39,6 @@ public interface ICredentialsRenewer { * @param topologyConf topology configuration. * @param topologyOwnerPrincipal the full principal name of the owner of the topology */ - void renew(Map credentials, Map topologyConf, String topologyOwnerPrincipal); + void renew(Map credentials, Map topologyConf, + String topologyOwnerPrincipal); } diff --git a/storm-client/src/jvm/org/apache/storm/security/auth/IGroupMappingServiceProvider.java b/storm-client/src/jvm/org/apache/storm/security/auth/IGroupMappingServiceProvider.java index 4bf3b0e2a59..f371decbcc9 100644 --- a/storm-client/src/jvm/org/apache/storm/security/auth/IGroupMappingServiceProvider.java +++ b/storm-client/src/jvm/org/apache/storm/security/auth/IGroupMappingServiceProvider.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -26,7 +32,8 @@ public interface IGroupMappingServiceProvider { void prepare(Map topoConf); /** - * Get all various group memberships of a given user. Returns EMPTY list in case of non-existing user. + * Get all various group memberships of a given user. Returns EMPTY list in case of non-existing + * user. * * @param user User's name * @return group memberships of user diff --git a/storm-client/src/jvm/org/apache/storm/security/auth/IPrincipalToLocal.java b/storm-client/src/jvm/org/apache/storm/security/auth/IPrincipalToLocal.java index 3163cdb2500..26765d34919 100644 --- a/storm-client/src/jvm/org/apache/storm/security/auth/IPrincipalToLocal.java +++ b/storm-client/src/jvm/org/apache/storm/security/auth/IPrincipalToLocal.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -16,7 +22,8 @@ import java.util.Map; /** - * Storm can be configured to launch worker processed as a given user. Some transports need to map the Principal to a local user name. + * Storm can be configured to launch worker processed as a given user. Some transports need to map + * the Principal to a local user name. */ public interface IPrincipalToLocal { /** diff --git a/storm-client/src/jvm/org/apache/storm/security/auth/ITransportPlugin.java b/storm-client/src/jvm/org/apache/storm/security/auth/ITransportPlugin.java index d927e8fb4cf..131c8e1fb05 100644 --- a/storm-client/src/jvm/org/apache/storm/security/auth/ITransportPlugin.java +++ b/storm-client/src/jvm/org/apache/storm/security/auth/ITransportPlugin.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -45,13 +51,17 @@ public interface ITransportPlugin { * * @param transport The underlying Thrift transport. * @param serverHost server host - * @param asUser the user as which the connection should be established, and all the subsequent actions should be executed. Only - * applicable when using secure storm cluster. A null/blank value here will just indicate to use the logged in user. + * @param asUser the user as which the connection should be established, and all the subsequent + * actions should be executed. Only + * applicable when using secure storm cluster. A null/blank value here will just indicate to + * use the logged in user. */ - TTransport connect(TTransport transport, String serverHost, String asUser) throws IOException, TTransportException; + TTransport connect(TTransport transport, String serverHost, + String asUser) throws IOException, TTransportException; /** * Get port. + * * @return The port this transport is using. This is not known until * {@link #getServer(org.apache.storm.thrift.TProcessor)} has been called */ diff --git a/storm-client/src/jvm/org/apache/storm/security/auth/KerberosPrincipalToLocal.java b/storm-client/src/jvm/org/apache/storm/security/auth/KerberosPrincipalToLocal.java index 5ca3865cb95..94da26832cf 100644 --- a/storm-client/src/jvm/org/apache/storm/security/auth/KerberosPrincipalToLocal.java +++ b/storm-client/src/jvm/org/apache/storm/security/auth/KerberosPrincipalToLocal.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -36,7 +42,7 @@ public void prepare(Map topoConf) { */ @Override public String toLocal(String principal) { - //This technically does not conform with rfc1964, but should work so + // This technically does not conform with rfc1964, but should work so // long as you don't have any really odd names in your KDC. return principal == null ? null : principal.split("[/@]")[0]; } diff --git a/storm-client/src/jvm/org/apache/storm/security/auth/MultiThriftServer.java b/storm-client/src/jvm/org/apache/storm/security/auth/MultiThriftServer.java index 514685645be..4c47a5f07ce 100644 --- a/storm-client/src/jvm/org/apache/storm/security/auth/MultiThriftServer.java +++ b/storm-client/src/jvm/org/apache/storm/security/auth/MultiThriftServer.java @@ -1,12 +1,17 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -30,23 +35,25 @@ public class MultiThriftServer { private final Map thriftServerThreadMap = new HashMap<>(); private final Map thriftServerIsServingMap = new HashMap<>(); - public MultiThriftServer(String name) { this.name = name; } public void add(T thriftServer) { thriftServerMap.put(thriftServer.getType(), thriftServer); - thriftServerThreadMap.put(thriftServer.getType(), new Thread(thriftServer::serve, name + "-" + thriftServer.getPort())); + thriftServerThreadMap.put(thriftServer.getType(), new Thread(thriftServer::serve, name + + "-" + thriftServer.getPort())); } public void serve() { for (ThriftServer thriftServer : thriftServerMap.values()) { if (Boolean.TRUE.equals(thriftServerIsServingMap.get(thriftServer.getType()))) { throw new IllegalStateException("The MultiThriftServer " - + thriftServerThreadMap.get(thriftServer.getType()).getName() + " is already serving"); + + thriftServerThreadMap.get(thriftServer.getType()).getName() + + " is already serving"); } - LOG.info("Starting thrift server {}", thriftServerThreadMap.get(thriftServer.getType()).getName()); + LOG.info("Starting thrift server {}", thriftServerThreadMap.get(thriftServer.getType()) + .getName()); thriftServerThreadMap.get(thriftServer.getType()).start(); thriftServerIsServingMap.put(thriftServer.getType(), true); } @@ -59,7 +66,8 @@ public void stop() { thriftServerMap.get(thriftServer.getType()).stop(); thriftServerIsServingMap.put(thriftServer.getType(), false); } else { - LOG.warn("Can't stop the " + thriftServerThreadMap.get(thriftServer.getType()).getName() + LOG.warn("Can't stop the " + thriftServerThreadMap.get(thriftServer.getType()) + .getName() + " server since it is not currently serving"); } } @@ -78,6 +86,7 @@ public void stopTlsServer(ThriftConnectionType tlsConnectionType) { /** * Check if worker tokens are supported by any one of the thrift servers. + * * @return true if any thrift server supports Worker Tokens. */ public boolean supportsWorkerTokens() { diff --git a/storm-client/src/jvm/org/apache/storm/security/auth/NimbusPrincipal.java b/storm-client/src/jvm/org/apache/storm/security/auth/NimbusPrincipal.java index f21a81ecf77..d6e16912636 100644 --- a/storm-client/src/jvm/org/apache/storm/security/auth/NimbusPrincipal.java +++ b/storm-client/src/jvm/org/apache/storm/security/auth/NimbusPrincipal.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/security/auth/ReqContext.java b/storm-client/src/jvm/org/apache/storm/security/auth/ReqContext.java index 3e61daf4430..d38b43c85b4 100644 --- a/storm-client/src/jvm/org/apache/storm/security/auth/ReqContext.java +++ b/storm-client/src/jvm/org/apache/storm/security/auth/ReqContext.java @@ -36,7 +36,7 @@ public class ReqContext { private static final AtomicInteger uniqueId = new AtomicInteger(0); - //each thread will have its own request context + // each thread will have its own request context private static final ThreadLocal ctxt = ThreadLocal.withInitial(ReqContext::new); private Subject subject; @@ -44,14 +44,14 @@ public class ReqContext { private final int reqId; private Principal realPrincipal; - //private constructor + // private constructor @VisibleForTesting public ReqContext() { subject = currentSubject(); reqId = uniqueId.incrementAndGet(); } - //private constructor + // private constructor @VisibleForTesting public ReqContext(Subject sub) { subject = sub; @@ -71,6 +71,7 @@ public ReqContext(ReqContext other) { /** * Get context. + * * @return a request context associated with current thread */ public static ReqContext context() { @@ -96,7 +97,7 @@ public String toString() { } /** - * client address. + * Client address. */ public void setRemoteAddress(InetAddress addr) { remoteAddr = addr; @@ -147,6 +148,7 @@ public Principal realPrincipal() { /** * Check whether context is impersonating. + * * @return true if this request is an impersonation request. */ public boolean isImpersonating() { @@ -154,7 +156,7 @@ public boolean isImpersonating() { } /** - * request ID of this request. + * Request ID of this request. */ @SuppressWarnings("checkstyle:AbbreviationAsWordInName") public int requestID() { @@ -163,6 +165,7 @@ public int requestID() { /** * Maps to Subject.current() if available, otherwise maps to Subject.getSubject(). + * * @return the current subject * @see SubjectCompat#currentSubject() */ diff --git a/storm-client/src/jvm/org/apache/storm/security/auth/ShellBasedGroupsMapping.java b/storm-client/src/jvm/org/apache/storm/security/auth/ShellBasedGroupsMapping.java index c80e293d03f..47717c72ffb 100644 --- a/storm-client/src/jvm/org/apache/storm/security/auth/ShellBasedGroupsMapping.java +++ b/storm-client/src/jvm/org/apache/storm/security/auth/ShellBasedGroupsMapping.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -23,13 +29,12 @@ import org.apache.storm.utils.RotatingMap; import org.apache.storm.utils.ShellCommandRunner; import org.apache.storm.utils.ShellCommandRunnerImpl; -import org.apache.storm.utils.ShellUtils; import org.apache.storm.utils.ShellUtils.ExitCodeException; +import org.apache.storm.utils.ShellUtils; import org.apache.storm.utils.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - public class ShellBasedGroupsMapping implements IGroupMappingServiceProvider { @@ -55,7 +60,8 @@ public ShellBasedGroupsMapping() { */ @Override public void prepare(Map topoConf) { - timeoutMs = TimeUnit.SECONDS.toMillis(ObjectReader.getInt(topoConf.get(Config.STORM_GROUP_MAPPING_SERVICE_CACHE_DURATION_SECS))); + timeoutMs = TimeUnit.SECONDS.toMillis(ObjectReader.getInt(topoConf + .get(Config.STORM_GROUP_MAPPING_SERVICE_CACHE_DURATION_SECS))); lastRotationMs = Time.currentTimeMillis(); cachedGroups = new RotatingMap<>(2); } @@ -86,8 +92,8 @@ public Set getGroups(String user) throws IOException { private void rotateIfNeeded() { long nowMs = Time.currentTimeMillis(); if (nowMs >= lastRotationMs + timeoutMs) { - //Rotate once per timeout period that has passed since last time this was called. - //This is necessary since this method may be called at arbitrary intervals. + // Rotate once per timeout period that has passed since last time this was called. + // This is necessary since this method may be called at arbitrary intervals. int rotationsToDo = (int) ((nowMs - lastRotationMs) / timeoutMs); for (int i = 0; i < rotationsToDo; i++) { cachedGroups.rotate(); @@ -97,7 +103,8 @@ private void rotateIfNeeded() { } /** - * Get the current user's group list from Unix by running the command 'groups' NOTE. For non-existing user it will return EMPTY list + * Get the current user's group list from Unix by running the command 'groups' NOTE. For + * non-existing user it will return EMPTY list * * @param user user name * @return the groups set that the user belongs to @@ -115,7 +122,8 @@ private Set getUnixGroups(final String user) throws IOException { result = shellCommandRunner.execCommand(ShellUtils.getGroupsForUserCommand(user)); } catch (ExitCodeException e) { // if we didn't get the group - just return empty list; - LOG.debug("Unable to get groups for user " + user + ". ShellUtils command failed with exit code " + e.getExitCode()); + LOG.debug("Unable to get groups for user " + user + + ". ShellUtils command failed with exit code " + e.getExitCode()); return new HashSet<>(); } diff --git a/storm-client/src/jvm/org/apache/storm/security/auth/SimpleTransportPlugin.java b/storm-client/src/jvm/org/apache/storm/security/auth/SimpleTransportPlugin.java index f52aacf26c8..b3aae144c4c 100644 --- a/storm-client/src/jvm/org/apache/storm/security/auth/SimpleTransportPlugin.java +++ b/storm-client/src/jvm/org/apache/storm/security/auth/SimpleTransportPlugin.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -79,7 +85,7 @@ public TServer getServer(TProcessor processor) throws IOException, TTransportExc 60, TimeUnit.SECONDS, new ArrayBlockingQueue(queueSize))); } - //construct THsHaServer + // construct THsHaServer return new THsHaServer(serverArgs); } @@ -91,12 +97,13 @@ public TServer getServer(TProcessor processor) throws IOException, TTransportExc * @param asUser unused */ @Override - public TTransport connect(TTransport transport, String serverHost, String asUser) throws TTransportException { + public TTransport connect(TTransport transport, String serverHost, + String asUser) throws TTransportException { int maxBufferSize = type.getMaxBufferSize(topoConf); - //create a framed transport + // create a framed transport TTransport conn = new TFramedTransport(transport, maxBufferSize); - //connect + // connect conn.open(); LOG.debug("Simple client transport has been established"); @@ -105,6 +112,7 @@ public TTransport connect(TTransport transport, String serverHost, String asUser /** * Get default subject. + * * @return the subject that will be used for all connections */ protected Subject getDefaultSubject() { @@ -117,7 +125,8 @@ public int getPort() { } /** - * Processor that populate simple transport info into ReqContext, and then invoke a service handler. + * Processor that populate simple transport info into ReqContext, and then invoke a service + * handler. */ private class SimpleWrapProcessor implements TProcessor { final TProcessor wrapped; @@ -128,7 +137,7 @@ private class SimpleWrapProcessor implements TProcessor { @Override public void process(final TProtocol inProt, final TProtocol outProt) throws TException { - //populating request context + // populating request context ReqContext reqContext = ReqContext.context(); TTransport trans = inProt.getTransport(); @@ -140,12 +149,12 @@ public void process(final TProtocol inProt, final TProtocol outProt) throws TExc } } else if (trans instanceof TSocket) { TSocket tsocket = (TSocket) trans; - //remote address + // remote address Socket socket = tsocket.getSocket(); reqContext.setRemoteAddress(socket.getInetAddress()); } - //anonymous user + // anonymous user Subject s = getDefaultSubject(); if (s == null) { final String user = (String) topoConf.get("debug.simple.transport.user"); @@ -167,7 +176,7 @@ public String toString() { } reqContext.setSubject(s); - //invoke service handler + // invoke service handler wrapped.process(inProt, outProt); } } diff --git a/storm-client/src/jvm/org/apache/storm/security/auth/SingleUserPrincipal.java b/storm-client/src/jvm/org/apache/storm/security/auth/SingleUserPrincipal.java index 2885103aa08..da7739d186e 100644 --- a/storm-client/src/jvm/org/apache/storm/security/auth/SingleUserPrincipal.java +++ b/storm-client/src/jvm/org/apache/storm/security/auth/SingleUserPrincipal.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -27,7 +33,8 @@ public SingleUserPrincipal(String userName) { @Override public boolean equals(Object another) { - return another instanceof SingleUserPrincipal && userName.equals(((SingleUserPrincipal) another).userName); + return another instanceof SingleUserPrincipal && userName + .equals(((SingleUserPrincipal) another).userName); } @Override diff --git a/storm-client/src/jvm/org/apache/storm/security/auth/SubjectCompat.java b/storm-client/src/jvm/org/apache/storm/security/auth/SubjectCompat.java index a54541373a0..4bbee4bd6c9 100644 --- a/storm-client/src/jvm/org/apache/storm/security/auth/SubjectCompat.java +++ b/storm-client/src/jvm/org/apache/storm/security/auth/SubjectCompat.java @@ -33,7 +33,8 @@ *

  • {@code Subject.doAs(Subject, PrivilegedExceptionAction)} → {@code Subject.callAs(Subject, Callable)} (Java 18+)
  • * * - *

    All dispatch is resolved once at class-init via {@link MethodHandle}, so there is no per-call reflection overhead. + *

    All dispatch is resolved once at class-init via {@link MethodHandle}, so there is no per-call + * reflection overhead. */ public final class SubjectCompat { diff --git a/storm-client/src/jvm/org/apache/storm/security/auth/TBackoffConnect.java b/storm-client/src/jvm/org/apache/storm/security/auth/TBackoffConnect.java index 5b09aac07fa..346a314ea56 100644 --- a/storm-client/src/jvm/org/apache/storm/security/auth/TBackoffConnect.java +++ b/storm-client/src/jvm/org/apache/storm/security/auth/TBackoffConnect.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -26,7 +32,8 @@ public class TBackoffConnect { private StormBoundedExponentialBackoffRetry waitGrabber; private boolean retryForever = false; - public TBackoffConnect(int retryTimes, int retryInterval, int retryIntervalCeiling, boolean retryForever) { + public TBackoffConnect(int retryTimes, int retryInterval, int retryIntervalCeiling, + boolean retryForever) { this.retryForever = retryForever; this.retryTimes = retryTimes; @@ -39,7 +46,8 @@ public TBackoffConnect(int retryTimes, int retryInterval, int retryIntervalCeili this(retryTimes, retryInterval, retryIntervalCeiling, false); } - public TTransport doConnectWithRetry(ITransportPlugin transportPlugin, TTransport underlyingTransport, String host, + public TTransport doConnectWithRetry(ITransportPlugin transportPlugin, + TTransport underlyingTransport, String host, String asUser) throws IOException { boolean connected = false; TTransport transportResult = null; @@ -61,7 +69,8 @@ private void retryNext(TTransportException ex) { try { long sleeptime = waitGrabber.getSleepTimeMs(completedRetries, 0); - LOG.debug("Failed to connect. Retrying... (" + Integer.toString(completedRetries) + ") in " + Long.toString(sleeptime) + "ms"); + LOG.debug("Failed to connect. Retrying... (" + Integer.toString(completedRetries) + + ") in " + Long.toString(sleeptime) + "ms"); Thread.sleep(sleeptime); } catch (InterruptedException e) { diff --git a/storm-client/src/jvm/org/apache/storm/security/auth/ThriftClient.java b/storm-client/src/jvm/org/apache/storm/security/auth/ThriftClient.java index f7becc3f385..7f2a96d30e9 100644 --- a/storm-client/src/jvm/org/apache/storm/security/auth/ThriftClient.java +++ b/storm-client/src/jvm/org/apache/storm/security/auth/ThriftClient.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -27,7 +33,6 @@ import java.security.cert.X509Certificate; import java.util.Map; import java.util.UUID; - import org.apache.storm.Config; import org.apache.storm.thrift.protocol.TBinaryProtocol; import org.apache.storm.thrift.protocol.TProtocol; @@ -60,13 +65,15 @@ public ThriftClient(Map topoConf, ThriftConnectionType type, Str this(topoConf, type, host, null, null, null); } - public ThriftClient(Map topoConf, ThriftConnectionType type, String host, Integer port, Integer timeout) { + public ThriftClient(Map topoConf, ThriftConnectionType type, String host, + Integer port, Integer timeout) { this(topoConf, type, host, port, timeout, null); } - public ThriftClient(Map topoConf, ThriftConnectionType type, String host, Integer port, Integer timeout, + public ThriftClient(Map topoConf, ThriftConnectionType type, String host, + Integer port, Integer timeout, String asUser) { - //create a socket with server + // create a socket with server if (host == null) { throw new IllegalArgumentException("host is not set"); } @@ -100,6 +107,7 @@ public synchronized TTransport transport() { /** * Get the private key using BouncyCastle library from a PKCS#1 format file. + * * @return The Private Key * @throws IOException The IOException */ @@ -115,6 +123,7 @@ protected PrivateKey getPrivateKey() throws IOException { /** * Function to create a keystore and return the keystore file. + * * @param keyStorePass The keystore password. * @return The keystore file * @throws CertificateException CertificateException @@ -156,26 +165,35 @@ public synchronized void reconnect() { LOG.debug("Tls is enabled"); TSSLTransportFactory.TSSLTransportParameters params = new TSSLTransportFactory.TSSLTransportParameters(); - if (type.getClientTrustStorePath(conf) != null && type.getClientTrustStorePassword(conf) != null) { + if (type.getClientTrustStorePath(conf) != null && type + .getClientTrustStorePassword(conf) != null) { // override the keyStoreType, there is no direct setter method - params.setTrustStore(type.getClientTrustStorePath(conf), type.getClientTrustStorePassword(conf), null, - SecurityUtils.inferKeyStoreTypeFromPath(type.getClientTrustStorePath(conf))); + params.setTrustStore(type.getClientTrustStorePath(conf), type + .getClientTrustStorePassword(conf), null, + SecurityUtils.inferKeyStoreTypeFromPath(type + .getClientTrustStorePath(conf))); } else { - throw new IllegalArgumentException("The client truststore is not configured properly"); + throw new IllegalArgumentException("The client truststore is not configured " + + "properly"); } if (type.isClientAuthRequired(conf)) { - if (type.getClientKeyPath(conf) != null && type.getClientCertPath(conf) != null) { + if (type.getClientKeyPath(conf) != null && type + .getClientCertPath(conf) != null) { String keyStorePass = UUID.randomUUID().toString(); file = getKeyStoreFile(keyStorePass); params.setKeyStore(file.getAbsolutePath(), keyStorePass, null, SecurityUtils.inferKeyStoreTypeFromPath(file.getAbsolutePath())); - } else if (type.getClientKeyStorePath(conf) != null && type.getClientKeyStorePassword(conf) != null) { + } else if (type.getClientKeyStorePath(conf) != null && type + .getClientKeyStorePassword(conf) != null) { // override the keyStoreType, there is no direct setter method - params.setKeyStore(type.getClientKeyStorePath(conf), type.getClientKeyStorePassword(conf), null, - SecurityUtils.inferKeyStoreTypeFromPath(type.getClientKeyStorePath(conf))); + params.setKeyStore(type.getClientKeyStorePath(conf), type + .getClientKeyStorePassword(conf), null, + SecurityUtils.inferKeyStoreTypeFromPath(type + .getClientKeyStorePath(conf))); } else { - throw new IllegalArgumentException("The client credentials are not configured properly"); + throw new IllegalArgumentException("The client credentials are not " + + "configured properly"); } } socket = TSSLTransportFactory.getClientSocket(host, port, 0, params); @@ -186,12 +204,12 @@ public synchronized void reconnect() { socket.setTimeout(timeout); } - //construct a transport plugin + // construct a transport plugin ITransportPlugin transportPlugin = ClientAuthUtils.getTransportPlugin(type, conf); - //TODO: get this from type instead of hardcoding to Nimbus. - //establish client-server transport via plugin - //do retries if the connect fails + // TODO: get this from type instead of hardcoding to Nimbus. + // establish client-server transport via plugin + // do retries if the connect fails TBackoffConnect connectionRetry = new TBackoffConnect( ObjectReader.getInt(conf.get(Config.STORM_NIMBUS_RETRY_TIMES)), @@ -205,7 +223,7 @@ public synchronized void reconnect() { try { socket.close(); } catch (Exception e) { - //ignore + // ignore } } if (file != null) { diff --git a/storm-client/src/jvm/org/apache/storm/security/auth/ThriftConnectionType.java b/storm-client/src/jvm/org/apache/storm/security/auth/ThriftConnectionType.java index a969e68661d..ad91784edac 100644 --- a/storm-client/src/jvm/org/apache/storm/security/auth/ThriftConnectionType.java +++ b/storm-client/src/jvm/org/apache/storm/security/auth/ThriftConnectionType.java @@ -1,12 +1,17 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -21,10 +26,12 @@ * The purpose for which the Thrift server is created. */ public enum ThriftConnectionType { - NIMBUS(Config.NIMBUS_THRIFT_TRANSPORT_PLUGIN, Config.NIMBUS_THRIFT_PORT, Config.NIMBUS_QUEUE_SIZE, + NIMBUS(Config.NIMBUS_THRIFT_TRANSPORT_PLUGIN, Config.NIMBUS_THRIFT_PORT, + Config.NIMBUS_QUEUE_SIZE, Config.NIMBUS_THRIFT_THREADS, Config.NIMBUS_THRIFT_MAX_BUFFER_SIZE, Config.STORM_THRIFT_SOCKET_TIMEOUT_MS, WorkerTokenServiceType.NIMBUS, true), - NIMBUS_TLS(Config.NIMBUS_THRIFT_TLS_TRANSPORT_PLUGIN, Config.NIMBUS_THRIFT_TLS_PORT, Config.NIMBUS_QUEUE_SIZE, + NIMBUS_TLS(Config.NIMBUS_THRIFT_TLS_TRANSPORT_PLUGIN, Config.NIMBUS_THRIFT_TLS_PORT, + Config.NIMBUS_QUEUE_SIZE, Config.NIMBUS_THRIFT_TLS_THREADS, Config.NIMBUS_THRIFT_TLS_MAX_BUFFER_SIZE, Config.STORM_THRIFT_TLS_SOCKET_TIMEOUT_MS, false, null, false, true, Config.NIMBUS_THRIFT_TLS_CLIENT_AUTH_REQUIRED, @@ -33,10 +40,12 @@ public enum ThriftConnectionType { Config.NIMBUS_THRIFT_TLS_CLIENT_KEYSTORE_PATH, Config.NIMBUS_THRIFT_TLS_CLIENT_KEYSTORE_PASSWORD, Config.NIMBUS_THRIFT_TLS_CLIENT_TRUSTSTORE_PATH, Config.NIMBUS_THRIFT_TLS_CLIENT_TRUSTSTORE_PASSWORD, Config.NIMBUS_THRIFT_TLS_CLIENT_KEY_PATH, Config.NIMBUS_THRIFT_TLS_CLIENT_CERT_PATH), - SUPERVISOR(Config.SUPERVISOR_THRIFT_TRANSPORT_PLUGIN, Config.SUPERVISOR_THRIFT_PORT, Config.SUPERVISOR_QUEUE_SIZE, + SUPERVISOR(Config.SUPERVISOR_THRIFT_TRANSPORT_PLUGIN, Config.SUPERVISOR_THRIFT_PORT, + Config.SUPERVISOR_QUEUE_SIZE, Config.SUPERVISOR_THRIFT_THREADS, Config.SUPERVISOR_THRIFT_MAX_BUFFER_SIZE, Config.SUPERVISOR_THRIFT_SOCKET_TIMEOUT_MS, WorkerTokenServiceType.SUPERVISOR, false), - SUPERVISOR_TLS(Config.SUPERVISOR_THRIFT_TRANSPORT_PLUGIN, Config.SUPERVISOR_THRIFT_PORT, Config.SUPERVISOR_QUEUE_SIZE, + SUPERVISOR_TLS(Config.SUPERVISOR_THRIFT_TRANSPORT_PLUGIN, Config.SUPERVISOR_THRIFT_PORT, + Config.SUPERVISOR_QUEUE_SIZE, Config.SUPERVISOR_THRIFT_THREADS, Config.SUPERVISOR_THRIFT_MAX_BUFFER_SIZE, Config.SUPERVISOR_THRIFT_SOCKET_TIMEOUT_MS, false, null, false, true, Config.SUPERVISOR_THRIFT_TLS_CLIENT_AUTH_REQUIRED, @@ -45,10 +54,11 @@ public enum ThriftConnectionType { Config.SUPERVISOR_THRIFT_TLS_CLIENT_KEYSTORE_PATH, Config.SUPERVISOR_THRIFT_TLS_CLIENT_KEYSTORE_PASSWORD, Config.SUPERVISOR_THRIFT_TLS_CLIENT_TRUSTSTORE_PATH, Config.SUPERVISOR_THRIFT_TLS_CLIENT_TRUSTSTORE_PASSWORD, Config.SUPERVISOR_THRIFT_TLS_CLIENT_KEY_PATH, Config.SUPERVISOR_THRIFT_TLS_CLIENT_CERT_PATH), - //A DRPC token only works for the invocations transport, not for the basic thrift transport. + // A DRPC token only works for the invocations transport, not for the basic thrift transport. DRPC(Config.DRPC_THRIFT_TRANSPORT_PLUGIN, Config.DRPC_PORT, Config.DRPC_QUEUE_SIZE, Config.DRPC_WORKER_THREADS, Config.DRPC_MAX_BUFFER_SIZE, null, null, false), - DRPC_INVOCATIONS(Config.DRPC_INVOCATIONS_THRIFT_TRANSPORT_PLUGIN, Config.DRPC_INVOCATIONS_PORT, null, + DRPC_INVOCATIONS(Config.DRPC_INVOCATIONS_THRIFT_TRANSPORT_PLUGIN, Config.DRPC_INVOCATIONS_PORT, + null, Config.DRPC_INVOCATIONS_THREADS, Config.DRPC_MAX_BUFFER_SIZE, null, WorkerTokenServiceType.DRPC, false), LOCAL_FAKE; @@ -81,13 +91,15 @@ public enum ThriftConnectionType { ThriftConnectionType(String transConf, String portConf, String queueConf, String threadsConf, String buffConf, String socketTimeoutConf, WorkerTokenServiceType wtType, boolean impersonationAllowed) { - this(transConf, portConf, queueConf, threadsConf, buffConf, socketTimeoutConf, false, wtType, impersonationAllowed); + this(transConf, portConf, queueConf, threadsConf, buffConf, socketTimeoutConf, false, + wtType, impersonationAllowed); } ThriftConnectionType(String transConf, String portConf, String queueConf, String threadsConf, String buffConf, String socketTimeoutConf, boolean isFake, WorkerTokenServiceType wtType, boolean impersonationAllowed) { - this(transConf, portConf, queueConf, threadsConf, buffConf, socketTimeoutConf, isFake, wtType, impersonationAllowed, + this(transConf, portConf, queueConf, threadsConf, buffConf, socketTimeoutConf, isFake, + wtType, impersonationAllowed, false, null, null, null, null, null, null, null, null, null, null, null); } diff --git a/storm-client/src/jvm/org/apache/storm/security/auth/ThriftServer.java b/storm-client/src/jvm/org/apache/storm/security/auth/ThriftServer.java index 5c21ce98ecd..81d97cbb62d 100644 --- a/storm-client/src/jvm/org/apache/storm/security/auth/ThriftServer.java +++ b/storm-client/src/jvm/org/apache/storm/security/auth/ThriftServer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -24,7 +30,7 @@ public class ThriftServer { private static final Logger LOG = LoggerFactory.getLogger(ThriftServer.class); protected final TProcessor processor; - private final Map conf; //storm configuration + private final Map conf; // storm configuration private final ThriftConnectionType type; private TServer server; private int port; @@ -37,9 +43,9 @@ public ThriftServer(Map conf, TProcessor processor, ThriftConnec this.type = type; try { - //locate our thrift transport plugin + // locate our thrift transport plugin transportPlugin = ClientAuthUtils.getTransportPlugin(this.type, this.conf); - //server + // server server = transportPlugin.getServer(this.processor); port = transportPlugin.getPort(); areWorkerTokensSupported = transportPlugin.areWorkerTokensSupported(); @@ -58,6 +64,7 @@ public void stop() { /** * Check whether serving. + * * @return true if ThriftServer is listening to requests? */ public boolean isServing() { @@ -66,7 +73,7 @@ public boolean isServing() { public void serve() { try { - //start accepting requests + // start accepting requests server.serve(); } catch (Exception ex) { handleServerException(ex); @@ -78,11 +85,13 @@ private void handleServerException(Exception ex) { if (server != null) { server.stop(); } - Runtime.getRuntime().halt(1); //shutdown server process since we could not handle Thrift requests any more + Runtime.getRuntime() + .halt(1); // shutdown server process since we could not handle Thrift requests any more } /** * Get port. + * * @return The port this server is/will be listening on */ public int getPort() { @@ -91,6 +100,7 @@ public int getPort() { /** * Get type. + * * @return The type of server */ public ThriftConnectionType getType() { diff --git a/storm-client/src/jvm/org/apache/storm/security/auth/X509CertOrKerberosPrincipalToLocal.java b/storm-client/src/jvm/org/apache/storm/security/auth/X509CertOrKerberosPrincipalToLocal.java index f7ea86e8884..f001df22979 100644 --- a/storm-client/src/jvm/org/apache/storm/security/auth/X509CertOrKerberosPrincipalToLocal.java +++ b/storm-client/src/jvm/org/apache/storm/security/auth/X509CertOrKerberosPrincipalToLocal.java @@ -1,24 +1,29 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.security.auth; import java.util.Map; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class X509CertOrKerberosPrincipalToLocal implements IPrincipalToLocal { - private static final Logger LOG = LoggerFactory.getLogger(X509CertOrKerberosPrincipalToLocal.class); + private static final Logger LOG = LoggerFactory + .getLogger(X509CertOrKerberosPrincipalToLocal.class); X509CertPrincipalToLocal x509CertPrincipalToLocal; KerberosPrincipalToLocal kerberosPrincipalToLocal; @@ -38,16 +43,19 @@ public String toLocal(String principalName) { try { localName = x509CertPrincipalToLocal.toLocal(principalName); LOG.debug("{} translates principal {} to {}", - x509CertPrincipalToLocal.getClass().getCanonicalName(), principalName, localName); + x509CertPrincipalToLocal.getClass() + .getCanonicalName(), principalName, localName); } catch (RuntimeException e) { - //ignore x509CertPrincipalToLocal error. - LOG.debug("Error reading localName from x509CertPrincipalToLocal. The error will be ignored. Error: {}", e.getMessage()); + // ignore x509CertPrincipalToLocal error. + LOG.debug("Error reading localName from x509CertPrincipalToLocal. The error will be " + + "ignored. Error: {}", e.getMessage()); } if (localName == null) { localName = kerberosPrincipalToLocal.toLocal(principalName); LOG.debug("{} translates principal {} to {}", - kerberosPrincipalToLocal.getClass().getCanonicalName(), principalName, localName); + kerberosPrincipalToLocal.getClass() + .getCanonicalName(), principalName, localName); } return localName; diff --git a/storm-client/src/jvm/org/apache/storm/security/auth/X509CertPrincipalToLocal.java b/storm-client/src/jvm/org/apache/storm/security/auth/X509CertPrincipalToLocal.java index 0a8d6fed72d..b5f2f6a8c55 100644 --- a/storm-client/src/jvm/org/apache/storm/security/auth/X509CertPrincipalToLocal.java +++ b/storm-client/src/jvm/org/apache/storm/security/auth/X509CertPrincipalToLocal.java @@ -1,12 +1,17 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -27,7 +32,8 @@ public class X509CertPrincipalToLocal implements IPrincipalToLocal { private static final Logger LOG = LoggerFactory.getLogger(X509CertPrincipalToLocal.class); - public static final String X509_CERT_PRINCIPAL_TO_LOCAL_REGEX = "x509.cert.principal.to.local.regex"; + public static final String X509_CERT_PRINCIPAL_TO_LOCAL_REGEX = + "x509.cert.principal.to.local.regex"; private Pattern pattern; private static String extractCn(final String subjectPrincipal) { @@ -44,14 +50,16 @@ private static String extractCn(final String subjectPrincipal) { } return null; } catch (final InvalidNameException e) { - throw new AccessControlException(subjectPrincipal + " is not a valid X500 distinguished name"); + throw new AccessControlException(subjectPrincipal + + " is not a valid X500 distinguished name"); } } @Override public void prepare(Map conf) { if (conf.get(X509_CERT_PRINCIPAL_TO_LOCAL_REGEX) == null) { - throw new IllegalStateException(X509_CERT_PRINCIPAL_TO_LOCAL_REGEX + " is not configured"); + throw new IllegalStateException(X509_CERT_PRINCIPAL_TO_LOCAL_REGEX + + " is not configured"); } pattern = Pattern.compile(conf.get(X509_CERT_PRINCIPAL_TO_LOCAL_REGEX).toString()); } diff --git a/storm-client/src/jvm/org/apache/storm/security/auth/authorizer/DRPCAuthorizerBase.java b/storm-client/src/jvm/org/apache/storm/security/auth/authorizer/DRPCAuthorizerBase.java index dcb1d48aa24..8aa6c266af8 100644 --- a/storm-client/src/jvm/org/apache/storm/security/auth/authorizer/DRPCAuthorizerBase.java +++ b/storm-client/src/jvm/org/apache/storm/security/auth/authorizer/DRPCAuthorizerBase.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -30,9 +36,11 @@ public abstract class DRPCAuthorizerBase implements IAuthorizer { @Override public abstract void prepare(Map conf); - protected abstract boolean permitClientRequest(ReqContext context, String operation, Map params); + protected abstract boolean permitClientRequest(ReqContext context, String operation, Map params); - protected abstract boolean permitInvocationRequest(ReqContext context, String operation, Map params); + protected abstract boolean permitInvocationRequest(ReqContext context, String operation, + Map params); /** * Authorizes request from to the DRPC server. @@ -51,7 +59,8 @@ public boolean permit(ReqContext context, String operation, Map return permitInvocationRequest(context, operation, params); } // Deny unsupported operations. - LOG.warn("Denying unsupported operation \"" + operation + "\" from " + context.remoteAddress()); + LOG.warn("Denying unsupported operation \"" + operation + "\" from " + context + .remoteAddress()); return false; } } diff --git a/storm-client/src/jvm/org/apache/storm/security/auth/authorizer/DRPCSimpleACLAuthorizer.java b/storm-client/src/jvm/org/apache/storm/security/auth/authorizer/DRPCSimpleACLAuthorizer.java index e45f388c88f..f4351670aa0 100644 --- a/storm-client/src/jvm/org/apache/storm/security/auth/authorizer/DRPCSimpleACLAuthorizer.java +++ b/storm-client/src/jvm/org/apache/storm/security/auth/authorizer/DRPCSimpleACLAuthorizer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -44,15 +50,16 @@ public class DRPCSimpleACLAuthorizer extends DRPCAuthorizerBase { private volatile long lastUpdate = 0; protected Map readAclFromConfig() { - //Thread safety is mostly around acl. If acl needs to be updated it is changed atomically - //More then one thread may be trying to update it at a time, but that is OK, because the - //change is atomic + // Thread safety is mostly around acl. If acl needs to be updated it is changed atomically + // More then one thread may be trying to update it at a time, but that is OK, because the + // change is atomic long now = System.currentTimeMillis(); if ((now - 5000) > lastUpdate || acl == null) { Map acl = new HashMap<>(); Map conf = Utils.findAndReadConfigFile(aclFileName); if (conf.containsKey(Config.DRPC_AUTHORIZER_ACL)) { - Map> confAcl = (Map>) conf.get(Config.DRPC_AUTHORIZER_ACL); + Map> confAcl = (Map>) conf + .get(Config.DRPC_AUTHORIZER_ACL); if (confAcl != null) { for (Map.Entry> entry : confAcl.entrySet()) { @@ -71,7 +78,8 @@ protected Map readAclFromConfig() { this.acl = acl; if (this.acl.isEmpty() && !permitWhenMissingFunctionEntry) { - LOG.warn("Requiring explicit ACL entries, but none given. Therefore, all operations will be denied."); + LOG.warn("Requiring explicit ACL entries, but none given. Therefore, all " + + "operations will be denied."); } lastUpdate = System.currentTimeMillis(); @@ -106,7 +114,8 @@ private String getLocalUserFromContext(ReqContext context) { return null; } - protected boolean permitClientOrInvocationRequest(ReqContext context, Map params, + protected boolean permitClientOrInvocationRequest(ReqContext context, Map params, String fieldName) { Map acl = readAclFromConfig(); String function = (String) params.get(FUNCTION_KEY); @@ -128,7 +137,8 @@ protected boolean permitClientOrInvocationRequest(ReqContext context, Map) value).contains(principal) || ((Set) value).contains(user))) { return true; diff --git a/storm-client/src/jvm/org/apache/storm/security/auth/authorizer/DenyAuthorizer.java b/storm-client/src/jvm/org/apache/storm/security/auth/authorizer/DenyAuthorizer.java index 8ce426a3d01..b62bc24be8c 100644 --- a/storm-client/src/jvm/org/apache/storm/security/auth/authorizer/DenyAuthorizer.java +++ b/storm-client/src/jvm/org/apache/storm/security/auth/authorizer/DenyAuthorizer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/security/auth/authorizer/ImpersonationAuthorizer.java b/storm-client/src/jvm/org/apache/storm/security/auth/authorizer/ImpersonationAuthorizer.java index 22aa4e1669c..9bfe0a4a539 100644 --- a/storm-client/src/jvm/org/apache/storm/security/auth/authorizer/ImpersonationAuthorizer.java +++ b/storm-client/src/jvm/org/apache/storm/security/auth/authorizer/ImpersonationAuthorizer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -29,7 +35,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; - public class ImpersonationAuthorizer implements IAuthorizer { protected static final String WILD_CARD = "*"; private static final Logger LOG = LoggerFactory.getLogger(ImpersonationAuthorizer.class); @@ -46,7 +51,8 @@ public void prepare(Map conf) { (Map>>) conf.get(Config.NIMBUS_IMPERSONATION_ACL); if (userToHostAndGroup != null) { - for (Map.Entry>> entry : userToHostAndGroup.entrySet()) { + for (Map.Entry>> entry : userToHostAndGroup + .entrySet()) { String user = entry.getKey(); Set groups = ImmutableSet.copyOf(entry.getValue().get("groups")); Set hosts = ImmutableSet.copyOf(entry.getValue().get("hosts")); @@ -70,16 +76,20 @@ public boolean permit(ReqContext context, String operation, Map String userBeingImpersonated = ptol.toLocal(context.principal()); InetAddress remoteAddress = context.remoteAddress(); - LOG.info("user = {}, principal = {} is attempting to impersonate user = {} for operation = {} from host = {}", + LOG.info("user = {}, principal = {} is attempting to impersonate user = {} for operation " + + "= {} from host = {}", impersonatingUser, impersonatingPrincipal, userBeingImpersonated, operation, remoteAddress); /* * no config is present for impersonating principal or user, do not permit impersonation. */ - if (!userImpersonationACL.containsKey(impersonatingPrincipal) && !userImpersonationACL.containsKey(impersonatingUser)) { - LOG.info("user = {}, principal = {} is trying to impersonate user {}, but config {} does not have entry " + if (!userImpersonationACL.containsKey(impersonatingPrincipal) && !userImpersonationACL + .containsKey(impersonatingUser)) { + LOG.info("user = {}, principal = {} is trying to impersonate user {}, but config {} " + + "does not have entry " + "for impersonating user or principal." - + "Please see SECURITY.MD to learn how to configure users for impersonation.", + + "Please see SECURITY.MD to learn how to configure users for " + + "impersonation.", impersonatingUser, impersonatingPrincipal, userBeingImpersonated, @@ -105,7 +115,8 @@ public boolean permit(ReqContext context, String operation, Map authorizedGroups.addAll(userACL.authorizedGroups); } - LOG.debug("user = {}, principal = {} is allowed to impersonate groups = {} from hosts = {} ", + LOG.debug("user = {}, principal = {} is allowed to impersonate groups = {} from hosts = " + + "{} ", impersonatingUser, impersonatingPrincipal, authorizedGroups, authorizedHosts); if (!isAllowedToImpersonateFromHost(authorizedHosts, remoteAddress)) { @@ -115,23 +126,27 @@ public boolean permit(ReqContext context, String operation, Map } if (!isAllowedToImpersonateUser(authorizedGroups, userBeingImpersonated)) { - LOG.info("user = {}, principal = {} is not allowed to impersonate any group that user {} is part of.", + LOG.info("user = {}, principal = {} is not allowed to impersonate any group that user " + + "{} is part of.", impersonatingUser, impersonatingPrincipal, userBeingImpersonated); return false; } - LOG.info("Allowing impersonation of user {} by user {}", userBeingImpersonated, impersonatingUser); + LOG.info("Allowing impersonation of user {} by user {}", userBeingImpersonated, + impersonatingUser); return true; } - private boolean isAllowedToImpersonateFromHost(Set authorizedHosts, InetAddress remoteAddress) { + private boolean isAllowedToImpersonateFromHost(Set authorizedHosts, + InetAddress remoteAddress) { return authorizedHosts.contains(WILD_CARD) || authorizedHosts.contains(remoteAddress.getCanonicalHostName()) || authorizedHosts.contains(remoteAddress.getHostName()) || authorizedHosts.contains(remoteAddress.getHostAddress()); } - private boolean isAllowedToImpersonateUser(Set authorizedGroups, String userBeingImpersonated) { + private boolean isAllowedToImpersonateUser(Set authorizedGroups, + String userBeingImpersonated) { if (authorizedGroups.contains(WILD_CARD)) { return true; } @@ -159,12 +174,13 @@ private boolean isAllowedToImpersonateUser(Set authorizedGroups, String @SuppressWarnings("checkstyle:AbbreviationAsWordInName") protected static class ImpersonationACL { public String impersonatingUser; - //Groups this user is authorized to impersonate. + // Groups this user is authorized to impersonate. public Set authorizedGroups; - //Hosts this user is authorized to impersonate from. + // Hosts this user is authorized to impersonate from. public Set authorizedHosts; - private ImpersonationACL(String impersonatingUser, Set authorizedGroups, Set authorizedHosts) { + private ImpersonationACL(String impersonatingUser, Set authorizedGroups, + Set authorizedHosts) { this.impersonatingUser = impersonatingUser; this.authorizedGroups = authorizedGroups; this.authorizedHosts = authorizedHosts; diff --git a/storm-client/src/jvm/org/apache/storm/security/auth/authorizer/NoopAuthorizer.java b/storm-client/src/jvm/org/apache/storm/security/auth/authorizer/NoopAuthorizer.java index bef7a76956b..a41868c54e6 100644 --- a/storm-client/src/jvm/org/apache/storm/security/auth/authorizer/NoopAuthorizer.java +++ b/storm-client/src/jvm/org/apache/storm/security/auth/authorizer/NoopAuthorizer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/security/auth/authorizer/SimpleACLAuthorizer.java b/storm-client/src/jvm/org/apache/storm/security/auth/authorizer/SimpleACLAuthorizer.java index baaba221d94..21fce7ebbbc 100644 --- a/storm-client/src/jvm/org/apache/storm/security/auth/authorizer/SimpleACLAuthorizer.java +++ b/storm-client/src/jvm/org/apache/storm/security/auth/authorizer/SimpleACLAuthorizer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -30,7 +36,8 @@ import org.slf4j.LoggerFactory; /** - * An authorization implementation that simply checks if a user is allowed to perform specific operations. + * An authorization implementation that simply checks if a user is allowed to perform specific + * operations. */ @SuppressWarnings("checkstyle:AbbreviationAsWordInName") public class SimpleACLAuthorizer implements IAuthorizer { @@ -153,7 +160,8 @@ public boolean permit(ReqContext context, String operation, Map } } - if (admins.contains(principal) || admins.contains(user) || checkUserGroupAllowed(userGroups, adminsGroups)) { + if (admins.contains(principal) || admins.contains(user) || checkUserGroupAllowed(userGroups, + adminsGroups)) { return true; } @@ -162,7 +170,8 @@ public boolean permit(ReqContext context, String operation, Map } if (userCommands.contains(operation)) { - // Only an empty nimbus.users AND an empty nimbus.groups means no restriction is configured. + // Only an empty nimbus.users AND an empty nimbus.groups means no restriction is + // configured. if (nimbusUsers.size() == 0 && nimbusGroups.size() == 0) { return true; } @@ -170,11 +179,13 @@ public boolean permit(ReqContext context, String operation, Map } if (topoCommands.contains(operation)) { - if (checkTopoPermission(principal, user, userGroups, topoConf, Config.TOPOLOGY_USERS, Config.TOPOLOGY_GROUPS)) { + if (checkTopoPermission(principal, user, userGroups, topoConf, Config.TOPOLOGY_USERS, + Config.TOPOLOGY_GROUPS)) { return true; } - if (topoReadOnlyCommands.contains(operation) && checkTopoPermission(principal, user, userGroups, + if (topoReadOnlyCommands.contains(operation) && checkTopoPermission(principal, user, + userGroups, topoConf, Config.TOPOLOGY_READONLY_USERS, Config.TOPOLOGY_READONLY_GROUPS)) { return true; diff --git a/storm-client/src/jvm/org/apache/storm/security/auth/authorizer/SimpleWhitelistAuthorizer.java b/storm-client/src/jvm/org/apache/storm/security/auth/authorizer/SimpleWhitelistAuthorizer.java index 98160561256..3fb28afe271 100644 --- a/storm-client/src/jvm/org/apache/storm/security/auth/authorizer/SimpleWhitelistAuthorizer.java +++ b/storm-client/src/jvm/org/apache/storm/security/auth/authorizer/SimpleWhitelistAuthorizer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -20,7 +26,8 @@ import org.apache.storm.security.auth.ReqContext; /** - * An authorization implementation that simply checks a whitelist of users that are allowed to use the cluster. + * An authorization implementation that simply checks a whitelist of users that are allowed to use + * the cluster. */ public class SimpleWhitelistAuthorizer implements IAuthorizer { public static final String WHITELIST_USERS_CONF = "storm.auth.simple-white-list.users"; diff --git a/storm-client/src/jvm/org/apache/storm/security/auth/authorizer/SupervisorSimpleACLAuthorizer.java b/storm-client/src/jvm/org/apache/storm/security/auth/authorizer/SupervisorSimpleACLAuthorizer.java index 194e8721b46..12806a341b4 100644 --- a/storm-client/src/jvm/org/apache/storm/security/auth/authorizer/SupervisorSimpleACLAuthorizer.java +++ b/storm-client/src/jvm/org/apache/storm/security/auth/authorizer/SupervisorSimpleACLAuthorizer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -30,7 +36,8 @@ import org.slf4j.LoggerFactory; /** - * An authorization implementation that simply checks if a user is allowed to perform specific operations. + * An authorization implementation that simply checks if a user is allowed to perform specific + * operations. */ @SuppressWarnings("checkstyle:AbbreviationAsWordInName") public class SupervisorSimpleACLAuthorizer implements IAuthorizer { @@ -70,12 +77,15 @@ public void prepare(Map conf) { if (conf.containsKey(Config.NIMBUS_DAEMON_USERS)) { nimbus.addAll((Collection) conf.get(Config.NIMBUS_DAEMON_USERS)); } else if (conf.containsKey(Config.NIMBUS_SUPERVISOR_USERS)) { - LOG.warn("{} is not set falling back to using {}.", Config.NIMBUS_DAEMON_USERS, Config.NIMBUS_SUPERVISOR_USERS); - //In almost all cases these should be the same, but warn the user just in case something goes wrong... + LOG.warn("{} is not set falling back to using {}.", Config.NIMBUS_DAEMON_USERS, + Config.NIMBUS_SUPERVISOR_USERS); + // In almost all cases these should be the same, but warn the user just in case + // something goes wrong... nimbus.addAll((Collection) conf.get(Config.NIMBUS_SUPERVISOR_USERS)); } else { - //If it is not set a lot of things are not really going to work all that well - LOG.error("Could not find {} things might now work correctly...", Config.NIMBUS_DAEMON_USERS); + // If it is not set a lot of things are not really going to work all that well + LOG.error("Could not find {} things might now work correctly...", + Config.NIMBUS_DAEMON_USERS); } ptol = ClientAuthUtils.getPrincipalToLocalPlugin(conf); @@ -105,7 +115,8 @@ public boolean permit(ReqContext context, String operation, Map } } - if (admins.contains(principal) || admins.contains(user) || checkUserGroupAllowed(userGroups, adminsGroups)) { + if (admins.contains(principal) || admins.contains(user) || checkUserGroupAllowed(userGroups, + adminsGroups)) { return true; } @@ -115,7 +126,8 @@ public boolean permit(ReqContext context, String operation, Map if (topoCommands.contains(operation)) { if (topoConf != null) { - if (checkTopoPermission(principal, user, userGroups, topoConf, Config.TOPOLOGY_USERS, Config.TOPOLOGY_GROUPS)) { + if (checkTopoPermission(principal, user, userGroups, topoConf, + Config.TOPOLOGY_USERS, Config.TOPOLOGY_GROUPS)) { return true; } } diff --git a/storm-client/src/jvm/org/apache/storm/security/auth/digest/DigestSaslTransportPlugin.java b/storm-client/src/jvm/org/apache/storm/security/auth/digest/DigestSaslTransportPlugin.java index 5a77d515ee3..b81ed393c46 100644 --- a/storm-client/src/jvm/org/apache/storm/security/auth/digest/DigestSaslTransportPlugin.java +++ b/storm-client/src/jvm/org/apache/storm/security/auth/digest/DigestSaslTransportPlugin.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -17,7 +23,6 @@ import javax.security.auth.callback.CallbackHandler; import javax.security.auth.login.AppConfigurationEntry; import javax.security.auth.login.Configuration; - import org.apache.storm.generated.WorkerToken; import org.apache.storm.security.auth.ClientAuthUtils; import org.apache.storm.security.auth.sasl.SaslTransportPlugin; @@ -43,21 +48,24 @@ protected TTransportFactory getServerTransportFactory(boolean impersonationAllow if (workerTokenAuthorizer == null) { workerTokenAuthorizer = new WorkerTokenAuthorizer(conf, type); } - //create an authentication callback handler - CallbackHandler serverCallbackHandler = new SimpleSaslServerCallbackHandler(impersonationAllowed, + // create an authentication callback handler + CallbackHandler serverCallbackHandler = + new SimpleSaslServerCallbackHandler(impersonationAllowed, workerTokenAuthorizer, new JassPasswordProvider(conf)); - //create a transport factory that will invoke our auth callback for digest + // create a transport factory that will invoke our auth callback for digest TSaslServerTransport.Factory factory = new TSaslServerTransport.Factory(); - factory.addServerDefinition(DIGEST, ClientAuthUtils.SERVICE, "localhost", null, serverCallbackHandler); + factory.addServerDefinition(DIGEST, ClientAuthUtils.SERVICE, "localhost", null, + serverCallbackHandler); LOG.info("SASL DIGEST-MD5 transport factory will be used"); return factory; } @Override - public TTransport connect(TTransport transport, String serverHost, String asUser) throws TTransportException, IOException { + public TTransport connect(TTransport transport, String serverHost, + String asUser) throws TTransportException, IOException { CallbackHandler clientCallbackHandler; WorkerToken token = WorkerTokenClientCallbackHandler.findWorkerTokenInSubject(type); if (token != null) { @@ -67,7 +75,8 @@ public TTransport connect(TTransport transport, String serverHost, String asUser if (loginConf == null) { throw new IOException("Could not find any way to authenticate with the server."); } - AppConfigurationEntry[] configurationEntries = loginConf.getAppConfigurationEntry(ClientAuthUtils.LOGIN_CONTEXT_CLIENT); + AppConfigurationEntry[] configurationEntries = loginConf + .getAppConfigurationEntry(ClientAuthUtils.LOGIN_CONTEXT_CLIENT); if (configurationEntries == null) { String errorMessage = "Could not find a '" + ClientAuthUtils.LOGIN_CONTEXT_CLIENT + "' entry in this configuration: Client cannot start."; diff --git a/storm-client/src/jvm/org/apache/storm/security/auth/digest/JassPasswordProvider.java b/storm-client/src/jvm/org/apache/storm/security/auth/digest/JassPasswordProvider.java index 1d5c7e33d46..2a828a5392a 100644 --- a/storm-client/src/jvm/org/apache/storm/security/auth/digest/JassPasswordProvider.java +++ b/storm-client/src/jvm/org/apache/storm/security/auth/digest/JassPasswordProvider.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -26,10 +32,12 @@ */ public class JassPasswordProvider implements PasswordProvider { /** - * The system property that sets a super user password. This can be used in addition to the jaas conf, and takes precedent over a + * The system property that sets a super user password. This can be used in addition to the jaas + * conf, and takes precedent over a * "super" user in the jaas conf if this is set. */ - public static final String SYSPROP_SUPER_PASSWORD = "storm.SASLAuthenticationProvider.superPassword"; + public static final String SYSPROP_SUPER_PASSWORD = + "storm.SASLAuthenticationProvider.superPassword"; private static final String USER_PREFIX = "user_"; private Map credentials = new ConcurrentHashMap<>(); @@ -46,7 +54,8 @@ public JassPasswordProvider(Map topoConf) throws IOException { return; } - AppConfigurationEntry[] configurationEntries = configuration.getAppConfigurationEntry(ClientAuthUtils.LOGIN_CONTEXT_SERVER); + AppConfigurationEntry[] configurationEntries = configuration + .getAppConfigurationEntry(ClientAuthUtils.LOGIN_CONTEXT_SERVER); if (configurationEntries == null) { String errorMessage = "Could not find a '" + ClientAuthUtils.LOGIN_CONTEXT_SERVER + "' entry in this configuration: Server cannot start."; @@ -55,8 +64,10 @@ public JassPasswordProvider(Map topoConf) throws IOException { credentials.clear(); for (AppConfigurationEntry entry : configurationEntries) { Map options = entry.getOptions(); - // Populate user -> password map with JAAS configuration entries from the "Server" section. - // Usernames are distinguished from other options by prefixing the username with a "user_" prefix. + // Populate user -> password map with JAAS configuration entries from the "Server" + // section. + // Usernames are distinguished from other options by prefixing the username with a + // "user_" prefix. for (Map.Entry pair : options.entrySet()) { String key = pair.getKey(); if (key.startsWith(USER_PREFIX)) { diff --git a/storm-client/src/jvm/org/apache/storm/security/auth/kerberos/AutoTGT.java b/storm-client/src/jvm/org/apache/storm/security/auth/kerberos/AutoTGT.java index c3f181b2709..b88e8ac5382 100644 --- a/storm-client/src/jvm/org/apache/storm/security/auth/kerberos/AutoTGT.java +++ b/storm-client/src/jvm/org/apache/storm/security/auth/kerberos/AutoTGT.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -42,7 +48,7 @@ public class AutoTGT implements IAutoCredentials, ICredentialsRenewer, IMetricsRegistrant { protected static final AtomicReference kerbTicket = new AtomicReference<>(); private static final Logger LOG = LoggerFactory.getLogger(AutoTGT.class); - private static final float TICKET_RENEW_WINDOW = 0.80f; + private static final float TICKET_RENEW_WINDOW = 0.80F; private Map conf; private Map credentials; @@ -61,8 +67,10 @@ private static KerberosTicket getTGT(Subject subject) { @SuppressWarnings("checkstyle:AbbreviationAsWordInName") public static KerberosTicket getTGT(Map credentials) { KerberosTicket ret = null; - if (credentials != null && credentials.containsKey("TGT") && credentials.get("TGT") != null) { - ret = ClientAuthUtils.deserializeKerberosTicket(DatatypeConverter.parseBase64Binary(credentials.get("TGT"))); + if (credentials != null && credentials.containsKey("TGT") && credentials + .get("TGT") != null) { + ret = ClientAuthUtils.deserializeKerberosTicket(DatatypeConverter + .parseBase64Binary(credentials.get("TGT"))); } return ret; } @@ -120,33 +128,38 @@ public void prepare(Map conf) { @Override public void populateCredentials(Map credentials) { this.credentials = credentials; - //Log the user in and get the TGT + // Log the user in and get the TGT try { Configuration loginConf = ClientAuthUtils.getConfiguration(conf); ClientCallbackHandler clientCallbackHandler = new ClientCallbackHandler(conf); - //login our user - LoginContext lc = new LoginContext(ClientAuthUtils.LOGIN_CONTEXT_CLIENT, null, clientCallbackHandler, loginConf); + // login our user + LoginContext lc = new LoginContext(ClientAuthUtils.LOGIN_CONTEXT_CLIENT, null, + clientCallbackHandler, loginConf); try { lc.login(); final Subject subject = lc.getSubject(); KerberosTicket tgt = getTGT(subject); - if (tgt == null) { //error + if (tgt == null) { // error throw new RuntimeException("Fail to verify user principal with section \"" - + ClientAuthUtils.LOGIN_CONTEXT_CLIENT + "\" in login configuration file " + loginConf); + + ClientAuthUtils.LOGIN_CONTEXT_CLIENT + + "\" in login configuration file " + loginConf); } if (!tgt.isForwardable()) { - throw new RuntimeException("The TGT found is not forwardable. Please use -f option with 'kinit'."); + throw new RuntimeException("The TGT found is not forwardable. Please use -f " + + "option with 'kinit'."); } if (!tgt.isRenewable()) { - throw new RuntimeException("The TGT found is not renewable. Please use -r option with 'kinit'."); + throw new RuntimeException("The TGT found is not renewable. Please use -r " + + "option with 'kinit'."); } if (tgt.getClientAddresses() != null) { - throw new RuntimeException("The TGT found is not address-less. Please use -A option with 'kinit'."); + throw new RuntimeException("The TGT found is not address-less. Please use -A " + + "option with 'kinit'."); } LOG.info("Pushing TGT for " + tgt.getClient() + " to topology."); @@ -212,14 +225,18 @@ private void loginHadoopUser(Subject subject) { Method login = ugi.getMethod("loginUserFromSubject", Subject.class); login.invoke(null, subject); - //Refer to STORM-3606 for details - LOG.warn("UserGroupInformation.loginUserFromSubject will spawn a TGT renewal thread (\"TGT Renewer for \") " + // Refer to STORM-3606 for details + LOG.warn("UserGroupInformation.loginUserFromSubject will spawn a TGT renewal thread " + + "(\"TGT Renewer for \") " + "to execute \"kinit -R\" command some time before the current TGT expires. " - + "It will fail because TGT is not in the local TGT cache and the thread will eventually abort. " - + "Exceptions from this TGT renewal thread can be ignored. Note: TGT for the Worker is kept in memory. " + + "It will fail because TGT is not in the local TGT cache and the thread will " + + "eventually abort. " + + "Exceptions from this TGT renewal thread can be ignored. Note: TGT for the " + + "Worker is kept in memory. " + "Please refer to STORM-3606 for detailed explanations"); } catch (Exception e) { - LOG.error("Something went wrong while trying to initialize Hadoop through reflection. This version of hadoop " + LOG.error("Something went wrong while trying to initialize Hadoop through reflection. " + + "This version of hadoop " + "may not be compatible.", e); } } @@ -231,7 +248,8 @@ private long getRefreshTime(KerberosTicket tgt) { } @Override - public void renew(Map credentials, Map topologyConf, String topologyOwnerPrincipal) { + public void renew(Map credentials, Map topologyConf, + String topologyOwnerPrincipal) { this.credentials = credentials; KerberosTicket tgt = getTGT(credentials); if (tgt != null) { diff --git a/storm-client/src/jvm/org/apache/storm/security/auth/kerberos/AutoTGTKrb5LoginModule.java b/storm-client/src/jvm/org/apache/storm/security/auth/kerberos/AutoTGTKrb5LoginModule.java index cc070f57873..b5b4c0ae776 100644 --- a/storm-client/src/jvm/org/apache/storm/security/auth/kerberos/AutoTGTKrb5LoginModule.java +++ b/storm-client/src/jvm/org/apache/storm/security/auth/kerberos/AutoTGTKrb5LoginModule.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -75,7 +81,8 @@ public boolean commit() throws LoginException { throw new LoginException("Authentication failed because the Subject is invalid."); } // Let us add the kerbClientPrinc and kerbTicket - // We need to clone the ticket because java.security.auth.kerberos assumes TGT is unique for each subject + // We need to clone the ticket because java.security.auth.kerberos assumes TGT is unique for + // each subject // So, sharing TGT with multiple subjects can cause expired TGT to never refresh. KerberosTicket kerbTicketCopy = ClientAuthUtils.cloneKerberosTicket(kerbTicket); subject.getPrivateCredentials().add(kerbTicketCopy); diff --git a/storm-client/src/jvm/org/apache/storm/security/auth/kerberos/AutoTGTKrb5LoginModuleTest.java b/storm-client/src/jvm/org/apache/storm/security/auth/kerberos/AutoTGTKrb5LoginModuleTest.java index 16b8eb080fe..925ddae6ba8 100644 --- a/storm-client/src/jvm/org/apache/storm/security/auth/kerberos/AutoTGTKrb5LoginModuleTest.java +++ b/storm-client/src/jvm/org/apache/storm/security/auth/kerberos/AutoTGTKrb5LoginModuleTest.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/security/auth/kerberos/ClientCallbackHandler.java b/storm-client/src/jvm/org/apache/storm/security/auth/kerberos/ClientCallbackHandler.java index f9a8e0965b8..fa4becb92d0 100644 --- a/storm-client/src/jvm/org/apache/storm/security/auth/kerberos/ClientCallbackHandler.java +++ b/storm-client/src/jvm/org/apache/storm/security/auth/kerberos/ClientCallbackHandler.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -42,7 +48,8 @@ public ClientCallbackHandler(Map topoConf) throws IOException { if (configuration == null) { return; } - AppConfigurationEntry[] configurationEntries = configuration.getAppConfigurationEntry(ClientAuthUtils.LOGIN_CONTEXT_CLIENT); + AppConfigurationEntry[] configurationEntries = configuration + .getAppConfigurationEntry(ClientAuthUtils.LOGIN_CONTEXT_CLIENT); if (configurationEntries == null) { String errorMessage = "Could not find a '" + ClientAuthUtils.LOGIN_CONTEXT_CLIENT + "' entry in this configuration: Client cannot start."; @@ -64,19 +71,29 @@ public void handle(Callback[] callbacks) throws IOException, UnsupportedCallback } else if (c instanceof PasswordCallback) { LOG.debug("password callback"); LOG.warn("Could not login: the client is being asked for a password, but the " - + " client code does not currently support obtaining a password from the user." + + " client code does not currently support obtaining a password from the " + + "user." + " Make sure that the client is configured to use a ticket cache (using" - + " the JAAS configuration setting 'useTicketCache=true)' and restart the client. If" - + " you still get this message after that, the TGT in the ticket cache has expired and must" - + " be manually refreshed. To do so, first determine if you are using a password or a" - + " keytab. If the former, run kinit in a Unix shell in the environment of the user who" + + " the JAAS configuration setting 'useTicketCache=true)' and restart the " + + "client. If" + + " you still get this message after that, the TGT in the ticket cache " + + "has expired and must" + + " be manually refreshed. To do so, first determine if you are using a " + + "password or a" + + " keytab. If the former, run kinit in a Unix shell in the environment " + + "of the user who" + " is running this client using the command" - + " 'kinit ' (where is the name of the client's Kerberos principal)." + + " 'kinit ' (where is the name of the client's Kerberos " + + "principal)." + " If the latter, do" - + " 'kinit -k -t ' (where is the name of the Kerberos principal, and" - + " is the location of the keytab file). After manually refreshing your cache," - + " restart this client. If you continue to see this message after manually refreshing" - + " your cache, ensure that your KDC host's clock is in sync with this host's clock."); + + " 'kinit -k -t ' (where is the name of the " + + "Kerberos principal, and" + + " is the location of the keytab file). After manually " + + "refreshing your cache," + + " restart this client. If you continue to see this message after " + + "manually refreshing" + + " your cache, ensure that your KDC host's clock is in sync with this " + + "host's clock."); } else if (c instanceof AuthorizeCallback) { LOG.debug("authorization callback"); AuthorizeCallback ac = (AuthorizeCallback) c; diff --git a/storm-client/src/jvm/org/apache/storm/security/auth/kerberos/KerberosSaslTransportPlugin.java b/storm-client/src/jvm/org/apache/storm/security/auth/kerberos/KerberosSaslTransportPlugin.java index 3956323630f..4e17c386d8b 100644 --- a/storm-client/src/jvm/org/apache/storm/security/auth/kerberos/KerberosSaslTransportPlugin.java +++ b/storm-client/src/jvm/org/apache/storm/security/auth/kerberos/KerberosSaslTransportPlugin.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -55,16 +61,18 @@ public TTransportFactory getServerTransportFactory(boolean impersonationAllowed) if (workerTokenAuthorizer == null) { workerTokenAuthorizer = new WorkerTokenAuthorizer(conf, type); } - //create an authentication callback handler - CallbackHandler serverCallbackHandler = new ServerCallbackHandler(conf, impersonationAllowed); + // create an authentication callback handler + CallbackHandler serverCallbackHandler = new ServerCallbackHandler(conf, + impersonationAllowed); String jaasConfFile = ClientAuthUtils.getJaasConf(conf); - //login our principal + // login our principal Subject subject = null; try { - //now login - Login login = new Login(ClientAuthUtils.LOGIN_CONTEXT_SERVER, serverCallbackHandler, jaasConfFile); + // now login + Login login = new Login(ClientAuthUtils.LOGIN_CONTEXT_SERVER, serverCallbackHandler, + jaasConfFile); subject = login.getSubject(); login.startThreadIfNeeded(); } catch (LoginException ex) { @@ -72,13 +80,15 @@ public TTransportFactory getServerTransportFactory(boolean impersonationAllowed) throw new RuntimeException(ex); } - //check the credential of our principal + // check the credential of our principal if (subject.getPrivateCredentials(KerberosTicket.class).isEmpty()) { throw new RuntimeException("Fail to verify user principal with section \"" - + ClientAuthUtils.LOGIN_CONTEXT_SERVER + "\" in login configuration file " + jaasConfFile); + + ClientAuthUtils.LOGIN_CONTEXT_SERVER + + "\" in login configuration file " + jaasConfFile); } - String principal = ClientAuthUtils.get(conf, ClientAuthUtils.LOGIN_CONTEXT_SERVER, "principal"); + String principal = ClientAuthUtils.get(conf, ClientAuthUtils.LOGIN_CONTEXT_SERVER, + "principal"); LOG.debug("principal:" + principal); KerberosName serviceKerberosName = new KerberosName(principal); String serviceName = serviceKerberosName.getServiceName(); @@ -87,16 +97,18 @@ public TTransportFactory getServerTransportFactory(boolean impersonationAllowed) props.put(Sasl.QOP, "auth"); props.put(Sasl.SERVER_AUTH, "false"); - //create a transport factory that will invoke our auth callback for digest + // create a transport factory that will invoke our auth callback for digest TSaslServerTransport.Factory factory = new TSaslServerTransport.Factory(); factory.addServerDefinition(KERBEROS, serviceName, hostName, props, serverCallbackHandler); - //Also add in support for worker tokens + // Also add in support for worker tokens factory.addServerDefinition(DIGEST, ClientAuthUtils.SERVICE, hostName, null, - new SimpleSaslServerCallbackHandler(impersonationAllowed, workerTokenAuthorizer)); + new SimpleSaslServerCallbackHandler(impersonationAllowed, + workerTokenAuthorizer)); - //create a wrap transport factory so that we could apply user credential during connections - TUGIAssumingTransportFactory wrapFactory = new TUGIAssumingTransportFactory(factory, subject); + // create a wrap transport factory so that we could apply user credential during connections + TUGIAssumingTransportFactory wrapFactory = new TUGIAssumingTransportFactory(factory, + subject); LOG.info("SASL GSSAPI transport factory will be used"); return wrapFactory; @@ -104,10 +116,11 @@ public TTransportFactory getServerTransportFactory(boolean impersonationAllowed) private Login mkLogin() throws IOException { try { - //create an authentication callback handler + // create an authentication callback handler ClientCallbackHandler clientCallbackHandler = new ClientCallbackHandler(conf); - //now login - Login login = new Login(ClientAuthUtils.LOGIN_CONTEXT_CLIENT, clientCallbackHandler, ClientAuthUtils.getJaasConf(conf)); + // now login + Login login = new Login(ClientAuthUtils.LOGIN_CONTEXT_CLIENT, clientCallbackHandler, + ClientAuthUtils.getJaasConf(conf)); login.startThreadIfNeeded(); return login; } catch (LoginException ex) { @@ -117,7 +130,8 @@ private Login mkLogin() throws IOException { } @Override - public TTransport connect(TTransport transport, String serverHost, String asUser) throws IOException, TTransportException { + public TTransport connect(TTransport transport, String serverHost, + String asUser) throws IOException, TTransportException { WorkerToken token = WorkerTokenClientCallbackHandler.findWorkerTokenInSubject(type); if (token != null) { CallbackHandler clientCallbackHandler = new WorkerTokenClientCallbackHandler(token); @@ -136,11 +150,14 @@ public TTransport connect(TTransport transport, String serverHost, String asUser return kerberosConnect(transport, serverHost, asUser); } - private TTransport kerberosConnect(TTransport transport, String serverHost, String asUser) throws IOException, TTransportException { - //login our user - SortedMap authConf = ClientAuthUtils.pullConfig(conf, ClientAuthUtils.LOGIN_CONTEXT_CLIENT); + private TTransport kerberosConnect(TTransport transport, String serverHost, + String asUser) throws IOException, TTransportException { + // login our user + SortedMap authConf = ClientAuthUtils.pullConfig(conf, + ClientAuthUtils.LOGIN_CONTEXT_CLIENT); if (authConf == null) { - throw new RuntimeException("Error in parsing the kerberos login Configuration, returned null"); + throw new RuntimeException("Error in parsing the kerberos login Configuration, " + + "returned null"); } boolean disableLoginCache = false; @@ -153,10 +170,11 @@ private TTransport kerberosConnect(TTransport transport, String serverHost, Stri if (disableLoginCache) { LOG.debug("Kerberos Login Cache is disabled, attempting to contact the Kerberos Server"); login = mkLogin(); - //this is to prevent the potential bug that - //if the Login Cache is (1) enabled, and then (2) disabled and then (3) enabled again, - //and if the LoginCacheKey remains unchanged, (3) will use the Login cache from (1), which could be wrong, - //because the TGT cache (as well as the principle) could have been changed during (2) + // this is to prevent the potential bug that + // if the Login Cache is (1) enabled, and then (2) disabled and then (3) enabled again, + // and if the LoginCacheKey remains unchanged, (3) will use the Login cache from (1), + // which could be wrong, + // because the TGT cache (as well as the principle) could have been changed during (2) loginCache.remove(key); } else { LOG.debug("Trying to get the Kerberos Login from the Login Cache"); @@ -165,7 +183,8 @@ private TTransport kerberosConnect(TTransport transport, String serverHost, Stri synchronized (loginCache) { login = loginCache.get(key); if (login == null) { - LOG.debug("Kerberos Login was not found in the Login Cache, attempting to contact the Kerberos Server"); + LOG.debug("Kerberos Login was not found in the Login Cache, attempting to " + + "contact the Kerberos Server"); login = mkLogin(); loginCache.put(key, login); } @@ -174,13 +193,15 @@ private TTransport kerberosConnect(TTransport transport, String serverHost, Stri } final Subject subject = login.getSubject(); - if (subject.getPrivateCredentials(KerberosTicket.class).isEmpty()) { //error + if (subject.getPrivateCredentials(KerberosTicket.class).isEmpty()) { // error throw new RuntimeException("Fail to verify user principal with section \"" - + ClientAuthUtils.LOGIN_CONTEXT_CLIENT + "\" in login configuration file " + ClientAuthUtils.getJaasConf(conf)); + + ClientAuthUtils.LOGIN_CONTEXT_CLIENT + "\" in login configuration file " + + ClientAuthUtils.getJaasConf(conf)); } final String principal = StringUtils.isBlank(asUser) ? getPrincipal(subject) : asUser; - String serviceName = ClientAuthUtils.get(conf, ClientAuthUtils.LOGIN_CONTEXT_CLIENT, "serviceName"); + String serviceName = ClientAuthUtils.get(conf, ClientAuthUtils.LOGIN_CONTEXT_CLIENT, + "serviceName"); if (serviceName == null) { serviceName = ClientAuthUtils.SERVICE; } @@ -197,14 +218,15 @@ private TTransport kerberosConnect(TTransport transport, String serverHost, Stri null, transport); - //open Sasl transport with the login credential + // open Sasl transport with the login credential try { SubjectCompat.doAs(subject, () -> { try { LOG.debug("do as:" + principal); sasalTransport.open(); } catch (Exception e) { - LOG.error("Client failed to open SaslClientTransport to interact with a server during " + LOG.error("Client failed to open SaslClientTransport to interact with a " + + "server during " + "session initiation: " + e, e); @@ -238,7 +260,8 @@ public void close() { } /** - * A TransportFactory that wraps another one, but assumes a specified UGI before calling through. + * A TransportFactory that wraps another one, but assumes a specified UGI before calling + * through. * *

    This is used on the server side to assume the server's Principal when accepting clients. */ @@ -264,7 +287,8 @@ public TTransport getTransport(final TTransport trans) { try { return wrapped.getTransport(trans); } catch (Exception e) { - LOG.debug("Storm server failed to open transport to interact with a client during " + LOG.debug("Storm server failed to open transport to interact with a " + + "client during " + "session initiation: " + e, e); @@ -288,8 +312,8 @@ private class LoginCacheKey { if (authConf != null) { StringBuilder stringBuilder = new StringBuilder(); for (String configKey : authConf.keySet()) { - //DISABLE_LOGIN_CACHE indicates whether or not to use the LoginCache. - //So we exclude it from the keyString + // DISABLE_LOGIN_CACHE indicates whether or not to use the LoginCache. + // So we exclude it from the keyString if (configKey.equals(DISABLE_LOGIN_CACHE)) { continue; } @@ -310,7 +334,8 @@ public int hashCode() { @Override public boolean equals(Object obj) { - return (obj instanceof LoginCacheKey) && keyString.equals(((LoginCacheKey) obj).keyString); + return (obj instanceof LoginCacheKey) && keyString + .equals(((LoginCacheKey) obj).keyString); } @Override diff --git a/storm-client/src/jvm/org/apache/storm/security/auth/kerberos/NoOpTTrasport.java b/storm-client/src/jvm/org/apache/storm/security/auth/kerberos/NoOpTTrasport.java index 39948502d8c..6139fecde32 100644 --- a/storm-client/src/jvm/org/apache/storm/security/auth/kerberos/NoOpTTrasport.java +++ b/storm-client/src/jvm/org/apache/storm/security/auth/kerberos/NoOpTTrasport.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/security/auth/kerberos/ServerCallbackHandler.java b/storm-client/src/jvm/org/apache/storm/security/auth/kerberos/ServerCallbackHandler.java index daa5fbff9c5..d4458d1f5c5 100644 --- a/storm-client/src/jvm/org/apache/storm/security/auth/kerberos/ServerCallbackHandler.java +++ b/storm-client/src/jvm/org/apache/storm/security/auth/kerberos/ServerCallbackHandler.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -36,7 +42,8 @@ public class ServerCallbackHandler implements CallbackHandler { private static final Logger LOG = LoggerFactory.getLogger(ServerCallbackHandler.class); private final boolean impersonationAllowed; - public ServerCallbackHandler(Map topoConf, boolean impersonationAllowed) throws IOException { + public ServerCallbackHandler(Map topoConf, + boolean impersonationAllowed) throws IOException { this.impersonationAllowed = impersonationAllowed; Configuration configuration = ClientAuthUtils.getConfiguration(topoConf); @@ -44,7 +51,8 @@ public ServerCallbackHandler(Map topoConf, boolean impersonation return; } - AppConfigurationEntry[] configurationEntries = configuration.getAppConfigurationEntry(ClientAuthUtils.LOGIN_CONTEXT_SERVER); + AppConfigurationEntry[] configurationEntries = configuration + .getAppConfigurationEntry(ClientAuthUtils.LOGIN_CONTEXT_SERVER); if (configurationEntries == null) { String errorMessage = "Could not find a '" + ClientAuthUtils.LOGIN_CONTEXT_SERVER + "' entry in this configuration: Server cannot start."; @@ -66,7 +74,7 @@ public void handle(Callback[] callbacks) throws UnsupportedCallbackException { } else if (callback instanceof PasswordCallback) { pc = (PasswordCallback) callback; } else if (callback instanceof RealmCallback) { - //Ignored... + // Ignored... } else { throw new UnsupportedCallbackException(callback, "Unrecognized SASL Callback"); @@ -86,22 +94,27 @@ public void handle(Callback[] callbacks) throws UnsupportedCallbackException { if (ac != null) { String authenticationId = ac.getAuthenticationID(); - LOG.debug("Successfully authenticated client: authenticationID={} authorizationID= {}", authenticationId, + LOG.debug("Successfully authenticated client: authenticationID={} authorizationID= {}", + authenticationId, ac.getAuthorizationID()); - //if authorizationId is not set, set it to authenticationId. + // if authorizationId is not set, set it to authenticationId. if (ac.getAuthorizationID() == null) { ac.setAuthorizedID(authenticationId); } - //When authNid and authZid are not equal , authNId is attempting to impersonate authZid, We - //add the authNid as the real user in reqContext's subject which will be used during authorization. + // When authNid and authZid are not equal , authNId is attempting to impersonate + // authZid, We + // add the authNid as the real user in reqContext's subject which will be used during + // authorization. if (!ac.getAuthenticationID().equals(ac.getAuthorizationID())) { if (!impersonationAllowed) { - throw new IllegalArgumentException(ac.getAuthenticationID() + " attempting to impersonate " + ac.getAuthorizationID() + throw new IllegalArgumentException(ac.getAuthenticationID() + + " attempting to impersonate " + ac.getAuthorizationID() + ". This is not allowed by this server."); } - ReqContext.context().setRealPrincipal(new SaslTransportPlugin.User(ac.getAuthenticationID())); + ReqContext.context().setRealPrincipal(new SaslTransportPlugin.User(ac + .getAuthenticationID())); } else { ReqContext.context().setRealPrincipal(null); } diff --git a/storm-client/src/jvm/org/apache/storm/security/auth/plain/PlainClientCallbackHandler.java b/storm-client/src/jvm/org/apache/storm/security/auth/plain/PlainClientCallbackHandler.java index 2e48f25cbec..74ace7465f9 100644 --- a/storm-client/src/jvm/org/apache/storm/security/auth/plain/PlainClientCallbackHandler.java +++ b/storm-client/src/jvm/org/apache/storm/security/auth/plain/PlainClientCallbackHandler.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -15,7 +21,8 @@ import org.apache.storm.security.auth.sasl.SimpleSaslClientCallbackHandler; /** - * This should only ever be used for testing. It provides no security at all. DO NOT USE THIS. The user name is the current user and the + * This should only ever be used for testing. It provides no security at all. DO NOT USE THIS. The + * user name is the current user and the * password is "password". */ @Deprecated diff --git a/storm-client/src/jvm/org/apache/storm/security/auth/plain/PlainSaslTransportPlugin.java b/storm-client/src/jvm/org/apache/storm/security/auth/plain/PlainSaslTransportPlugin.java index 697fa5ecb0f..7eb9674e7ca 100644 --- a/storm-client/src/jvm/org/apache/storm/security/auth/plain/PlainSaslTransportPlugin.java +++ b/storm-client/src/jvm/org/apache/storm/security/auth/plain/PlainSaslTransportPlugin.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -28,7 +34,8 @@ import org.slf4j.LoggerFactory; /** - * This should never be used except for testing. It provides no security at all. The password is hard coded, and even if it were not it is + * This should never be used except for testing. It provides no security at all. The password is + * hard coded, and even if it were not it is * sent in plain text. */ @Deprecated @@ -38,22 +45,26 @@ public class PlainSaslTransportPlugin extends SaslTransportPlugin { @Override protected TTransportFactory getServerTransportFactory(boolean impersonationAllowed) throws IOException { - //create an authentication callback handler - CallbackHandler serverCallbackHandler = new SimpleSaslServerCallbackHandler(impersonationAllowed, - (userName) -> Optional.of("password".toCharArray())); + // create an authentication callback handler + CallbackHandler serverCallbackHandler = + new SimpleSaslServerCallbackHandler(impersonationAllowed, + (userName) -> Optional.of("password".toCharArray())); if (Security.getProvider(SaslPlainServer.SecurityProvider.SASL_PLAIN_SERVER) == null) { Security.addProvider(new SaslPlainServer.SecurityProvider()); } - //create a transport factory that will invoke our auth callback for digest + // create a transport factory that will invoke our auth callback for digest TSaslServerTransport.Factory factory = new TSaslServerTransport.Factory(); - factory.addServerDefinition(PLAIN, ClientAuthUtils.SERVICE, "localhost", null, serverCallbackHandler); + factory.addServerDefinition(PLAIN, ClientAuthUtils.SERVICE, "localhost", null, + serverCallbackHandler); - LOG.error("SASL PLAIN transport factory will be used. This is totally insecure. Please do not use this."); + LOG.error("SASL PLAIN transport factory will be used. This is totally insecure. Please " + + "do not use this."); return factory; } @Override - public TTransport connect(TTransport transport, String serverHost, String asUser) throws IOException, TTransportException { + public TTransport connect(TTransport transport, String serverHost, + String asUser) throws IOException, TTransportException { PlainClientCallbackHandler clientCallbackHandler = new PlainClientCallbackHandler(); TSaslClientTransport wrapperTransport = new TSaslClientTransport(PLAIN, null, @@ -64,7 +75,8 @@ public TTransport connect(TTransport transport, String serverHost, String asUser transport); wrapperTransport.open(); - LOG.error("SASL PLAIN client transport has been established. This is totally insecure. Please do not use this."); + LOG.error("SASL PLAIN client transport has been established. This is totally insecure. " + + "Please do not use this."); return wrapperTransport; } diff --git a/storm-client/src/jvm/org/apache/storm/security/auth/plain/SaslPlainServer.java b/storm-client/src/jvm/org/apache/storm/security/auth/plain/SaslPlainServer.java index e27fe2e75af..c09c53d2eda 100644 --- a/storm-client/src/jvm/org/apache/storm/security/auth/plain/SaslPlainServer.java +++ b/storm-client/src/jvm/org/apache/storm/security/auth/plain/SaslPlainServer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/security/auth/sasl/PasswordProvider.java b/storm-client/src/jvm/org/apache/storm/security/auth/sasl/PasswordProvider.java index a1fc7d3ac53..d61cb8f2d33 100644 --- a/storm-client/src/jvm/org/apache/storm/security/auth/sasl/PasswordProvider.java +++ b/storm-client/src/jvm/org/apache/storm/security/auth/sasl/PasswordProvider.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -15,12 +21,14 @@ import java.util.Optional; /** - * A very basic API that will provide a password for a given user name. This is intended to be used with the SimpleSaslServerCallbackHandler + * A very basic API that will provide a password for a given user name. This is intended to be used + * with the SimpleSaslServerCallbackHandler * to verify a user that is attempting to log in. */ public interface PasswordProvider { /** - * Get an optional password for a user. If no password for the user is found the option will be empty and another PasswordProvider + * Get an optional password for a user. If no password for the user is found the option will be + * empty and another PasswordProvider * would be tried. * * @param user the user this is for. @@ -38,8 +46,10 @@ default boolean isImpersonationAllowed() { } /** - * Convert the supplied user name to the actual user name that should be used in the system. This may be called on any name. If it - * cannot be translated then a null may be returned or an exception thrown. If getPassword returns successfully this should not return + * Convert the supplied user name to the actual user name that should be used in the system. + * This may be called on any name. If it + * cannot be translated then a null may be returned or an exception thrown. If getPassword + * returns successfully this should not return * null, nor throw an exception for the same user. * * @param user the SASL negotiated user name. diff --git a/storm-client/src/jvm/org/apache/storm/security/auth/sasl/SaslTransportPlugin.java b/storm-client/src/jvm/org/apache/storm/security/auth/sasl/SaslTransportPlugin.java index 745d360a072..aed69564abd 100644 --- a/storm-client/src/jvm/org/apache/storm/security/auth/sasl/SaslTransportPlugin.java +++ b/storm-client/src/jvm/org/apache/storm/security/auth/sasl/SaslTransportPlugin.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -61,7 +67,8 @@ public void prepare(ThriftConnectionType type, Map conf) { public TServer getServer(TProcessor processor) throws IOException, TTransportException { int configuredPort = type.getPort(conf); Integer socketTimeout = type.getSocketTimeOut(conf); - TTransportFactory serverTransportFactory = getServerTransportFactory(type.isImpersonationAllowed()); + TTransportFactory serverTransportFactory = getServerTransportFactory(type + .isImpersonationAllowed()); TServerSocket serverTransport = null; if (socketTimeout != null) { serverTransport = new TServerSocket(configuredPort, socketTimeout); @@ -85,7 +92,8 @@ public TServer getServer(TProcessor processor) throws IOException, TTransportExc if (queueSize != null) { workQueue = new ArrayBlockingQueue<>(queueSize); } - ThreadPoolExecutor executorService = new ExtendedThreadPoolExecutor(numWorkerThreads, numWorkerThreads, + ThreadPoolExecutor executorService = new ExtendedThreadPoolExecutor(numWorkerThreads, + numWorkerThreads, 60, TimeUnit.SECONDS, workQueue); serverArgs.executorService(executorService); return new TThreadPoolServer(serverArgs); @@ -112,7 +120,8 @@ public int getPort() { /** - * Processor that pulls the SaslServer object out of the transport, and assumes the remote user's UGI before calling through to the + * Processor that pulls the SaslServer object out of the transport, and assumes the remote + * user's UGI before calling through to the * original processor. This is used on the server side to set the UGI for each specific call. */ @SuppressWarnings("checkstyle:AbbreviationAsWordInName") @@ -125,30 +134,30 @@ private static class TUGIWrapProcessor implements TProcessor { @Override public void process(final TProtocol inProt, final TProtocol outProt) throws TException { - //populating request context + // populating request context ReqContext reqContext = ReqContext.context(); TTransport trans = inProt.getTransport(); - //Sasl transport + // Sasl transport TSaslServerTransport saslTrans = (TSaslServerTransport) trans; if (trans instanceof NoOpTTrasport) { return; } - //remote address + // remote address TSocket tsocket = (TSocket) saslTrans.getUnderlyingTransport(); Socket socket = tsocket.getSocket(); reqContext.setRemoteAddress(socket.getInetAddress()); - //remote subject + // remote subject SaslServer saslServer = saslTrans.getSaslServer(); String authId = saslServer.getAuthorizationID(); Subject remoteUser = new Subject(); remoteUser.getPrincipals().add(new User(authId)); reqContext.setSubject(remoteUser); - //invoke service handler + // invoke service handler wrapped.process(inProt, outProt); } } diff --git a/storm-client/src/jvm/org/apache/storm/security/auth/sasl/SimpleSaslClientCallbackHandler.java b/storm-client/src/jvm/org/apache/storm/security/auth/sasl/SimpleSaslClientCallbackHandler.java index ad0aa328910..77c120639dc 100644 --- a/storm-client/src/jvm/org/apache/storm/security/auth/sasl/SimpleSaslClientCallbackHandler.java +++ b/storm-client/src/jvm/org/apache/storm/security/auth/sasl/SimpleSaslClientCallbackHandler.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/security/auth/sasl/SimpleSaslServerCallbackHandler.java b/storm-client/src/jvm/org/apache/storm/security/auth/sasl/SimpleSaslServerCallbackHandler.java index 462e87280fd..1c058dbbda4 100644 --- a/storm-client/src/jvm/org/apache/storm/security/auth/sasl/SimpleSaslServerCallbackHandler.java +++ b/storm-client/src/jvm/org/apache/storm/security/auth/sasl/SimpleSaslServerCallbackHandler.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -31,7 +37,8 @@ import org.slf4j.LoggerFactory; public class SimpleSaslServerCallbackHandler implements CallbackHandler { - private static final Logger LOG = LoggerFactory.getLogger(SimpleSaslServerCallbackHandler.class); + private static final Logger LOG = LoggerFactory + .getLogger(SimpleSaslServerCallbackHandler.class); private final List providers; private final boolean impersonationAllowed; @@ -39,10 +46,12 @@ public class SimpleSaslServerCallbackHandler implements CallbackHandler { * Constructor with different password providers. * * @param impersonationAllowed true if impersonation is allowed else false. - * @param providers what will provide a password. They will be checked in order, and the first one to return a password + * @param providers what will provide a password. They will be checked in order, and the first + * one to return a password * wins. */ - public SimpleSaslServerCallbackHandler(boolean impersonationAllowed, PasswordProvider... providers) { + public SimpleSaslServerCallbackHandler(boolean impersonationAllowed, + PasswordProvider... providers) { this(impersonationAllowed, Arrays.asList(providers)); } @@ -50,19 +59,23 @@ public SimpleSaslServerCallbackHandler(boolean impersonationAllowed, PasswordPro * Constructor with different password providers. * * @param impersonationAllowed true if impersonation is allowed else false. - * @param providers what will provide a password. They will be checked in order, and the first one to return a password + * @param providers what will provide a password. They will be checked in order, and the first + * one to return a password * wins. */ - public SimpleSaslServerCallbackHandler(boolean impersonationAllowed, List providers) { + public SimpleSaslServerCallbackHandler(boolean impersonationAllowed, + List providers) { this.impersonationAllowed = impersonationAllowed; this.providers = new ArrayList<>(providers); } - private static void log(String type, AuthorizeCallback ac, NameCallback nc, PasswordCallback pc, RealmCallback rc) { + private static void log(String type, AuthorizeCallback ac, NameCallback nc, PasswordCallback pc, + RealmCallback rc) { if (LOG.isDebugEnabled()) { String acs = "null"; if (ac != null) { - acs = "athz: " + ac.getAuthorizationID() + " athn: " + ac.getAuthenticationID() + " authorized: " + ac.getAuthorizedID(); + acs = "athz: " + ac.getAuthorizationID() + " athn: " + ac.getAuthenticationID() + + " authorized: " + ac.getAuthorizedID(); } String ncs = "null"; @@ -92,18 +105,22 @@ private Pair translateName(String orig) { return Pair.of(ret, provider.isImpersonationAllowed()); } } catch (Exception e) { - //Translating the name (this call) happens in a different callback from validating + // Translating the name (this call) happens in a different callback from validating // the user name and password. This has to be stateless though, so we cannot save - // the password provider away to be sure we got the same one that validated the password. + // the password provider away to be sure we got the same one that validated the + // password. // If the password providers are written correctly this should never happen, // because if they cannot read the name they would return a null. - // But on the off chance that something goes wrong with the translation because of a mismatch + // But on the off chance that something goes wrong with the translation because of a + // mismatch // we try to skip the bad one. LOG.debug("{} could not read name from {}", provider, orig, e); } } - // In the worst case we will return a serialized name after a password provider said that the password - // was okay. In that case the ACLs are likely to prevent the request from going through anyways. + // In the worst case we will return a serialized name after a password provider said that + // the password + // was okay. In that case the ACLs are likely to prevent the request from going through + // anyways. // But that is only if there is a bug in one of the password providers. return Pair.of(orig, false); } @@ -168,10 +185,11 @@ public void handle(Callback[] callbacks) throws UnsupportedCallbackException, IO zid = tmp.getFirst(); allowImpersonation = allowImpersonation && tmp.getSecond(); } - LOG.debug("Successfully authenticated client: authenticationID = {} authorizationID = {}", + LOG.debug("Successfully authenticated client: authenticationID = {} authorizationID = " + + "{}", nid, zid); - //if authorizationId is not set, set it to authenticationId. + // if authorizationId is not set, set it to authenticationId. if (zid == null) { ac.setAuthorizedID(nid); zid = nid; @@ -179,13 +197,15 @@ public void handle(Callback[] callbacks) throws UnsupportedCallbackException, IO ac.setAuthorizedID(zid); } - //When nid and zid are not equal, nid is attempting to impersonate zid, We - //add the nid as the real user in reqContext's subject which will be used during authorization. + // When nid and zid are not equal, nid is attempting to impersonate zid, We + // add the nid as the real user in reqContext's subject which will be used during + // authorization. if (!Objects.equals(nid, zid)) { LOG.info("Impersonation attempt authenticationID = {} authorizationID = {}", nid, zid); if (!allowImpersonation) { - throw new IllegalArgumentException(ac.getAuthenticationID() + " attempting to impersonate " + ac.getAuthorizationID() + throw new IllegalArgumentException(ac.getAuthenticationID() + + " attempting to impersonate " + ac.getAuthorizationID() + ". This is not allowed."); } ReqContext.context().setRealPrincipal(new SaslTransportPlugin.User(nid)); diff --git a/storm-client/src/jvm/org/apache/storm/security/auth/tls/ReloadableTsslTransportFactory.java b/storm-client/src/jvm/org/apache/storm/security/auth/tls/ReloadableTsslTransportFactory.java index 34c9534ec21..49d97c6e3b5 100644 --- a/storm-client/src/jvm/org/apache/storm/security/auth/tls/ReloadableTsslTransportFactory.java +++ b/storm-client/src/jvm/org/apache/storm/security/auth/tls/ReloadableTsslTransportFactory.java @@ -1,16 +1,21 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE * file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the * Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of * the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed * on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language * governing permissions * and limitations under the License. */ @@ -40,31 +45,38 @@ public class ReloadableTsslTransportFactory extends TSSLTransportFactory { public static TServerSocket getServerSocket(int port, int clientTimeout, InetAddress ifAddress, ThriftConnectionType type, Map conf) throws Exception { SSLContext ctx = createSslContext(type, conf); - return createServerSocket(ctx.getServerSocketFactory(), port, clientTimeout, type.isClientAuthRequired(conf), + return createServerSocket(ctx.getServerSocketFactory(), port, clientTimeout, type + .isClientAuthRequired(conf), ifAddress, type); } - private static SSLContext createSslContext(ThriftConnectionType type, Map conf) throws Exception { - X509TrustManager trustManager = new ReloadableX509TrustManager(type.getServerTrustStorePath(conf), + private static SSLContext createSslContext(ThriftConnectionType type, Map conf) throws Exception { + X509TrustManager trustManager = new ReloadableX509TrustManager(type + .getServerTrustStorePath(conf), type.getServerTrustStorePassword(conf)); X509KeyManager keyManager = new ReloadableX509KeyManager(type.getServerKeyStorePath(conf), type.getServerKeyStorePassword(conf)); SSLContext ctx = SSLContext.getInstance("TLSv1.2"); - ctx.init(new KeyManager[]{keyManager}, new TrustManager[]{trustManager}, new SecureRandom()); + ctx.init(new KeyManager[]{keyManager}, new TrustManager[]{trustManager}, + new SecureRandom()); return ctx; } - private static TServerSocket createServerSocket(SSLServerSocketFactory factory, int port, int timeout, + private static TServerSocket createServerSocket(SSLServerSocketFactory factory, int port, + int timeout, boolean clientAuth, InetAddress ifAddress, ThriftConnectionType type) throws TTransportException { try { - SSLServerSocket serverSocket = (SSLServerSocket) factory.createServerSocket(port, 100, ifAddress); + SSLServerSocket serverSocket = (SSLServerSocket) factory.createServerSocket(port, 100, + ifAddress); serverSocket.setEnabledProtocols(new String[]{"TLSv1.2"}); serverSocket.setSoTimeout(timeout); serverSocket.setNeedClientAuth(clientAuth); - return new TServerSocket(new TServerSocket.ServerSocketTransportArgs().serverSocket(serverSocket).clientTimeout(timeout)); + return new TServerSocket(new TServerSocket.ServerSocketTransportArgs() + .serverSocket(serverSocket).clientTimeout(timeout)); } catch (Exception e) { throw new TTransportException("Could not bind to port " + port, e); } diff --git a/storm-client/src/jvm/org/apache/storm/security/auth/tls/ReloadableX509KeyManager.java b/storm-client/src/jvm/org/apache/storm/security/auth/tls/ReloadableX509KeyManager.java index 6afde704468..07f83351e19 100644 --- a/storm-client/src/jvm/org/apache/storm/security/auth/tls/ReloadableX509KeyManager.java +++ b/storm-client/src/jvm/org/apache/storm/security/auth/tls/ReloadableX509KeyManager.java @@ -1,16 +1,22 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE * file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the * Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of * the License at - *

    - * http://www.apache.org/licenses/LICENSE-2.0 - *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed + * + *

    http://www.apache.org/licenses/LICENSE-2.0 + * + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed * on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language * governing permissions * and limitations under the License. */ @@ -48,18 +54,22 @@ public class ReloadableX509KeyManager implements X509KeyManager { private volatile X509KeyManager keyManager; public ReloadableX509KeyManager(String keystorePath, String keystorePassword) throws Exception { - keyManager = createKeyManager(getKeyStore(keystorePath, keystorePassword), keystorePassword); + keyManager = createKeyManager(getKeyStore(keystorePath, keystorePassword), + keystorePassword); FileWatcher.Callback keyStoreWatcherCallback = () -> { reloadCert(keystorePath, keystorePassword); }; - FileWatcher keyStoreWatcher = new FileWatcher(Paths.get(keystorePath), keyStoreWatcherCallback); + FileWatcher keyStoreWatcher = new FileWatcher(Paths.get(keystorePath), + keyStoreWatcherCallback); keyStoreWatcher.start(); } - public X509KeyManager createKeyManager(KeyStore keystore, String keystorePassword) throws Exception { + public X509KeyManager createKeyManager(KeyStore keystore, + String keystorePassword) throws Exception { // Load the keystore - KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); + KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory + .getDefaultAlgorithm()); kmf.init(keystore, keystorePassword.toCharArray()); X509KeyManager x509KeyManager = null; @@ -112,12 +122,13 @@ public PrivateKey getPrivateKey(String s) { private synchronized void reloadCert(String keystorePath, String keystorePassword) { try { LOG.info("Reloading KeyManager"); - keyManager = createKeyManager(getKeyStore(keystorePath, keystorePassword), keystorePassword); + keyManager = createKeyManager(getKeyStore(keystorePath, keystorePassword), + keystorePassword); LOG.info("Reloading KeyManager - Done"); } catch (Exception e) { LOG.error("Error reloading KeyManager. Setting keyManager to null", e); keyManager = null; - //on error set keyManager to null + // on error set keyManager to null } } diff --git a/storm-client/src/jvm/org/apache/storm/security/auth/tls/ReloadableX509TrustManager.java b/storm-client/src/jvm/org/apache/storm/security/auth/tls/ReloadableX509TrustManager.java index e78e11afdf4..9776582352a 100644 --- a/storm-client/src/jvm/org/apache/storm/security/auth/tls/ReloadableX509TrustManager.java +++ b/storm-client/src/jvm/org/apache/storm/security/auth/tls/ReloadableX509TrustManager.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

    - * http://www.apache.org/licenses/LICENSE-2.0 - *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

    http://www.apache.org/licenses/LICENSE-2.0 + * + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -34,41 +40,49 @@ public class ReloadableX509TrustManager extends X509ExtendedTrustManager { private static final X509Certificate[] EMPTY = new X509Certificate[0]; private volatile X509ExtendedTrustManager trustManager; - public ReloadableX509TrustManager(String trustStorePath, String trustStorePassword) throws Exception { + public ReloadableX509TrustManager(String trustStorePath, + String trustStorePassword) throws Exception { trustManager = createTrustManager(trustStorePath, trustStorePassword); FileWatcher.Callback keyStoreWatcherCallback = () -> reloadCert(trustStorePath, trustStorePassword); - FileWatcher keyStoreWatcher = new FileWatcher(Paths.get(trustStorePath), keyStoreWatcherCallback); + FileWatcher keyStoreWatcher = new FileWatcher(Paths.get(trustStorePath), + keyStoreWatcherCallback); keyStoreWatcher.start(); } @Override - public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException { + public void checkClientTrusted(X509Certificate[] chain, + String authType) throws CertificateException { trustManager.checkClientTrusted(chain, authType); } @Override - public void checkClientTrusted(X509Certificate[] x509Certificates, String s, Socket socket) throws CertificateException { + public void checkClientTrusted(X509Certificate[] x509Certificates, String s, + Socket socket) throws CertificateException { trustManager.checkClientTrusted(x509Certificates, s, socket); } @Override - public void checkClientTrusted(X509Certificate[] x509Certificates, String s, SSLEngine sslEngine) throws CertificateException { + public void checkClientTrusted(X509Certificate[] x509Certificates, String s, + SSLEngine sslEngine) throws CertificateException { trustManager.checkClientTrusted(x509Certificates, s, sslEngine); } @Override - public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException { + public void checkServerTrusted(X509Certificate[] chain, + String authType) throws CertificateException { trustManager.checkServerTrusted(chain, authType); } @Override - public void checkServerTrusted(X509Certificate[] x509Certificates, String s, Socket socket) throws CertificateException { + public void checkServerTrusted(X509Certificate[] x509Certificates, String s, + Socket socket) throws CertificateException { trustManager.checkServerTrusted(x509Certificates, s, socket); } @Override - public void checkServerTrusted(X509Certificate[] x509Certificates, String s, SSLEngine sslEngine) throws CertificateException { + public void checkServerTrusted(X509Certificate[] x509Certificates, String s, + SSLEngine sslEngine) throws CertificateException { trustManager.checkServerTrusted(x509Certificates, s, sslEngine); } @@ -77,8 +91,8 @@ public X509Certificate[] getAcceptedIssuers() { return trustManager.getAcceptedIssuers(); } - - public X509ExtendedTrustManager createTrustManager(String trustStorePath, String keystorePassword) throws Exception { + public X509ExtendedTrustManager createTrustManager(String trustStorePath, + String keystorePassword) throws Exception { LOG.info(" createTrustManager trustStorePath {}", trustStorePath); KeyStore keystore = KeyStore.getInstance(KeyStore.getDefaultType()); @@ -86,7 +100,8 @@ public X509ExtendedTrustManager createTrustManager(String trustStorePath, String keystore.load(keystoreStream, keystorePassword.toCharArray()); } - TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory + .getDefaultAlgorithm()); tmf.init(keystore); X509ExtendedTrustManager x509TrustManager = null; diff --git a/storm-client/src/jvm/org/apache/storm/security/auth/tls/TlsTransportPlugin.java b/storm-client/src/jvm/org/apache/storm/security/auth/tls/TlsTransportPlugin.java index 211cd7bdcd6..041ad9c1e54 100644 --- a/storm-client/src/jvm/org/apache/storm/security/auth/tls/TlsTransportPlugin.java +++ b/storm-client/src/jvm/org/apache/storm/security/auth/tls/TlsTransportPlugin.java @@ -1,12 +1,17 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -70,12 +75,14 @@ public TServer getServer(TProcessor processor) throws IOException, TTransportExc int configuredPort = type.getPort(conf); Integer socketTimeout = type.getSocketTimeOut(conf); - if (type.getServerKeyStorePath(conf) == null || type.getServerKeyStorePassword(conf) == null) { + if (type.getServerKeyStorePath(conf) == null || type + .getServerKeyStorePassword(conf) == null) { throw new IllegalArgumentException("The server keystore is not configured properly"); } if (type.isClientAuthRequired(conf) - && (type.getServerTrustStorePath(conf) == null || type.getServerTrustStorePassword(conf) == null)) { + && (type.getServerTrustStorePath(conf) == null || type + .getServerTrustStorePassword(conf) == null)) { throw new IllegalArgumentException("The server truststore is not configured properly"); } @@ -83,7 +90,8 @@ public TServer getServer(TProcessor processor) throws IOException, TTransportExc TServerSocket serverTransport = null; try { - serverTransport = ReloadableTsslTransportFactory.getServerSocket(configuredPort, clientTimeout, + serverTransport = ReloadableTsslTransportFactory.getServerSocket(configuredPort, + clientTimeout, InetAddress.getLocalHost(), type, conf); } catch (Exception e) { throw new IOException(e); @@ -106,7 +114,8 @@ public TServer getServer(TProcessor processor) throws IOException, TTransportExc if (queueSize != null) { workQueue = new ArrayBlockingQueue<>(queueSize); } - ThreadPoolExecutor executorService = new ExtendedThreadPoolExecutor(numWorkerThreads, numWorkerThreads, + ThreadPoolExecutor executorService = new ExtendedThreadPoolExecutor(numWorkerThreads, + numWorkerThreads, 60, TimeUnit.SECONDS, workQueue); serverArgs.executorService(executorService); tThreadPoolServer = new TThreadPoolServer(serverArgs); @@ -114,7 +123,8 @@ public TServer getServer(TProcessor processor) throws IOException, TTransportExc } @Override - public TTransport connect(TTransport transport, String serverHost, String asUser) throws IOException, TTransportException { + public TTransport connect(TTransport transport, String serverHost, + String asUser) throws IOException, TTransportException { return transport; } @@ -148,7 +158,8 @@ public void process(final TProtocol inProt, final TProtocol outProt) throws TExc try { Certificate[] peers = socket.getSession().getPeerCertificates(); if (peers.length > 0 && peers[0] instanceof X509Certificate) { - principalName = ((X509Certificate) peers[0]).getSubjectX500Principal().getName(); + principalName = ((X509Certificate) peers[0]).getSubjectX500Principal() + .getName(); } else if (clientAuthRequired) { throw new TException("TLS peer presented no X.509 certificate"); } @@ -161,15 +172,16 @@ public void process(final TProtocol inProt, final TProtocol outProt) throws TExc socket.getInetAddress()); throw new TException("TLS peer not verified", e); } - LOG.debug("Client cert not presented; clientAuthRequired=false, using {}", principalName); + LOG.debug("Client cert not presented; clientAuthRequired=false, using {}", + principalName); } LOG.debug("principalName : {} ", principalName); ReqContext reqContext = ReqContext.context(); - //remote address + // remote address reqContext.setRemoteAddress(socket.getInetAddress()); - //remote subject + // remote subject Subject remoteUser = new Subject(); remoteUser.getPrincipals().add(new SingleUserPrincipal(principalName)); reqContext.setSubject(remoteUser); diff --git a/storm-client/src/jvm/org/apache/storm/security/auth/workertoken/WorkerTokenAuthorizer.java b/storm-client/src/jvm/org/apache/storm/security/auth/workertoken/WorkerTokenAuthorizer.java index 80f3d543ec0..daba572b0f6 100644 --- a/storm-client/src/jvm/org/apache/storm/security/auth/workertoken/WorkerTokenAuthorizer.java +++ b/storm-client/src/jvm/org/apache/storm/security/auth/workertoken/WorkerTokenAuthorizer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -58,7 +64,8 @@ public WorkerTokenAuthorizer(Map conf, ThriftConnectionType conn } @VisibleForTesting - WorkerTokenAuthorizer(final WorkerTokenServiceType serviceType, final IStormClusterState state) { + WorkerTokenAuthorizer(final WorkerTokenServiceType serviceType, + final IStormClusterState state) { LoadingCache tmpKeyCache = null; if (state != null) { tmpKeyCache = @@ -79,12 +86,14 @@ public PrivateWorkerKey load(WorkerTokenInfo wtInfo) { this.state = state; } - private static IStormClusterState buildStateIfNeeded(Map conf, ThriftConnectionType connectionType) { + private static IStormClusterState buildStateIfNeeded(Map conf, + ThriftConnectionType connectionType) { IStormClusterState state = null; if (ClientAuthUtils.areWorkerTokensEnabledServer(connectionType, conf)) { try { - state = ClusterUtils.mkStormClusterState(conf, new ClusterStateContext(DaemonType.UNKNOWN, conf)); + state = ClusterUtils.mkStormClusterState(conf, + new ClusterStateContext(DaemonType.UNKNOWN, conf)); } catch (Exception e) { throw new RuntimeException(e); } @@ -96,7 +105,8 @@ private static IStormClusterState buildStateIfNeeded(Map conf, T byte[] getSignedPasswordFor(byte[] user, WorkerTokenInfo deser) { assert keyCache != null; - if (deser.is_set_expirationTimeMillis() && deser.get_expirationTimeMillis() <= Time.currentTimeMillis()) { + if (deser.is_set_expirationTimeMillis() && deser.get_expirationTimeMillis() <= Time + .currentTimeMillis()) { throw new IllegalArgumentException("Token is not valid, token has expired."); } @@ -104,16 +114,19 @@ byte[] getSignedPasswordFor(byte[] user, WorkerTokenInfo deser) { try { key = keyCache.getUnchecked(deser); } catch (CacheLoader.InvalidCacheLoadException e) { - //This happens when the key is not found, the cache loader returns a null and this exception is thrown. + // This happens when the key is not found, the cache loader returns a null and this + // exception is thrown. // because the cache cannot store a null. throw new IllegalArgumentException("Token is not valid, private key not found.", e); } - if (key.is_set_expirationTimeMillis() && key.get_expirationTimeMillis() <= Time.currentTimeMillis()) { + if (key.is_set_expirationTimeMillis() && key.get_expirationTimeMillis() <= Time + .currentTimeMillis()) { throw new IllegalArgumentException("Token is not valid, key has expired."); } - return WorkerTokenSigner.createPassword(user, new SecretKeySpec(key.get_key(), WorkerTokenSigner.DEFAULT_HMAC_ALGORITHM)); + return WorkerTokenSigner.createPassword(user, new SecretKeySpec(key.get_key(), + WorkerTokenSigner.DEFAULT_HMAC_ALGORITHM)); } @Override @@ -136,7 +149,8 @@ public Optional getPasswordFor(String userName) { return Optional.of(Base64.getEncoder().encodeToString(password).toCharArray()); } catch (Exception e) { passwordFailures.mark(); - LOG.error("Could not get password for token {}/{}", deser.get_userName(), deser.get_topologyId(), e); + LOG.error("Could not get password for token {}/{}", deser.get_userName(), deser + .get_topologyId(), e); return Optional.empty(); } } diff --git a/storm-client/src/jvm/org/apache/storm/security/auth/workertoken/WorkerTokenClientCallbackHandler.java b/storm-client/src/jvm/org/apache/storm/security/auth/workertoken/WorkerTokenClientCallbackHandler.java index d85ff0382c9..1974f4bf28e 100644 --- a/storm-client/src/jvm/org/apache/storm/security/auth/workertoken/WorkerTokenClientCallbackHandler.java +++ b/storm-client/src/jvm/org/apache/storm/security/auth/workertoken/WorkerTokenClientCallbackHandler.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -22,16 +28,20 @@ import org.apache.storm.security.auth.sasl.SimpleSaslClientCallbackHandler; /** - * A Client callback handler for a WorkerToken. In general a client that wants to support worker tokens should first check if a WorkerToken - * is available for the specific connection type by calling findWorkerTokenInSubject. If that returns a token, then proceed to create and - * use this with a DIGEST-MD5 SaslClient. If not you should fall back to whatever other client auth you want to do. + * A Client callback handler for a WorkerToken. In general a client that wants to support worker + * tokens should first check if a WorkerToken + * is available for the specific connection type by calling findWorkerTokenInSubject. If that + * returns a token, then proceed to create and + * use this with a DIGEST-MD5 SaslClient. If not you should fall back to whatever other client auth + * you want to do. */ public class WorkerTokenClientCallbackHandler extends SimpleSaslClientCallbackHandler { /** * Constructor. * - * @param token the token to use to authenticate. This was probably retrieved by calling findWorkerTokenInSubject. + * @param token the token to use to authenticate. This was probably retrieved by calling + * findWorkerTokenInSubject. */ public WorkerTokenClientCallbackHandler(WorkerToken token) { super(Base64.getEncoder().encodeToString(token.get_info()), @@ -39,7 +49,8 @@ public WorkerTokenClientCallbackHandler(WorkerToken token) { } /** - * Look in the current subject for a WorkerToken. This should really only happen when we are in a worker, because the tokens will not + * Look in the current subject for a WorkerToken. This should really only happen when we are in + * a worker, because the tokens will not * be placed in anything else. * * @param type the type of connection we need a token for. diff --git a/storm-client/src/jvm/org/apache/storm/security/auth/workertoken/WorkerTokenSigner.java b/storm-client/src/jvm/org/apache/storm/security/auth/workertoken/WorkerTokenSigner.java index 0c780f6fbd9..5204849710e 100644 --- a/storm-client/src/jvm/org/apache/storm/security/auth/workertoken/WorkerTokenSigner.java +++ b/storm-client/src/jvm/org/apache/storm/security/auth/workertoken/WorkerTokenSigner.java @@ -40,7 +40,8 @@ class WorkerTokenSigner { try { return Mac.getInstance(DEFAULT_HMAC_ALGORITHM); } catch (NoSuchAlgorithmException nsa) { - throw new IllegalArgumentException("Can't find " + DEFAULT_HMAC_ALGORITHM + " algorithm."); + throw new IllegalArgumentException("Can't find " + DEFAULT_HMAC_ALGORITHM + + " algorithm."); } }); diff --git a/storm-client/src/jvm/org/apache/storm/security/serialization/BlowfishTupleSerializer.java b/storm-client/src/jvm/org/apache/storm/security/serialization/BlowfishTupleSerializer.java index 185817b8225..f8dac2e3a82 100644 --- a/storm-client/src/jvm/org/apache/storm/security/serialization/BlowfishTupleSerializer.java +++ b/storm-client/src/jvm/org/apache/storm/security/serialization/BlowfishTupleSerializer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -31,16 +37,19 @@ /** * Apply Blowfish encryption for tuple communication to bolts. * - * @deprecated since 2.8.6. Blowfish uses a 64-bit block size which is vulnerable to birthday attacks (Sweet32). + * @deprecated since 2.8.6. Blowfish uses a 64-bit block size which is vulnerable to birthday + * attacks (Sweet32). * Use TLS-based transport encryption instead (see storm.messaging.netty.tls.enable). */ @Deprecated(since = "2.8.6", forRemoval = true) public class BlowfishTupleSerializer extends Serializer { /** - * The secret key (if any) for data encryption by blowfish payload serialization factory (BlowfishSerializationFactory). You should use + * The secret key (if any) for data encryption by blowfish payload serialization factory + * (BlowfishSerializationFactory). You should use * in via: * - *

    ```storm -c topology.tuple.serializer.blowfish.key=YOURKEY -c topology.tuple.serializer=org.apache.storm.security.serialization + *

    ```storm -c topology.tuple.serializer.blowfish.key=YOURKEY -c + * topology.tuple.serializer=org.apache.storm.security.serialization * .BlowfishTupleSerializer * jar ...``` */ @@ -52,16 +61,20 @@ public BlowfishTupleSerializer(Kryo unused, Map topoConf) { String encryptionkey; try { encryptionkey = (String) topoConf.get(SECRET_KEY); - LOG.warn("BlowfishTupleSerializer is deprecated and will be removed in a future release. " - + "Blowfish uses a 64-bit block size which is vulnerable to birthday attacks (Sweet32). " - + "Use TLS-based transport encryption instead (storm.messaging.netty.tls.enable)."); + LOG.warn("BlowfishTupleSerializer is deprecated and will be removed in a future " + + "release. " + + "Blowfish uses a 64-bit block size which is vulnerable to birthday attacks " + + "(Sweet32). " + + "Use TLS-based transport encryption instead " + + "(storm.messaging.netty.tls.enable)."); byte[] bytes; if (encryptionkey != null) { bytes = Hex.decodeHex(encryptionkey.toCharArray()); } else { // try to use zookeeper secret - String payload = (String) topoConf.get(Config.STORM_ZOOKEEPER_TOPOLOGY_AUTH_PAYLOAD); + String payload = (String) topoConf + .get(Config.STORM_ZOOKEEPER_TOPOLOGY_AUTH_PAYLOAD); if (payload != null) { LOG.debug("{} is not present. Use {} as Blowfish encryption key", SECRET_KEY, Config.STORM_ZOOKEEPER_TOPOLOGY_AUTH_PAYLOAD); diff --git a/storm-client/src/jvm/org/apache/storm/serialization/DefaultKryoFactory.java b/storm-client/src/jvm/org/apache/storm/serialization/DefaultKryoFactory.java index 6ed3be28273..4b3a348e7f1 100644 --- a/storm-client/src/jvm/org/apache/storm/serialization/DefaultKryoFactory.java +++ b/storm-client/src/jvm/org/apache/storm/serialization/DefaultKryoFactory.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -21,31 +27,35 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; - public class DefaultKryoFactory implements IKryoFactory { private static final Logger LOG = LoggerFactory.getLogger(DefaultKryoFactory.class); @Override public Kryo getKryo(Map conf) { KryoSerializableDefault k = new KryoSerializableDefault(getJavaSerializationFilter(conf)); - k.setRegistrationRequired(!((Boolean) conf.get(Config.TOPOLOGY_FALL_BACK_ON_JAVA_SERIALIZATION))); + k.setRegistrationRequired(!((Boolean) conf + .get(Config.TOPOLOGY_FALL_BACK_ON_JAVA_SERIALIZATION))); k.setReferences(false); return k; } /** - * Parses the pattern once at kryo construction so an invalid pattern fails worker setup with a clear error - * instead of failing per-tuple on the read path. Returns null when the key is unset or empty (no filter). + * Parses the pattern once at kryo construction so an invalid pattern fails worker setup with a + * clear error + * instead of failing per-tuple on the read path. Returns null when the key is unset or empty + * (no filter). */ private static ObjectInputFilter getJavaSerializationFilter(Map conf) { - String filterSpec = (String) conf.get(Config.TOPOLOGY_FALL_BACK_ON_JAVA_SERIALIZATION_FILTER); + String filterSpec = (String) conf + .get(Config.TOPOLOGY_FALL_BACK_ON_JAVA_SERIALIZATION_FILTER); if (filterSpec == null || filterSpec.isEmpty()) { return null; } try { return ObjectInputFilter.Config.createFilter(filterSpec); } catch (IllegalArgumentException e) { - throw new IllegalArgumentException("Invalid " + Config.TOPOLOGY_FALL_BACK_ON_JAVA_SERIALIZATION_FILTER + throw new IllegalArgumentException("Invalid " + + Config.TOPOLOGY_FALL_BACK_ON_JAVA_SERIALIZATION_FILTER + " pattern: \"" + filterSpec + "\"", e); } } diff --git a/storm-client/src/jvm/org/apache/storm/serialization/GzipBridgeThriftSerializationDelegate.java b/storm-client/src/jvm/org/apache/storm/serialization/GzipBridgeThriftSerializationDelegate.java index 67ae15ada00..b87cbdd1b23 100644 --- a/storm-client/src/jvm/org/apache/storm/serialization/GzipBridgeThriftSerializationDelegate.java +++ b/storm-client/src/jvm/org/apache/storm/serialization/GzipBridgeThriftSerializationDelegate.java @@ -1,12 +1,17 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -16,8 +21,10 @@ import java.util.zip.GZIPInputStream; /** - * Always writes gzip out, but tests incoming to see if it's gzipped. If it is, deserializes with gzip. If not, uses {@link - * org.apache.storm.serialization.ThriftSerializationDelegate} to deserialize. Any logic needing to be enabled via {@link + * Always writes gzip out, but tests incoming to see if it's gzipped. If it is, deserializes with + * gzip. If not, uses {@link + * org.apache.storm.serialization.ThriftSerializationDelegate} to deserialize. Any logic needing to + * be enabled via {@link * #prepare(java.util.Map)} is passed through to both delegates. */ public class GzipBridgeThriftSerializationDelegate implements SerializationDelegate { diff --git a/storm-client/src/jvm/org/apache/storm/serialization/GzipSerializationDelegate.java b/storm-client/src/jvm/org/apache/storm/serialization/GzipSerializationDelegate.java index d12c2b01879..b2624ed9a9a 100644 --- a/storm-client/src/jvm/org/apache/storm/serialization/GzipSerializationDelegate.java +++ b/storm-client/src/jvm/org/apache/storm/serialization/GzipSerializationDelegate.java @@ -1,12 +1,17 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -25,7 +30,8 @@ import org.apache.storm.utils.ObjectReader; /** - * Note, this assumes it's deserializing a gzip byte stream, and will err if it encounters any other serialization. + * Note, this assumes it's deserializing a gzip byte stream, and will err if it encounters any other + * serialization. */ public class GzipSerializationDelegate implements SerializationDelegate { @@ -34,7 +40,8 @@ public class GzipSerializationDelegate implements SerializationDelegate { @Override public void prepare(Map topoConf) { - this.maxDecompressedBytes = ObjectReader.getInt(topoConf.getOrDefault(Config.STORM_COMPRESSION_GZIP_MAX_DECOMPRESSED_BYTES, + this.maxDecompressedBytes = ObjectReader.getInt(topoConf + .getOrDefault(Config.STORM_COMPRESSION_GZIP_MAX_DECOMPRESSED_BYTES, DEFAULT_MAX_DECOMPRESSED_BYTES)); } @@ -64,7 +71,8 @@ public T deserialize(byte[] bytes, Class clazz) { ObjectInputStream ois = new ObjectInputStream(lis)) { Object ret = ois.readObject(); if (gis.read() != -1) { - throw new IOException("Decompression threshold exceeded! Possible security risk or invalid data size."); + throw new IOException("Decompression threshold exceeded! Possible security risk " + + "or invalid data size."); } return clazz.cast(ret); } catch (IOException | ClassNotFoundException e) { diff --git a/storm-client/src/jvm/org/apache/storm/serialization/GzipThriftSerializationDelegate.java b/storm-client/src/jvm/org/apache/storm/serialization/GzipThriftSerializationDelegate.java index ae3bc7ac1cf..a21ccf8a353 100644 --- a/storm-client/src/jvm/org/apache/storm/serialization/GzipThriftSerializationDelegate.java +++ b/storm-client/src/jvm/org/apache/storm/serialization/GzipThriftSerializationDelegate.java @@ -28,7 +28,8 @@ import org.apache.storm.utils.Utils; /** - * Note, this assumes it's deserializing a gzip byte stream, and will err if it encounters any other serialization. + * Note, this assumes it's deserializing a gzip byte stream, and will err if it encounters any other + * serialization. */ public class GzipThriftSerializationDelegate implements SerializationDelegate { @@ -37,7 +38,8 @@ public class GzipThriftSerializationDelegate implements SerializationDelegate { @Override public void prepare(Map topoConf) { - this.maxDecompressedBytes = ObjectReader.getInt(topoConf.getOrDefault(Config.STORM_COMPRESSION_GZIP_MAX_DECOMPRESSED_BYTES, + this.maxDecompressedBytes = ObjectReader.getInt(topoConf + .getOrDefault(Config.STORM_COMPRESSION_GZIP_MAX_DECOMPRESSED_BYTES, DEFAULT_MAX_DECOMPRESSED_BYTES)); } @@ -54,7 +56,8 @@ public byte[] serialize(Object object) { public T deserialize(byte[] bytes, Class clazz) { try { TBase instance = (TBase) clazz.newInstance(); - new TDeserializer().deserialize(instance, Utils.GzipUtils.decompress(bytes, this.maxDecompressedBytes)); + new TDeserializer().deserialize(instance, Utils.GzipUtils.decompress(bytes, + this.maxDecompressedBytes)); return (T) instance; } catch (Exception e) { throw new RuntimeException(e); diff --git a/storm-client/src/jvm/org/apache/storm/serialization/IKryoDecorator.java b/storm-client/src/jvm/org/apache/storm/serialization/IKryoDecorator.java index 2d6dc989823..2a04bde6c9c 100644 --- a/storm-client/src/jvm/org/apache/storm/serialization/IKryoDecorator.java +++ b/storm-client/src/jvm/org/apache/storm/serialization/IKryoDecorator.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -18,6 +24,7 @@ public interface IKryoDecorator { /** * Decorate the Kryo instance with custom serializations. + * * @deprecated use {@link #decorate(Kryo, Map)} instead. */ @Deprecated @@ -26,6 +33,7 @@ default void decorate(Kryo k) { /** * Decorate the Kryo instance with custom serializations. + * * @param k the Kryo instance to decorate * @param conf the topology configuration */ diff --git a/storm-client/src/jvm/org/apache/storm/serialization/IKryoFactory.java b/storm-client/src/jvm/org/apache/storm/serialization/IKryoFactory.java index 3d452b9528b..9e6521b5caa 100644 --- a/storm-client/src/jvm/org/apache/storm/serialization/IKryoFactory.java +++ b/storm-client/src/jvm/org/apache/storm/serialization/IKryoFactory.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/serialization/ITupleDeserializer.java b/storm-client/src/jvm/org/apache/storm/serialization/ITupleDeserializer.java index 94e2cfd9682..cf6c1b62caa 100644 --- a/storm-client/src/jvm/org/apache/storm/serialization/ITupleDeserializer.java +++ b/storm-client/src/jvm/org/apache/storm/serialization/ITupleDeserializer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/serialization/ITupleSerializer.java b/storm-client/src/jvm/org/apache/storm/serialization/ITupleSerializer.java index 2ee0335c8a0..96cc60d7161 100644 --- a/storm-client/src/jvm/org/apache/storm/serialization/ITupleSerializer.java +++ b/storm-client/src/jvm/org/apache/storm/serialization/ITupleSerializer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -14,7 +20,6 @@ import org.apache.storm.tuple.Tuple; - public interface ITupleSerializer { byte[] serialize(Tuple tuple); // long crc32(Tuple tuple); diff --git a/storm-client/src/jvm/org/apache/storm/serialization/KryoTupleDeserializer.java b/storm-client/src/jvm/org/apache/storm/serialization/KryoTupleDeserializer.java index 301a8ca96de..8601708484b 100644 --- a/storm-client/src/jvm/org/apache/storm/serialization/KryoTupleDeserializer.java +++ b/storm-client/src/jvm/org/apache/storm/serialization/KryoTupleDeserializer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -37,12 +43,14 @@ public class KryoTupleDeserializer implements ITupleDeserializer { private final int maxZstdDecompressedBytes; private final boolean anyTupleCompressionEnabled; - public KryoTupleDeserializer(final Map conf, final GeneralTopologyContext context) { + public KryoTupleDeserializer(final Map conf, + final GeneralTopologyContext context) { kryo = new KryoValuesDeserializer(conf); this.context = context; ids = new SerializationFactory.IdDictionary(context.getRawTopology()); kryoInput = new Input(1); - maxZstdDecompressedBytes = ObjectReader.getInt(conf.get(Config.TOPOLOGY_TUPLE_COMPRESSION_MAX_DECOMPRESSED_BYTES), + maxZstdDecompressedBytes = ObjectReader.getInt(conf + .get(Config.TOPOLOGY_TUPLE_COMPRESSION_MAX_DECOMPRESSED_BYTES), DEFAULT_MAX_DECOMPRESSED_BYTES); anyTupleCompressionEnabled = isTupleCompressionEnabled(conf, context); } @@ -52,17 +60,24 @@ public TupleImpl deserialize(byte[] ser) { // check zstd header if at least one component is compressing tuples. if (anyTupleCompressionEnabled && Utils.ZstdUtils.isZstd(ser)) { try { - byte[] decompressed = Utils.ZstdUtils.decompress(ser, this.maxZstdDecompressedBytes); + byte[] decompressed = Utils.ZstdUtils.decompress(ser, + this.maxZstdDecompressedBytes); return deserializeTuple(decompressed); } catch (RuntimeException e) { - if (e.getMessage() != null && e.getMessage().contains(FAILED_TO_DESERIALIZE_TUPLE)) { - // isZstd() false positive: a raw Kryo tuple's first 4 bytes matched ZSTD_MAGIC_HEADER by chance. - // This is astronomically unlikely in practice. Because ZSTD_MAGIC_HEADER (0xFD2FB528) is little-endian - // on the wire, the first byte checked is 0x28. A Kryo writeInt(taskId, true) of 40 yields exactly 0x28. - // The collision is prevented not by the taskId range, but by the second field (streamId), + if (e.getMessage() != null && e.getMessage() + .contains(FAILED_TO_DESERIALIZE_TUPLE)) { + // isZstd() false positive: a raw Kryo tuple's first 4 bytes matched + // ZSTD_MAGIC_HEADER by chance. + // This is astronomically unlikely in practice. Because ZSTD_MAGIC_HEADER + // (0xFD2FB528) is little-endian + // on the wire, the first byte checked is 0x28. A Kryo writeInt(taskId, true) of + // 40 yields exactly 0x28. + // The collision is prevented not by the taskId range, but by the second field + // (streamId), // which would rigidly have to equal 6069 to match the remaining magic bytes. // Branch retained for correctness in case of an accidental collision. - LOG.debug("isZstd() false positive: raw Kryo tuple matched ZSTD_MAGIC_HEADER (0xFD2FB528)."); + LOG.debug("isZstd() false positive: raw Kryo tuple matched ZSTD_MAGIC_HEADER " + + "(0xFD2FB528)."); return deserializeTuple(ser); } else { throw e; @@ -90,14 +105,16 @@ private TupleImpl deserializeTuple(byte[] data) { } } - private static boolean isTupleCompressionEnabled(final Map conf, final GeneralTopologyContext context) { + private static boolean isTupleCompressionEnabled(final Map conf, + final GeneralTopologyContext context) { if (ObjectReader.getBoolean(conf.get(Config.TOPOLOGY_TUPLE_COMPRESSION_ENABLE), false)) { return true; } for (String componentId : context.getComponentIds()) { ComponentCommon common = context.getComponentCommon(componentId); Map componentConf = Utils.parseJson(common.get_json_conf()); - if (ObjectReader.getBoolean(componentConf.get(Config.TOPOLOGY_TUPLE_COMPRESSION_ENABLE), false)) { + if (ObjectReader.getBoolean(componentConf.get(Config.TOPOLOGY_TUPLE_COMPRESSION_ENABLE), + false)) { return true; } } diff --git a/storm-client/src/jvm/org/apache/storm/serialization/KryoTupleSerializer.java b/storm-client/src/jvm/org/apache/storm/serialization/KryoTupleSerializer.java index 7691faf062c..71a6cb0c328 100644 --- a/storm-client/src/jvm/org/apache/storm/serialization/KryoTupleSerializer.java +++ b/storm-client/src/jvm/org/apache/storm/serialization/KryoTupleSerializer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -33,13 +39,17 @@ public class KryoTupleSerializer implements ITupleSerializer { private final int compressionThreshold; private final int zstdCompressionLevel; - public KryoTupleSerializer(final Map conf, final GeneralTopologyContext context) { + public KryoTupleSerializer(final Map conf, + final GeneralTopologyContext context) { kryo = new KryoValuesSerializer(conf); kryoOut = new Output(2000, 2000000000); ids = new SerializationFactory.IdDictionary(context.getRawTopology()); - isCompressionEnabled = ObjectReader.getBoolean(conf.get(Config.TOPOLOGY_TUPLE_COMPRESSION_ENABLE), false); - compressionThreshold = ObjectReader.getInt(conf.get(Config.TOPOLOGY_TUPLE_COMPRESSION_THRESHOLD), DEFAULT_COMPRESSION_THRESHOLD); - zstdCompressionLevel = ObjectReader.getInt(conf.get(Config.STORM_COMPRESSION_ZSTD_LEVEL), DEFAULT_ZSTD_COMPRESSION_LEVEL); + isCompressionEnabled = ObjectReader.getBoolean(conf + .get(Config.TOPOLOGY_TUPLE_COMPRESSION_ENABLE), false); + compressionThreshold = ObjectReader.getInt(conf + .get(Config.TOPOLOGY_TUPLE_COMPRESSION_THRESHOLD), DEFAULT_COMPRESSION_THRESHOLD); + zstdCompressionLevel = ObjectReader.getInt(conf.get(Config.STORM_COMPRESSION_ZSTD_LEVEL), + DEFAULT_ZSTD_COMPRESSION_LEVEL); } @Override @@ -48,7 +58,8 @@ public byte[] serialize(Tuple tuple) { kryoOut.reset(); kryoOut.writeInt(tuple.getSourceTask(), true); - kryoOut.writeInt(ids.getStreamId(tuple.getSourceComponent(), tuple.getSourceStreamId()), true); + kryoOut.writeInt(ids.getStreamId(tuple.getSourceComponent(), tuple.getSourceStreamId()), + true); tuple.getMessageId().serialize(kryoOut); kryo.serializeInto(tuple.getValues(), kryoOut); diff --git a/storm-client/src/jvm/org/apache/storm/serialization/KryoValuesDeserializer.java b/storm-client/src/jvm/org/apache/storm/serialization/KryoValuesDeserializer.java index 873b0649815..3fad44d622a 100644 --- a/storm-client/src/jvm/org/apache/storm/serialization/KryoValuesDeserializer.java +++ b/storm-client/src/jvm/org/apache/storm/serialization/KryoValuesDeserializer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/serialization/KryoValuesSerializer.java b/storm-client/src/jvm/org/apache/storm/serialization/KryoValuesSerializer.java index 159db58456b..5d178fa78a1 100644 --- a/storm-client/src/jvm/org/apache/storm/serialization/KryoValuesSerializer.java +++ b/storm-client/src/jvm/org/apache/storm/serialization/KryoValuesSerializer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/serialization/SerializableSerializer.java b/storm-client/src/jvm/org/apache/storm/serialization/SerializableSerializer.java index 042dceade10..6bdb9d5d6f9 100644 --- a/storm-client/src/jvm/org/apache/storm/serialization/SerializableSerializer.java +++ b/storm-client/src/jvm/org/apache/storm/serialization/SerializableSerializer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -24,14 +30,16 @@ import java.io.ObjectInputStream; import java.io.ObjectOutputStream; - public class SerializableSerializer extends Serializer { /** - * Optional JEP-290 filter applied to each ObjectInputStream used for deserialization (null means unfiltered, + * Optional JEP-290 filter applied to each ObjectInputStream used for deserialization (null + * means unfiltered, * as before). The filter itself is created once from - * {@link org.apache.storm.Config#TOPOLOGY_FALL_BACK_ON_JAVA_SERIALIZATION_FILTER} by {@link DefaultKryoFactory}; - * instances returned by {@link ObjectInputFilter.Config#createFilter} are immutable and safe to share across streams. + * {@link org.apache.storm.Config#TOPOLOGY_FALL_BACK_ON_JAVA_SERIALIZATION_FILTER} by {@link + * DefaultKryoFactory}; + * instances returned by {@link ObjectInputFilter.Config#createFilter} are immutable and safe to + * share across streams. */ private final ObjectInputFilter serialFilter; @@ -64,13 +72,16 @@ public Object read(Kryo kryo, Input input, Class c) { if (len < 0) { throw new KryoException("Invalid java-serialized value length: " + len); } - // For a buffer-backed Input the remaining bytes are known (position/limit), so a declared length larger than the - // bytes actually left is refused before the new byte[len] allocation; a stream-backed Input may still deliver the + // For a buffer-backed Input the remaining bytes are known (position/limit), so a declared + // length larger than the + // bytes actually left is refused before the new byte[len] allocation; a stream-backed Input + // may still deliver the // declared bytes later, so the upper bound is not checked there. if (input.getInputStream() == null) { int remaining = input.limit() - input.position(); if (len > remaining) { - throw new KryoException("Declared java-serialized value length exceeds the input's remaining bytes " + throw new KryoException("Declared java-serialized value length exceeds the " + + "input's remaining bytes " + "(declared: " + len + ", remaining: " + remaining + ")"); } } @@ -80,7 +91,8 @@ public Object read(Kryo kryo, Input input, Class c) { try { ObjectInputStream ois = new ObjectInputStream(bis); if (serialFilter != null) { - ois.setObjectInputFilter(mergeWithExisting(serialFilter, ois.getObjectInputFilter())); + ois.setObjectInputFilter(mergeWithExisting(serialFilter, ois + .getObjectInputFilter())); } return ois.readObject(); } catch (Exception e) { @@ -91,9 +103,11 @@ public Object read(Kryo kryo, Input input, Class c) { /** * Combines the configured filter with the stream's existing filter (a JVM-wide {@code jdk.serialFilter}, if any), so both * the configured pattern and any process-wide filter apply to the stream: per JEP-290, - * {@link ObjectInputStream#setObjectInputFilter} overrides the process-wide filter for that stream unless the two are merged. + * {@link ObjectInputStream#setObjectInputFilter} overrides the process-wide filter for that + * stream unless the two are merged. */ - static ObjectInputFilter mergeWithExisting(ObjectInputFilter configured, ObjectInputFilter existing) { + static ObjectInputFilter mergeWithExisting(ObjectInputFilter configured, + ObjectInputFilter existing) { return existing != null ? ObjectInputFilter.merge(configured, existing) : configured; } } diff --git a/storm-client/src/jvm/org/apache/storm/serialization/SerializationDelegate.java b/storm-client/src/jvm/org/apache/storm/serialization/SerializationDelegate.java index d155ede9f52..610ac412781 100644 --- a/storm-client/src/jvm/org/apache/storm/serialization/SerializationDelegate.java +++ b/storm-client/src/jvm/org/apache/storm/serialization/SerializationDelegate.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/serialization/SerializationFactory.java b/storm-client/src/jvm/org/apache/storm/serialization/SerializationFactory.java index 18b471c99ff..f09aeea7bfb 100644 --- a/storm-client/src/jvm/org/apache/storm/serialization/SerializationFactory.java +++ b/storm-client/src/jvm/org/apache/storm/serialization/SerializationFactory.java @@ -50,10 +50,12 @@ public class SerializationFactory { public static final Logger LOG = LoggerFactory.getLogger(SerializationFactory.class); - public static final ServiceLoader loader = ServiceLoader.load(SerializationRegister.class); + public static final ServiceLoader loader = ServiceLoader + .load(SerializationRegister.class); public static Kryo getKryo(Map conf) { - IKryoFactory kryoFactory = (IKryoFactory) ReflectionUtils.newInstance((String) conf.get(Config.TOPOLOGY_KRYO_FACTORY)); + IKryoFactory kryoFactory = (IKryoFactory) ReflectionUtils.newInstance((String) conf + .get(Config.TOPOLOGY_KRYO_FACTORY)); Kryo k = kryoFactory.getKryo(conf); k.register(byte[].class); @@ -61,7 +63,8 @@ public static Kryo getKryo(Map conf) { String payloadSerializerName = (String) conf.get(Config.TOPOLOGY_TUPLE_SERIALIZER); try { Class serializerClass = Class.forName(payloadSerializerName); - Serializer serializer = resolveSerializerInstance(k, ListDelegate.class, serializerClass, conf); + Serializer serializer = resolveSerializerInstance(k, ListDelegate.class, + serializerClass, conf); k.register(ListDelegate.class, serializer); } catch (ClassNotFoundException ex) { throw new RuntimeException(ex); @@ -107,7 +110,8 @@ public static Kryo getKryo(Map conf) { decorator.decorate(k, conf); } catch (ClassNotFoundException e) { if (skipMissing) { - LOG.info("Could not find kryo decorator named " + klassName + ". Skipping registration..."); + LOG.info("Could not find kryo decorator named " + klassName + + ". Skipping registration..."); } else { throw new RuntimeException(e); } @@ -128,7 +132,8 @@ public static void register(Kryo k, List classesToRegister) { register(k, classesToRegister, Collections.emptyMap(), true); } - public static void register(Kryo k, Object kryoRegistrations, Map conf, boolean skipMissing) { + public static void register(Kryo k, Object kryoRegistrations, Map conf, + boolean skipMissing) { Map registrations = normalizeKryoRegister(kryoRegistrations); for (Map.Entry entry : registrations.entrySet()) { String serializerClassName = entry.getValue(); @@ -145,7 +150,8 @@ public static void register(Kryo k, Object kryoRegistrations, Map serializerClass, + private static Serializer resolveSerializerInstance(Kryo k, Class superClass, + Class serializerClass, Map conf) { try { try { - return serializerClass.getConstructor(Kryo.class, Class.class, Map.class).newInstance(k, superClass, conf); + return serializerClass.getConstructor(Kryo.class, Class.class, Map.class) + .newInstance(k, superClass, conf); } catch (Exception ex1) { try { - return serializerClass.getConstructor(Kryo.class, Class.class).newInstance(k, superClass); + return serializerClass.getConstructor(Kryo.class, Class.class).newInstance(k, + superClass); } catch (Exception ex2) { try { - return serializerClass.getConstructor(Kryo.class, Map.class).newInstance(k, conf); + return serializerClass.getConstructor(Kryo.class, Map.class).newInstance(k, + conf); } catch (Exception ex3) { try { return serializerClass.getConstructor(Kryo.class).newInstance(k); } catch (Exception ex4) { try { - return serializerClass.getConstructor(Class.class, Map.class).newInstance(superClass, conf); + return serializerClass.getConstructor(Class.class, Map.class) + .newInstance(superClass, conf); } catch (Exception ex5) { try { - return serializerClass.getConstructor(Class.class).newInstance(superClass); + return serializerClass.getConstructor(Class.class) + .newInstance(superClass); } catch (Exception ex6) { return serializerClass.newInstance(); } @@ -209,7 +221,7 @@ private static Map normalizeKryoRegister(Object kryoRegistration } } - //ensure always same order for registrations with TreeMap + // ensure always same order for registrations with TreeMap return new TreeMap<>(ret); } @@ -233,7 +245,8 @@ public IdDictionary(StormTopology topology) { /** * "{:a 1 :b 2} -> {1 :a 2 :b}". * - *

    Note: Only one key wins if there are duplicate values. Which key wins is indeterminate: "{:a 1 :b 1} -> {1 :a} *or* {1 :b}" + *

    Note: Only one key wins if there are duplicate values. Which key wins is + * indeterminate: "{:a 1 :b 1} -> {1 :a} *or* {1 :b}" */ private static Map simpleReverseMap(Map map) { Map ret = new HashMap(); diff --git a/storm-client/src/jvm/org/apache/storm/serialization/SerializationRegister.java b/storm-client/src/jvm/org/apache/storm/serialization/SerializationRegister.java index e714d46ff26..c42265b389d 100644 --- a/storm-client/src/jvm/org/apache/storm/serialization/SerializationRegister.java +++ b/storm-client/src/jvm/org/apache/storm/serialization/SerializationRegister.java @@ -21,8 +21,10 @@ import com.esotericsoftware.kryo.Kryo; /** - * Provides a way using a service loader to register Kryo serializers with the SerializationFactory without needing to modify the config. - * This allows for language bindings libraries or platforms to include their own registration without impacting a clients config. + * Provides a way using a service loader to register Kryo serializers with the SerializationFactory + * without needing to modify the config. + * This allows for language bindings libraries or platforms to include their own registration + * without impacting a clients config. */ public interface SerializationRegister { /** diff --git a/storm-client/src/jvm/org/apache/storm/serialization/ThriftSerializationDelegate.java b/storm-client/src/jvm/org/apache/storm/serialization/ThriftSerializationDelegate.java index a4863e3d32d..36c19543388 100644 --- a/storm-client/src/jvm/org/apache/storm/serialization/ThriftSerializationDelegate.java +++ b/storm-client/src/jvm/org/apache/storm/serialization/ThriftSerializationDelegate.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/serialization/ZstdBridgeThriftSerializationDelegate.java b/storm-client/src/jvm/org/apache/storm/serialization/ZstdBridgeThriftSerializationDelegate.java index a3484b7fd17..bb6b5814b9b 100644 --- a/storm-client/src/jvm/org/apache/storm/serialization/ZstdBridgeThriftSerializationDelegate.java +++ b/storm-client/src/jvm/org/apache/storm/serialization/ZstdBridgeThriftSerializationDelegate.java @@ -1,12 +1,17 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -22,8 +27,10 @@ */ public class ZstdBridgeThriftSerializationDelegate implements SerializationDelegate { - private final GzipBridgeThriftSerializationDelegate defaultDelegate = new GzipBridgeThriftSerializationDelegate(); - private final ZstdThriftSerializationDelegate zstdDelegate = new ZstdThriftSerializationDelegate(); + private final GzipBridgeThriftSerializationDelegate defaultDelegate = + new GzipBridgeThriftSerializationDelegate(); + private final ZstdThriftSerializationDelegate zstdDelegate = + new ZstdThriftSerializationDelegate(); @Override public void prepare(Map topoConf) { @@ -43,7 +50,8 @@ public T deserialize(byte[] bytes, Class clazz) { return zstdDelegate.deserialize(bytes, clazz); } else { // Fallback to ZstdBridgeThriftSerializationDelegate - // it delegates to the proper SerializationDelegate (GzipThriftSerializationDelegate or ThriftSerializationDelegate) + // it delegates to the proper SerializationDelegate (GzipThriftSerializationDelegate or + // ThriftSerializationDelegate) return defaultDelegate.deserialize(bytes, clazz); } } diff --git a/storm-client/src/jvm/org/apache/storm/serialization/ZstdThriftSerializationDelegate.java b/storm-client/src/jvm/org/apache/storm/serialization/ZstdThriftSerializationDelegate.java index d3b990a7b3e..f67f2e61029 100644 --- a/storm-client/src/jvm/org/apache/storm/serialization/ZstdThriftSerializationDelegate.java +++ b/storm-client/src/jvm/org/apache/storm/serialization/ZstdThriftSerializationDelegate.java @@ -29,7 +29,8 @@ import org.apache.storm.utils.Utils; /** - * Note, this assumes it's deserializing a zstd byte stream, and will err if it encounters any other serialization. + * Note, this assumes it's deserializing a zstd byte stream, and will err if it encounters any other + * serialization. */ public class ZstdThriftSerializationDelegate implements SerializationDelegate { @@ -41,9 +42,11 @@ public class ZstdThriftSerializationDelegate implements SerializationDelegate { @Override public void prepare(Map topoConf) { - this.zstdCompressionLevel = ObjectReader.getInt(topoConf.getOrDefault(Config.STORM_COMPRESSION_ZSTD_LEVEL, + this.zstdCompressionLevel = ObjectReader.getInt(topoConf + .getOrDefault(Config.STORM_COMPRESSION_ZSTD_LEVEL, DEFAULT_ZSTD_COMPRESSION_LEVEL)); - this.maxDecompressedBytes = ObjectReader.getInt(topoConf.getOrDefault(Config.STORM_COMPRESSION_ZSTD_MAX_DECOMPRESSED_BYTES, + this.maxDecompressedBytes = ObjectReader.getInt(topoConf + .getOrDefault(Config.STORM_COMPRESSION_ZSTD_MAX_DECOMPRESSED_BYTES, DEFAULT_MAX_DECOMPRESSED_BYTES)); } @@ -67,14 +70,16 @@ public byte[] serialize(Object object) { public T deserialize(byte[] bytes, Class clazz) { if (!Utils.ZstdUtils.isZstd(bytes)) { throw new RuntimeException( - String.format("Cannot deserialize [%s]. Expected zstd compressed bytes, but received unknown format.", + String.format("Cannot deserialize [%s]. Expected zstd compressed bytes, but " + + "received unknown format.", clazz.getSimpleName()) ); } try { TDeserializer deserializer = new TDeserializer(); byte[] decompressed = Utils.ZstdUtils.decompress(bytes, this.maxDecompressedBytes); - TBase instance = clazz.asSubclass(TBase.class).getDeclaredConstructor().newInstance(); + TBase instance = clazz.asSubclass(TBase.class).getDeclaredConstructor() + .newInstance(); deserializer.deserialize(instance, decompressed); return (T) instance; } catch (TTransportException e) { diff --git a/storm-client/src/jvm/org/apache/storm/serialization/types/ArrayListSerializer.java b/storm-client/src/jvm/org/apache/storm/serialization/types/ArrayListSerializer.java index 3235c323492..d859c30721f 100644 --- a/storm-client/src/jvm/org/apache/storm/serialization/types/ArrayListSerializer.java +++ b/storm-client/src/jvm/org/apache/storm/serialization/types/ArrayListSerializer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -18,7 +24,6 @@ import java.util.ArrayList; import java.util.Collection; - public class ArrayListSerializer extends CollectionSerializer { @Override public ArrayList create(Kryo kryo, Input input, Class type, int length) { diff --git a/storm-client/src/jvm/org/apache/storm/serialization/types/HashMapSerializer.java b/storm-client/src/jvm/org/apache/storm/serialization/types/HashMapSerializer.java index 73e6b9d57a1..d2c5e39bf99 100644 --- a/storm-client/src/jvm/org/apache/storm/serialization/types/HashMapSerializer.java +++ b/storm-client/src/jvm/org/apache/storm/serialization/types/HashMapSerializer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -18,10 +24,10 @@ import java.util.HashMap; import java.util.Map; - public class HashMapSerializer extends MapSerializer { @Override - protected HashMap create(Kryo kryo, Input input, Class type, int length) { + protected HashMap create(Kryo kryo, Input input, Class type, + int length) { return new HashMap<>(length); } } diff --git a/storm-client/src/jvm/org/apache/storm/serialization/types/HashSetSerializer.java b/storm-client/src/jvm/org/apache/storm/serialization/types/HashSetSerializer.java index a68f6b3f700..32e2a395f9e 100644 --- a/storm-client/src/jvm/org/apache/storm/serialization/types/HashSetSerializer.java +++ b/storm-client/src/jvm/org/apache/storm/serialization/types/HashSetSerializer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -18,7 +24,6 @@ import java.util.Collection; import java.util.HashSet; - public class HashSetSerializer extends CollectionSerializer { @Override protected HashSet create(Kryo kryo, Input input, Class type, int length) { diff --git a/storm-client/src/jvm/org/apache/storm/serialization/types/ListDelegateSerializer.java b/storm-client/src/jvm/org/apache/storm/serialization/types/ListDelegateSerializer.java index 35fde668d64..238e28a5a9a 100644 --- a/storm-client/src/jvm/org/apache/storm/serialization/types/ListDelegateSerializer.java +++ b/storm-client/src/jvm/org/apache/storm/serialization/types/ListDelegateSerializer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -18,10 +24,10 @@ import java.util.Collection; import org.apache.storm.utils.ListDelegate; - public class ListDelegateSerializer extends CollectionSerializer { @Override - public ListDelegate create(Kryo kryo, Input input, Class type, int length) { + public ListDelegate create(Kryo kryo, Input input, Class type, + int length) { return new ListDelegate(); } } diff --git a/storm-client/src/jvm/org/apache/storm/spout/CheckPointState.java b/storm-client/src/jvm/org/apache/storm/spout/CheckPointState.java index 7b327dac45c..cbe176a596c 100644 --- a/storm-client/src/jvm/org/apache/storm/spout/CheckPointState.java +++ b/storm-client/src/jvm/org/apache/storm/spout/CheckPointState.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -17,7 +23,8 @@ import static org.apache.storm.spout.CheckPointState.State.PREPARING; /** - * Captures the current state of the transaction in {@link CheckpointSpout}. The state transitions are as follows. + * Captures the current state of the transaction in {@link CheckpointSpout}. The state transitions + * are as follows. *

      *                  ROLLBACK(tx2)
      *               <-------------                  PREPARE(tx2)                     COMMIT(tx2)
    @@ -25,11 +32,14 @@
      *
      * 
    * - *

    During recovery, if a previous transaction is in PREPARING state, it is rolled back since all bolts in the topology might not have - * prepared (saved) the data for commit. If the previous transaction is in COMMITTING state, it is rolled forward (committed) since some + *

    During recovery, if a previous transaction is in PREPARING state, it is rolled back since all + * bolts in the topology might not have + * prepared (saved) the data for commit. If the previous transaction is in COMMITTING state, it is + * rolled forward (committed) since some * bolts might have already committed the data. * - *

    During normal flow, the state transitions from PREPARING to COMMITTING to COMMITTED. In case of failures the + *

    During normal flow, the state transitions from PREPARING to COMMITTING to COMMITTED. In case + * of failures the * prepare/commit operation is retried. */ public class CheckPointState { @@ -63,7 +73,8 @@ public CheckPointState nextState(boolean recovering) { CheckPointState nextState; switch (state) { case PREPARING: - nextState = recovering ? new CheckPointState(txid - 1, COMMITTED) : new CheckPointState(txid, COMMITTING); + nextState = recovering ? new CheckPointState(txid - 1, + COMMITTED) : new CheckPointState(txid, COMMITTING); break; case COMMITTING: nextState = new CheckPointState(txid, COMMITTED); @@ -140,30 +151,32 @@ public enum State { */ COMMITTED, /** - * The checkpoint spout has started committing the transaction and the commit is in progress. + * The checkpoint spout has started committing the transaction and the commit is in + * progress. */ COMMITTING, /** - * The checkpoint spout has started preparing the transaction for commit and the prepare is in progress. + * The checkpoint spout has started preparing the transaction for commit and the prepare is + * in progress. */ PREPARING } public enum Action { /** - * prepare transaction for commit. + * Prepare transaction for commit. */ PREPARE, /** - * commit the previously prepared transaction. + * Commit the previously prepared transaction. */ COMMIT, /** - * rollback the previously prepared transaction. + * Rollback the previously prepared transaction. */ ROLLBACK, /** - * initialize the state. + * Initialize the state. */ INITSTATE } diff --git a/storm-client/src/jvm/org/apache/storm/spout/CheckpointSpout.java b/storm-client/src/jvm/org/apache/storm/spout/CheckpointSpout.java index c6f6fe45afe..7294362a7be 100644 --- a/storm-client/src/jvm/org/apache/storm/spout/CheckpointSpout.java +++ b/storm-client/src/jvm/org/apache/storm/spout/CheckpointSpout.java @@ -36,8 +36,10 @@ import org.slf4j.LoggerFactory; /** - * Emits checkpoint tuples which is used to save the state of the {@link org.apache.storm.topology.IStatefulComponent} across the topology. - * If a topology contains Stateful bolts, Checkpoint spouts are automatically added to the topology. There is only one Checkpoint task per + * Emits checkpoint tuples which is used to save the state of the {@link + * org.apache.storm.topology.IStatefulComponent} across the topology. + * If a topology contains Stateful bolts, Checkpoint spouts are automatically added to the topology. + * There is only one Checkpoint task per * topology. Checkpoint spout stores its internal state in a {@link KeyValueState}. * * @see CheckPointState @@ -65,7 +67,8 @@ public static boolean isCheckpoint(Tuple input) { } @Override - public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { + public void open(Map conf, TopologyContext context, + SpoutOutputCollector collector) { open(context, collector, loadCheckpointInterval(conf), loadCheckpointState(conf, context)); } @@ -124,13 +127,15 @@ public void fail(Object msgId) { @Override public void declareOutputFields(OutputFieldsDeclarer declarer) { - declarer.declareStream(CHECKPOINT_STREAM_ID, new Fields(CHECKPOINT_FIELD_TXID, CHECKPOINT_FIELD_ACTION)); + declarer.declareStream(CHECKPOINT_STREAM_ID, new Fields(CHECKPOINT_FIELD_TXID, + CHECKPOINT_FIELD_ACTION)); } /** * Loads the last saved checkpoint state the from persistent storage. */ - private KeyValueState loadCheckpointState(Map conf, TopologyContext ctx) { + private KeyValueState loadCheckpointState(Map conf, + TopologyContext ctx) { String namespace = ctx.getThisComponentId() + "-" + ctx.getThisTaskId(); KeyValueState state = (KeyValueState) StateFactory.getState(namespace, conf, ctx); @@ -148,7 +153,8 @@ private KeyValueState loadCheckpointState(Map topoConf) { int interval = 0; if (topoConf.containsKey(Config.TOPOLOGY_STATE_CHECKPOINT_INTERVAL)) { - interval = ((Number) topoConf.get(Config.TOPOLOGY_STATE_CHECKPOINT_INTERVAL)).intValue(); + interval = ((Number) topoConf.get(Config.TOPOLOGY_STATE_CHECKPOINT_INTERVAL)) + .intValue(); } // ensure checkpoint interval is not less than a sane low value. interval = Math.max(100, interval); diff --git a/storm-client/src/jvm/org/apache/storm/spout/IMultiSchemableSpout.java b/storm-client/src/jvm/org/apache/storm/spout/IMultiSchemableSpout.java index fefad52831f..8f82865ad8b 100644 --- a/storm-client/src/jvm/org/apache/storm/spout/IMultiSchemableSpout.java +++ b/storm-client/src/jvm/org/apache/storm/spout/IMultiSchemableSpout.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/spout/ISchemableSpout.java b/storm-client/src/jvm/org/apache/storm/spout/ISchemableSpout.java index 9c1a32a2027..2896d114e63 100644 --- a/storm-client/src/jvm/org/apache/storm/spout/ISchemableSpout.java +++ b/storm-client/src/jvm/org/apache/storm/spout/ISchemableSpout.java @@ -1,18 +1,23 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.spout; - public interface ISchemableSpout { Scheme getScheme(); diff --git a/storm-client/src/jvm/org/apache/storm/spout/ISpout.java b/storm-client/src/jvm/org/apache/storm/spout/ISpout.java index 120ab363e80..eedc65a1464 100644 --- a/storm-client/src/jvm/org/apache/storm/spout/ISpout.java +++ b/storm-client/src/jvm/org/apache/storm/spout/ISpout.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -17,76 +23,101 @@ import org.apache.storm.task.TopologyContext; /** - * ISpout is the core interface for implementing spouts. A Spout is responsible for feeding messages into the topology - * for processing. For every tuple emitted by a spout, Storm will track the (potentially very large) DAG of tuples - * generated based on a tuple emitted by the spout. When Storm detects that every tuple in that DAG has been + * ISpout is the core interface for implementing spouts. A Spout is responsible for feeding messages + * into the topology + * for processing. For every tuple emitted by a spout, Storm will track the (potentially very large) + * DAG of tuples + * generated based on a tuple emitted by the spout. When Storm detects that every tuple in that DAG + * has been * successfully processed, it will send an ack message to the Spout. * *

    If a tuple fails to be fully processed within the configured timeout for the topology (see * {@link org.apache.storm.Config}), Storm will send a fail message to the spout for the message. * - *

    When a Spout emits a tuple, it can tag the tuple with a message id. The message id can be any type. When Storm - * acks or fails a message, it will pass back to the spout the same message id to identify which tuple it's referring - * to. If the spout leaves out the message id, or sets it to null, then Storm will not track the message and the spout + *

    When a Spout emits a tuple, it can tag the tuple with a message id. The message id can be any + * type. When Storm + * acks or fails a message, it will pass back to the spout the same message id to identify which + * tuple it's referring + * to. If the spout leaves out the message id, or sets it to null, then Storm will not track the + * message and the spout * will not receive any ack or fail callbacks for the message. * - *

    Storm executes ack, fail, and nextTuple all on the same thread. This means that an implementor of an ISpout does - * not need to worry about concurrency issues between those methods. However, it also means that an implementor must - * ensure that nextTuple is non-blocking: otherwise the method could block acks and fails that are pending to be + *

    Storm executes ack, fail, and nextTuple all on the same thread. This means that an implementor + * of an ISpout does + * not need to worry about concurrency issues between those methods. However, it also means that an + * implementor must + * ensure that nextTuple is non-blocking: otherwise the method could block acks and fails that are + * pending to be * processed. */ public interface ISpout extends Serializable { /** - * Called when a task for this component is initialized within a worker on the cluster. It provides the spout with the environment in + * Called when a task for this component is initialized within a worker on the cluster. It + * provides the spout with the environment in * which the spout executes. * *

    This includes the: * - * @param conf The Storm configuration for this spout. This is the configuration provided to the topology merged in with cluster + * @param conf The Storm configuration for this spout. This is the configuration provided to the + * topology merged in with cluster * configuration on this machine. - * @param context This object can be used to get information about this task's place within the topology, including the task id and + * @param context This object can be used to get information about this task's place within the + * topology, including the task id and * component id of this task, input and output information, etc. - * @param collector The collector is used to emit tuples from this spout. Tuples can be emitted at any time, including the open and - * close methods. The collector is thread-safe and should be saved as an instance variable of this spout object. + * @param collector The collector is used to emit tuples from this spout. Tuples can be emitted + * at any time, including the open and + * close methods. The collector is thread-safe and should be saved as an instance variable + * of this spout object. */ void open(Map conf, TopologyContext context, SpoutOutputCollector collector); /** - * Called when an ISpout is going to be shutdown. There is no guarentee that close will be called, because the supervisor kill -9's + * Called when an ISpout is going to be shutdown. There is no guarentee that close will be + * called, because the supervisor kill -9's * worker processes on the cluster. * - *

    The one context where close is guaranteed to be called is a topology is killed when running Storm in local mode. + *

    The one context where close is guaranteed to be called is a topology is killed when + * running Storm in local mode. */ void close(); /** - * Called when a spout has been activated out of a deactivated mode. nextTuple will be called on this spout soon. A spout can become - * activated after having been deactivated when the topology is manipulated using the `storm` client. + * Called when a spout has been activated out of a deactivated mode. nextTuple will be called on + * this spout soon. A spout can become + * activated after having been deactivated when the topology is manipulated using the `storm` + * client. */ void activate(); /** - * Called when a spout has been deactivated. nextTuple will not be called while a spout is deactivated. The spout may or may not be + * Called when a spout has been deactivated. nextTuple will not be called while a spout is + * deactivated. The spout may or may not be * reactivated in the future. */ void deactivate(); /** - * When this method is called, Storm is requesting that the Spout emit tuples to the output collector. This method should be - * non-blocking, so if the Spout has no tuples to emit, this method should return. nextTuple, ack, and fail are all called in a tight - * loop in a single thread in the spout task. When there are no tuples to emit, it is courteous to have nextTuple sleep for a short + * When this method is called, Storm is requesting that the Spout emit tuples to the output + * collector. This method should be + * non-blocking, so if the Spout has no tuples to emit, this method should return. nextTuple, + * ack, and fail are all called in a tight + * loop in a single thread in the spout task. When there are no tuples to emit, it is courteous + * to have nextTuple sleep for a short * amount of time (like a single millisecond) so as not to waste too much CPU. */ void nextTuple(); /** - * Storm has determined that the tuple emitted by this spout with the msgId identifier has been fully processed. Typically, an - * implementation of this method will take that message off the queue and prevent it from being replayed. + * Storm has determined that the tuple emitted by this spout with the msgId identifier has been + * fully processed. Typically, an + * implementation of this method will take that message off the queue and prevent it from being + * replayed. */ void ack(Object msgId); /** - * The tuple emitted by this spout with the msgId identifier has failed to be fully processed. Typically, an implementation of this + * The tuple emitted by this spout with the msgId identifier has failed to be fully processed. + * Typically, an implementation of this * method will put that message back on the queue to be replayed at a later time. */ void fail(Object msgId); diff --git a/storm-client/src/jvm/org/apache/storm/spout/ISpoutOutputCollector.java b/storm-client/src/jvm/org/apache/storm/spout/ISpoutOutputCollector.java index 9071cd81b55..0d08ad3b6e6 100644 --- a/storm-client/src/jvm/org/apache/storm/spout/ISpoutOutputCollector.java +++ b/storm-client/src/jvm/org/apache/storm/spout/ISpoutOutputCollector.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -16,7 +22,8 @@ import org.apache.storm.task.IErrorReporter; /** - * Methods are not expected to be thread safe. Each thread expected to have a separate instance of this type of object, or else externally + * Methods are not expected to be thread safe. Each thread expected to have a separate instance of + * this type of object, or else externally * synchronize any shared instance. */ diff --git a/storm-client/src/jvm/org/apache/storm/spout/MultiScheme.java b/storm-client/src/jvm/org/apache/storm/spout/MultiScheme.java index 0d123b3a18e..f82e3ebaf86 100644 --- a/storm-client/src/jvm/org/apache/storm/spout/MultiScheme.java +++ b/storm-client/src/jvm/org/apache/storm/spout/MultiScheme.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/spout/Scheme.java b/storm-client/src/jvm/org/apache/storm/spout/Scheme.java index 521dcf5682b..a6a9bfd5376 100644 --- a/storm-client/src/jvm/org/apache/storm/spout/Scheme.java +++ b/storm-client/src/jvm/org/apache/storm/spout/Scheme.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -17,7 +23,6 @@ import java.util.List; import org.apache.storm.tuple.Fields; - public interface Scheme extends Serializable { List deserialize(ByteBuffer ser); diff --git a/storm-client/src/jvm/org/apache/storm/spout/SchemeAsMultiScheme.java b/storm-client/src/jvm/org/apache/storm/spout/SchemeAsMultiScheme.java index 18ae0cbd5d1..a3b4b78098a 100644 --- a/storm-client/src/jvm/org/apache/storm/spout/SchemeAsMultiScheme.java +++ b/storm-client/src/jvm/org/apache/storm/spout/SchemeAsMultiScheme.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/spout/ShellSpout.java b/storm-client/src/jvm/org/apache/storm/spout/ShellSpout.java index 8487e113507..cec71ce3533 100644 --- a/storm-client/src/jvm/org/apache/storm/spout/ShellSpout.java +++ b/storm-client/src/jvm/org/apache/storm/spout/ShellSpout.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -37,7 +43,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; - public class ShellSpout implements ISpout { public static final Logger LOG = LoggerFactory.getLogger(ShellSpout.class); private static final long serialVersionUID = 5982357019665454L; @@ -79,10 +84,12 @@ public boolean shouldChangeChildCWD() { } /** - * Set if the current working directory of the child process should change to the resources dir from extracted from the jar, or if it + * Set if the current working directory of the child process should change to the resources dir + * from extracted from the jar, or if it * should stay the same as the worker process to access things from the blob store. * - * @param changeDirectory true change the directory (default) false leave the directory the same as the worker process. + * @param changeDirectory true change the directory (default) false leave the directory the same + * as the worker process. */ @SuppressWarnings("checkstyle:AbbreviationAsWordInName") public void changeChildCWD(boolean changeDirectory) { @@ -96,9 +103,11 @@ public void open(Map topoConf, TopologyContext context, this.context = context; if (topoConf.containsKey(Config.TOPOLOGY_SUBPROCESS_TIMEOUT_SECS)) { - workerTimeoutMills = 1000 * ObjectReader.getInt(topoConf.get(Config.TOPOLOGY_SUBPROCESS_TIMEOUT_SECS)); + workerTimeoutMills = 1000 * ObjectReader.getInt(topoConf + .get(Config.TOPOLOGY_SUBPROCESS_TIMEOUT_SECS)); } else { - workerTimeoutMills = 1000 * ObjectReader.getInt(topoConf.get(Config.SUPERVISOR_WORKER_TIMEOUT_SECS)); + workerTimeoutMills = 1000 * ObjectReader.getInt(topoConf + .get(Config.SUPERVISOR_WORKER_TIMEOUT_SECS)); } process = new ShellProcess(command); @@ -112,7 +121,8 @@ public void open(Map topoConf, TopologyContext context, logHandler = ShellUtils.getLogHandler(topoConf); logHandler.setUpContext(ShellSpout.class, process, this.context); - heartBeatExecutorService = MoreExecutors.getExitingScheduledExecutorService(new ScheduledThreadPoolExecutor(1)); + heartBeatExecutorService = MoreExecutors + .getExitingScheduledExecutorService(new ScheduledThreadPoolExecutor(1)); } @Override @@ -150,25 +160,25 @@ private void sendSyncCommand(String command, Object msgId) { querySubprocess(); } - private void handleMetrics(ShellMsg shellMsg) { - //get metric name + // get metric name String name = shellMsg.getMetricName(); if (name.isEmpty()) { throw new RuntimeException("Receive Metrics name is empty"); } - //get metric by name + // get metric by name IMetric metric = context.getRegisteredMetricByName(name); if (metric == null) { throw new RuntimeException("Could not find metric by name[" + name + "] "); } if (!(metric instanceof IShellMetric)) { - throw new RuntimeException("Metric[" + name + "] is not IShellMetric, can not call by RPC"); + throw new RuntimeException("Metric[" + name + + "] is not IShellMetric, can not call by RPC"); } IShellMetric shellMetric = (IShellMetric) metric; - //call updateMetricFromRPC with params + // call updateMetricFromRPC with params Object paramsObj = shellMsg.getMetricParams(); try { shellMetric.updateMetricFromRPC(paramsObj); @@ -188,7 +198,8 @@ private void querySubprocess() { ShellMsg shellMsg = process.readShellMsg(); String command = shellMsg.getCommand(); if (command == null) { - throw new IllegalArgumentException("Command not found in spout message: " + shellMsg); + throw new IllegalArgumentException("Command not found in spout message: " + + shellMsg); } setHeartbeat(); @@ -219,7 +230,8 @@ private void querySubprocess() { } } } catch (Exception e) { - String processInfo = process.getProcessInfoString() + process.getProcessTerminationInfoString(); + String processInfo = process.getProcessInfoString() + process + .getProcessTerminationInfoString(); throw new RuntimeException(processInfo, e); } finally { completedWaitingSubprocess(); @@ -236,10 +248,12 @@ public void activate() { // prevent timer to check heartbeat based on last thing before activate setHeartbeat(); if (heartBeatExecutorService.isShutdown()) { - //In case deactivate was called before - heartBeatExecutorService = MoreExecutors.getExitingScheduledExecutorService(new ScheduledThreadPoolExecutor(1)); + // In case deactivate was called before + heartBeatExecutorService = MoreExecutors + .getExitingScheduledExecutorService(new ScheduledThreadPoolExecutor(1)); } - heartBeatExecutorService.scheduleAtFixedRate(new SpoutHeartbeatTimerTask(this), 1, 1, TimeUnit.SECONDS); + heartBeatExecutorService.scheduleAtFixedRate(new SpoutHeartbeatTimerTask(this), 1, 1, + TimeUnit.SECONDS); this.sendSyncCommand("activate", ""); } @@ -267,14 +281,17 @@ private void completedWaitingSubprocess() { } private void die(Throwable exception) { - String processInfo = process.getProcessInfoString() + process.getProcessTerminationInfoString(); + String processInfo = process.getProcessInfoString() + process + .getProcessTerminationInfoString(); this.exception = new RuntimeException(processInfo, exception); - String message = String.format("Halting process: ShellSpout died. Command: %s, ProcessInfo %s", + String message = String + .format("Halting process: ShellSpout died. Command: %s, ProcessInfo %s", Arrays.toString(command), processInfo); LOG.error(message, exception); collector.reportError(exception); - if (running || (exception instanceof Error)) { //don't exit if not running, unless it is an Error + if (running + || (exception instanceof Error)) { // don't exit if not running, unless it is an Error System.exit(11); } } diff --git a/storm-client/src/jvm/org/apache/storm/spout/SpoutOutputCollector.java b/storm-client/src/jvm/org/apache/storm/spout/SpoutOutputCollector.java index 09f155d3138..f795fa4893c 100644 --- a/storm-client/src/jvm/org/apache/storm/spout/SpoutOutputCollector.java +++ b/storm-client/src/jvm/org/apache/storm/spout/SpoutOutputCollector.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -17,9 +23,12 @@ import org.apache.storm.utils.Utils; /** - * This output collector exposes the API for emitting tuples from an {@link org.apache.storm.topology.IRichSpout}. The main difference - * between this output collector and {@link OutputCollector} for {@link org.apache.storm.topology.IRichBolt} is that spouts can tag messages - * with ids so that they can be acked or failed later on. This is the Spout portion of Storm's API to guarantee that each message is fully + * This output collector exposes the API for emitting tuples from an {@link + * org.apache.storm.topology.IRichSpout}. The main difference + * between this output collector and {@link OutputCollector} for {@link + * org.apache.storm.topology.IRichBolt} is that spouts can tag messages + * with ids so that they can be acked or failed later on. This is the Spout portion of Storm's API + * to guarantee that each message is fully * processed at least once. */ public class SpoutOutputCollector implements ISpoutOutputCollector { @@ -30,10 +39,14 @@ public SpoutOutputCollector(ISpoutOutputCollector delegate) { } /** - * Emits a new tuple to the specified output stream with the given message ID. When Storm detects that this tuple has been fully - * processed, or has failed to be fully processed, the spout will receive an ack or fail callback respectively with the messageId as - * long as the messageId was not null. If the messageId was null, Storm will not track the tuple and no callback will be received. Note - * that Storm's event logging functionality will only work if the messageId is serializable via Kryo or the Serializable interface. The + * Emits a new tuple to the specified output stream with the given message ID. When Storm + * detects that this tuple has been fully + * processed, or has failed to be fully processed, the spout will receive an ack or fail + * callback respectively with the messageId as + * long as the messageId was not null. If the messageId was null, Storm will not track the tuple + * and no callback will be received. Note + * that Storm's event logging functionality will only work if the messageId is serializable via + * Kryo or the Serializable interface. The * emitted values must be immutable. * * @return the list of task ids that this tuple was sent to @@ -44,10 +57,14 @@ public List emit(String streamId, List tuple, Object messageId) } /** - * Emits a new tuple to the default output stream with the given message ID. When Storm detects that this tuple has been fully - * processed, or has failed to be fully processed, the spout will receive an ack or fail callback respectively with the messageId as - * long as the messageId was not null. If the messageId was null, Storm will not track the tuple and no callback will be received. Note - * that Storm's event logging functionality will only work if the messageId is serializable via Kryo or the Serializable interface. The + * Emits a new tuple to the default output stream with the given message ID. When Storm detects + * that this tuple has been fully + * processed, or has failed to be fully processed, the spout will receive an ack or fail + * callback respectively with the messageId as + * long as the messageId was not null. If the messageId was null, Storm will not track the tuple + * and no callback will be received. Note + * that Storm's event logging functionality will only work if the messageId is serializable via + * Kryo or the Serializable interface. The * emitted values must be immutable. * * @return the list of task ids that this tuple was sent to @@ -57,7 +74,8 @@ public List emit(List tuple, Object messageId) { } /** - * Emits a tuple to the default output stream with a null message id. Storm will not track this message so ack and fail will never be + * Emits a tuple to the default output stream with a null message id. Storm will not track this + * message so ack and fail will never be * called for this tuple. The emitted values must be immutable. */ public List emit(List tuple) { @@ -65,7 +83,8 @@ public List emit(List tuple) { } /** - * Emits a tuple to the specified output stream with a null message id. Storm will not track this message so ack and fail will never be + * Emits a tuple to the specified output stream with a null message id. Storm will not track + * this message so ack and fail will never be * called for this tuple. The emitted values must be immutable. */ public List emit(String streamId, List tuple) { @@ -73,9 +92,12 @@ public List emit(String streamId, List tuple) { } /** - * Emits a tuple to the specified task on the specified output stream. This output stream must have been declared as a direct stream, - * and the specified task must use a direct grouping on this stream to receive the message. Note that Storm's event logging - * functionality will only work if the messageId is serializable via Kryo or the Serializable interface. The emitted values must be + * Emits a tuple to the specified task on the specified output stream. This output stream must + * have been declared as a direct stream, + * and the specified task must use a direct grouping on this stream to receive the message. Note + * that Storm's event logging + * functionality will only work if the messageId is serializable via Kryo or the Serializable + * interface. The emitted values must be * immutable. */ @Override @@ -84,29 +106,38 @@ public void emitDirect(int taskId, String streamId, List tuple, Object m } /** - * Emits a tuple to the specified task on the default output stream. This output stream must have been declared as a direct stream, and - * the specified task must use a direct grouping on this stream to receive the message. Note that Storm's event logging functionality - * will only work if the messageId is serializable via Kryo or the Serializable interface. The emitted values must be immutable. + * Emits a tuple to the specified task on the default output stream. This output stream must + * have been declared as a direct stream, and + * the specified task must use a direct grouping on this stream to receive the message. Note + * that Storm's event logging functionality + * will only work if the messageId is serializable via Kryo or the Serializable interface. The + * emitted values must be immutable. */ public void emitDirect(int taskId, List tuple, Object messageId) { emitDirect(taskId, Utils.DEFAULT_STREAM_ID, tuple, messageId); } /** - * Emits a tuple to the specified task on the specified output stream. This output stream must have been declared as a direct stream, - * and the specified task must use a direct grouping on this stream to receive the message. The emitted values must be immutable. + * Emits a tuple to the specified task on the specified output stream. This output stream must + * have been declared as a direct stream, + * and the specified task must use a direct grouping on this stream to receive the message. The + * emitted values must be immutable. * - *

    Because no message id is specified, Storm will not track this message so ack and fail will never be called for this tuple. + *

    Because no message id is specified, Storm will not track this message so ack and fail will + * never be called for this tuple. */ public void emitDirect(int taskId, String streamId, List tuple) { emitDirect(taskId, streamId, tuple, null); } /** - * Emits a tuple to the specified task on the default output stream. This output stream must have been declared as a direct stream, and - * the specified task must use a direct grouping on this stream to receive the message. The emitted values must be immutable. + * Emits a tuple to the specified task on the default output stream. This output stream must + * have been declared as a direct stream, and + * the specified task must use a direct grouping on this stream to receive the message. The + * emitted values must be immutable. * - *

    Because no message id is specified, Storm will not track this message so ack and fail will never be called for this tuple. + *

    Because no message id is specified, Storm will not track this message so ack and fail will + * never be called for this tuple. */ public void emitDirect(int taskId, List tuple) { emitDirect(taskId, tuple, null); diff --git a/storm-client/src/jvm/org/apache/storm/state/BaseBinaryStateIterator.java b/storm-client/src/jvm/org/apache/storm/state/BaseBinaryStateIterator.java index ee32e6ae031..d8d87bc59fd 100644 --- a/storm-client/src/jvm/org/apache/storm/state/BaseBinaryStateIterator.java +++ b/storm-client/src/jvm/org/apache/storm/state/BaseBinaryStateIterator.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

    - * http://www.apache.org/licenses/LICENSE-2.0 - *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

    http://www.apache.org/licenses/LICENSE-2.0 + * + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -19,7 +25,8 @@ import org.apache.storm.shade.com.google.common.primitives.UnsignedBytes; /** - * Base implementation of iterator over {@link KeyValueState} which encoded types of key and value are both binary type. + * Base implementation of iterator over {@link KeyValueState} which encoded types of key and value + * are both binary type. */ public abstract class BaseBinaryStateIterator extends BaseStateIterator { @@ -31,7 +38,8 @@ public abstract class BaseBinaryStateIterator extends BaseStateIterator> pendingPrepareIterator, Iterator> pendingCommitIterator) { - super(Iterators.peekingIterator(pendingPrepareIterator), Iterators.peekingIterator(pendingCommitIterator), + super(Iterators.peekingIterator(pendingPrepareIterator), Iterators + .peekingIterator(pendingCommitIterator), new TreeSet<>(UnsignedBytes.lexicographicalComparator())); } diff --git a/storm-client/src/jvm/org/apache/storm/state/BaseStateIterator.java b/storm-client/src/jvm/org/apache/storm/state/BaseStateIterator.java index 4af027f18c7..22f1f5c26e3 100644 --- a/storm-client/src/jvm/org/apache/storm/state/BaseStateIterator.java +++ b/storm-client/src/jvm/org/apache/storm/state/BaseStateIterator.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

    - * http://www.apache.org/licenses/LICENSE-2.0 - *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

    http://www.apache.org/licenses/LICENSE-2.0 + * + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -21,7 +27,8 @@ import org.apache.storm.shade.com.google.common.collect.PeekingIterator; /** - * Base implementation of iterator over {@link KeyValueState}. Encoded/Decoded types of key and value are all generic. + * Base implementation of iterator over {@link KeyValueState}. Encoded/Decoded types of key and + * value are all generic. */ public abstract class BaseStateIterator implements Iterator> { @@ -151,7 +158,8 @@ public void remove() { */ protected abstract boolean isTombstoneValue(VENCODEDT value); - private boolean seekToAvailableEntry(PeekingIterator> iterator) { + private boolean seekToAvailableEntry(PeekingIterator> iterator) { if (iterator != null) { while (iterator.hasNext()) { Map.Entry entry = iterator.peek(); diff --git a/storm-client/src/jvm/org/apache/storm/state/DefaultStateEncoder.java b/storm-client/src/jvm/org/apache/storm/state/DefaultStateEncoder.java index 48afe473fc2..52a53fa2b68 100644 --- a/storm-client/src/jvm/org/apache/storm/state/DefaultStateEncoder.java +++ b/storm-client/src/jvm/org/apache/storm/state/DefaultStateEncoder.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -15,14 +21,17 @@ import java.util.Optional; /** - * Default state encoder class for encoding/decoding key values. This class assumes encoded types of key and value are both binary (byte + * Default state encoder class for encoding/decoding key values. This class assumes encoded types of + * key and value are both binary (byte * array) due to keep backward compatibility. */ public class DefaultStateEncoder implements StateEncoder { - public static final Serializer> INTERNAL_VALUE_SERIALIZER = new DefaultStateSerializer<>(); + public static final Serializer> INTERNAL_VALUE_SERIALIZER = + new DefaultStateSerializer<>(); - public static final byte[] TOMBSTONE = INTERNAL_VALUE_SERIALIZER.serialize(Optional.empty()); + public static final byte[] TOMBSTONE = INTERNAL_VALUE_SERIALIZER + .serialize(Optional.empty()); private final Serializer keySerializer; private final Serializer valueSerializer; diff --git a/storm-client/src/jvm/org/apache/storm/state/DefaultStateSerializer.java b/storm-client/src/jvm/org/apache/storm/state/DefaultStateSerializer.java index 75ef038c8cb..29386061f0a 100644 --- a/storm-client/src/jvm/org/apache/storm/state/DefaultStateSerializer.java +++ b/storm-client/src/jvm/org/apache/storm/state/DefaultStateSerializer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -17,7 +23,6 @@ import com.esotericsoftware.kryo.io.Output; import com.esotericsoftware.kryo.util.DefaultClassResolver; import com.esotericsoftware.kryo.util.DefaultInstantiatorStrategy; - import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -50,12 +55,15 @@ protected Kryo initialValue() { Kryo obj = new Kryo(new StateClassResolver(), null); // Registration bounds the set of classes this serializer will construct from stored // bytes to the ones the topology declared. It is keyed off the same config as the tuple - // path in DefaultKryoFactory, so a topology that has opted into permissive Kryo globally + // path in DefaultKryoFactory, so a topology that has opted into permissive Kryo + // globally // keeps the previous behaviour. // Note this Kryo is independent of the one SerializationFactory builds for tuples: the - // two id spaces must not be merged, since the tuple ids are a wire format between workers. + // two id spaces must not be merged, since the tuple ids are a wire format between + // workers. boolean fallBackOnJavaSerialization = ObjectReader.getBoolean( - topoConf == null ? null : topoConf.get(Config.TOPOLOGY_FALL_BACK_ON_JAVA_SERIALIZATION), false); + topoConf == null ? null : topoConf + .get(Config.TOPOLOGY_FALL_BACK_ON_JAVA_SERIALIZATION), false); obj.setRegistrationRequired(!fallBackOnJavaSerialization); if (context != null && topoConf != null) { KryoTupleSerializer ser = new KryoTupleSerializer(topoConf, context); @@ -79,16 +87,20 @@ protected Output initialValue() { }; /** - * Constructs a {@link DefaultStateSerializer} instance with the given list of classes registered in kryo. + * Constructs a {@link DefaultStateSerializer} instance with the given list of classes + * registered in kryo. * * @param classesToRegister the classes to register. */ - public DefaultStateSerializer(Map topoConf, TopologyContext context, List> classesToRegister) { + public DefaultStateSerializer(Map topoConf, TopologyContext context, + List> classesToRegister) { this.context = context; this.topoConf = topoConf; - registrations.addAll(classesToRegister.stream().map(Class::getName).collect(Collectors.toSet())); + registrations.addAll(classesToRegister.stream().map(Class::getName).collect(Collectors + .toSet())); // other classes from config - registrations.addAll((List) topoConf.getOrDefault(Config.TOPOLOGY_STATE_KRYO_REGISTER, Collections.emptyList())); + registrations.addAll((List) topoConf + .getOrDefault(Config.TOPOLOGY_STATE_KRYO_REGISTER, Collections.emptyList())); // defaults registrations.add(Optional.class.getName()); } @@ -115,10 +127,12 @@ public T deserialize(byte[] b) { } /** - * Registers the types Storm's own state encoding and checkpointing persist without the component + * Registers the types Storm's own state encoding and checkpointing persist without the + * component * declaring them. * - *

    Called after the configured registrations so that the ids assigned to those are unaffected. + *

    Called after the configured registrations so that the ids assigned to those are + * unaffected. * {@link Kryo#register(Class)} returns any existing registration, so a class already declared * through {@link Config#TOPOLOGY_STATE_KRYO_REGISTER} keeps the id assigned there. * @@ -165,7 +179,8 @@ private static class TupleSerializer extends com.esotericsoftware.kryo.Serialize private final KryoTupleSerializer tupleSerializer; private final KryoTupleDeserializer tupleDeserializer; - TupleSerializer(KryoTupleSerializer tupleSerializer, KryoTupleDeserializer tupleDeserializer) { + TupleSerializer(KryoTupleSerializer tupleSerializer, + KryoTupleDeserializer tupleDeserializer) { this.tupleSerializer = tupleSerializer; this.tupleDeserializer = tupleDeserializer; } diff --git a/storm-client/src/jvm/org/apache/storm/state/IStateSpout.java b/storm-client/src/jvm/org/apache/storm/state/IStateSpout.java index 0622a45eedc..39640d90397 100644 --- a/storm-client/src/jvm/org/apache/storm/state/IStateSpout.java +++ b/storm-client/src/jvm/org/apache/storm/state/IStateSpout.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/state/IStateSpoutOutputCollector.java b/storm-client/src/jvm/org/apache/storm/state/IStateSpoutOutputCollector.java index a26b72a2f3e..ad304c3da3c 100644 --- a/storm-client/src/jvm/org/apache/storm/state/IStateSpoutOutputCollector.java +++ b/storm-client/src/jvm/org/apache/storm/state/IStateSpoutOutputCollector.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/state/ISubscribedState.java b/storm-client/src/jvm/org/apache/storm/state/ISubscribedState.java index 5ff6dde731e..6b5f491713e 100644 --- a/storm-client/src/jvm/org/apache/storm/state/ISubscribedState.java +++ b/storm-client/src/jvm/org/apache/storm/state/ISubscribedState.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/state/ISynchronizeOutputCollector.java b/storm-client/src/jvm/org/apache/storm/state/ISynchronizeOutputCollector.java index 93062fffa29..9e1b469bf16 100644 --- a/storm-client/src/jvm/org/apache/storm/state/ISynchronizeOutputCollector.java +++ b/storm-client/src/jvm/org/apache/storm/state/ISynchronizeOutputCollector.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/state/InMemoryKeyValueState.java b/storm-client/src/jvm/org/apache/storm/state/InMemoryKeyValueState.java index c58bb27c2bd..191ede2ec95 100644 --- a/storm-client/src/jvm/org/apache/storm/state/InMemoryKeyValueState.java +++ b/storm-client/src/jvm/org/apache/storm/state/InMemoryKeyValueState.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/state/InMemoryKeyValueStateProvider.java b/storm-client/src/jvm/org/apache/storm/state/InMemoryKeyValueStateProvider.java index 8cbc61791ff..307a9094b98 100644 --- a/storm-client/src/jvm/org/apache/storm/state/InMemoryKeyValueStateProvider.java +++ b/storm-client/src/jvm/org/apache/storm/state/InMemoryKeyValueStateProvider.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/state/KeyValueState.java b/storm-client/src/jvm/org/apache/storm/state/KeyValueState.java index a8e2e68dbf8..925eddd941a 100644 --- a/storm-client/src/jvm/org/apache/storm/state/KeyValueState.java +++ b/storm-client/src/jvm/org/apache/storm/state/KeyValueState.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/state/Serializer.java b/storm-client/src/jvm/org/apache/storm/state/Serializer.java index df754309252..43cfe83ebca 100644 --- a/storm-client/src/jvm/org/apache/storm/state/Serializer.java +++ b/storm-client/src/jvm/org/apache/storm/state/Serializer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/state/State.java b/storm-client/src/jvm/org/apache/storm/state/State.java index bbb18172556..299c1897251 100644 --- a/storm-client/src/jvm/org/apache/storm/state/State.java +++ b/storm-client/src/jvm/org/apache/storm/state/State.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -15,14 +21,18 @@ import org.apache.storm.topology.IStatefulBolt; /** - * The state of the component that is either managed by the framework (e.g in case of {@link IStatefulBolt}) or managed by the the + * The state of the component that is either managed by the framework (e.g in case of {@link + * IStatefulBolt}) or managed by the the * individual components themselves. */ public interface State { /** - * Invoked by the framework to prepare a transaction for commit. It should be possible to commit the prepared state later. - *

    - * The same txid can be prepared again, but the next txid cannot be prepared when previous one is not yet committed. + * Invoked by the framework to prepare a transaction for commit. It should be possible to commit + * the prepared state later. + * + *

    The same txid can be prepared again, but the next txid cannot be prepared when previous + * one + * is not yet committed. *

    * * @param txid the transaction id @@ -30,7 +40,8 @@ public interface State { void prepareCommit(long txid); /** - * Commit a previously prepared transaction. It should be possible to retrieve a committed state later. + * Commit a previously prepared transaction. It should be possible to retrieve a committed state + * later. * * @param txid the transaction id */ diff --git a/storm-client/src/jvm/org/apache/storm/state/StateEncoder.java b/storm-client/src/jvm/org/apache/storm/state/StateEncoder.java index 687bd5edb2b..446f25e85f7 100644 --- a/storm-client/src/jvm/org/apache/storm/state/StateEncoder.java +++ b/storm-client/src/jvm/org/apache/storm/state/StateEncoder.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/state/StateFactory.java b/storm-client/src/jvm/org/apache/storm/state/StateFactory.java index 4f25402863b..be6165752ca 100644 --- a/storm-client/src/jvm/org/apache/storm/state/StateFactory.java +++ b/storm-client/src/jvm/org/apache/storm/state/StateFactory.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -24,10 +30,12 @@ public class StateFactory { private static final Logger LOG = LoggerFactory.getLogger(StateFactory.class); - private static final String DEFAULT_PROVIDER = "org.apache.storm.state.InMemoryKeyValueStateProvider"; + private static final String DEFAULT_PROVIDER = + "org.apache.storm.state.InMemoryKeyValueStateProvider"; /** - * Returns a new state instance using the {@link Config#TOPOLOGY_STATE_PROVIDER} or a {@link InMemoryKeyValueState} if no provider is + * Returns a new state instance using the {@link Config#TOPOLOGY_STATE_PROVIDER} or a {@link + * InMemoryKeyValueState} if no provider is * configured. * * @param namespace the state namespace @@ -35,7 +43,8 @@ public class StateFactory { * @param context the topology context * @return the state instance */ - public static State getState(String namespace, Map topoConf, TopologyContext context) { + public static State getState(String namespace, Map topoConf, + TopologyContext context) { State state; try { String provider = null; diff --git a/storm-client/src/jvm/org/apache/storm/state/StateProvider.java b/storm-client/src/jvm/org/apache/storm/state/StateProvider.java index e07226240c6..a88ea2df893 100644 --- a/storm-client/src/jvm/org/apache/storm/state/StateProvider.java +++ b/storm-client/src/jvm/org/apache/storm/state/StateProvider.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -20,7 +26,8 @@ */ public interface StateProvider { /** - * Returns a new state instance. Each state belongs unique namespace which is typically the componentid-task of the task, so that each + * Returns a new state instance. Each state belongs unique namespace which is typically the + * componentid-task of the task, so that each * task can have its own unique state. * * @param namespace a namespace of the state diff --git a/storm-client/src/jvm/org/apache/storm/state/StateSpoutOutputCollector.java b/storm-client/src/jvm/org/apache/storm/state/StateSpoutOutputCollector.java index 68a53069136..69c17adc34f 100644 --- a/storm-client/src/jvm/org/apache/storm/state/StateSpoutOutputCollector.java +++ b/storm-client/src/jvm/org/apache/storm/state/StateSpoutOutputCollector.java @@ -1,18 +1,23 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.state; - public class StateSpoutOutputCollector extends SynchronizeOutputCollector implements IStateSpoutOutputCollector { @Override diff --git a/storm-client/src/jvm/org/apache/storm/state/SynchronizeOutputCollector.java b/storm-client/src/jvm/org/apache/storm/state/SynchronizeOutputCollector.java index 9419db7a96a..8f5b4a66597 100644 --- a/storm-client/src/jvm/org/apache/storm/state/SynchronizeOutputCollector.java +++ b/storm-client/src/jvm/org/apache/storm/state/SynchronizeOutputCollector.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -14,7 +20,6 @@ import java.util.List; - public class SynchronizeOutputCollector implements ISynchronizeOutputCollector { @Override diff --git a/storm-client/src/jvm/org/apache/storm/stats/BoltExecutorStats.java b/storm-client/src/jvm/org/apache/storm/stats/BoltExecutorStats.java index 26e3776ce5e..2adcdb7c4ae 100644 --- a/storm-client/src/jvm/org/apache/storm/stats/BoltExecutorStats.java +++ b/storm-client/src/jvm/org/apache/storm/stats/BoltExecutorStats.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -16,7 +22,6 @@ import java.util.List; import java.util.Map; import java.util.Set; - import org.apache.storm.daemon.Task; import org.apache.storm.generated.BoltStats; import org.apache.storm.generated.ExecutorSpecificStats; @@ -58,13 +63,15 @@ public void cleanupStats() { super.cleanupStats(); } - public void boltExecuteTuple(String component, String stream, long latencyMs, long workerUptimeSecs, + public void boltExecuteTuple(String component, String stream, long latencyMs, + long workerUptimeSecs, Task firstExecutorTask) { List key = Lists.newArrayList(component, stream); this.getExecuted().incBy(key, this.rate); this.getExecuteLatencies().record(key, latencyMs); - // Calculate capacity: This is really for the whole executor, but we will use the executor's first task + // Calculate capacity: This is really for the whole executor, but we will use the executor's + // first task // for reporting the metric. double capacity = calculateCapacity(workerUptimeSecs); firstExecutorTask.getTaskMetrics().setCapacity(capacity); @@ -72,8 +79,10 @@ public void boltExecuteTuple(String component, String stream, long latencyMs, lo private double calculateCapacity(long workerUptimeSecs) { if (workerUptimeSecs > 0) { - Map execAvg = valueStat(this.getExecuteLatencies()).get(MultiCountStat.TEN_MIN_IN_SECONDS_STR); - Map exec = valueStat(this.getExecuted()).get(MultiCountStat.TEN_MIN_IN_SECONDS_STR); + Map execAvg = valueStat(this.getExecuteLatencies()) + .get(MultiCountStat.TEN_MIN_IN_SECONDS_STR); + Map exec = valueStat(this.getExecuted()) + .get(MultiCountStat.TEN_MIN_IN_SECONDS_STR); Set allKeys = new HashSet<>(); if (execAvg != null) { @@ -90,7 +99,8 @@ private double calculateCapacity(long workerUptimeSecs) { totalAvg += avg * cnt; } - return totalAvg / (Math.min(workerUptimeSecs, MultiCountStat.TEN_MIN_IN_SECONDS) * 1000); + return totalAvg / (Math.min(workerUptimeSecs, + MultiCountStat.TEN_MIN_IN_SECONDS) * 1000); } return 0.0; } @@ -128,11 +138,16 @@ public ExecutorStats renderStats() { // bolt stats BoltStats boltStats = new BoltStats( - ClientStatsUtil.windowSetConverter(valueStat(getAcked()), ClientStatsUtil.TO_GSID, ClientStatsUtil.IDENTITY), - ClientStatsUtil.windowSetConverter(valueStat(getFailed()), ClientStatsUtil.TO_GSID, ClientStatsUtil.IDENTITY), - ClientStatsUtil.windowSetConverter(valueStat(processLatencyStats), ClientStatsUtil.TO_GSID, ClientStatsUtil.IDENTITY), - ClientStatsUtil.windowSetConverter(valueStat(executedStats), ClientStatsUtil.TO_GSID, ClientStatsUtil.IDENTITY), - ClientStatsUtil.windowSetConverter(valueStat(executeLatencyStats), ClientStatsUtil.TO_GSID, ClientStatsUtil.IDENTITY)); + ClientStatsUtil.windowSetConverter(valueStat(getAcked()), ClientStatsUtil.TO_GSID, + ClientStatsUtil.IDENTITY), + ClientStatsUtil.windowSetConverter(valueStat(getFailed()), ClientStatsUtil.TO_GSID, + ClientStatsUtil.IDENTITY), + ClientStatsUtil.windowSetConverter(valueStat(processLatencyStats), + ClientStatsUtil.TO_GSID, ClientStatsUtil.IDENTITY), + ClientStatsUtil.windowSetConverter(valueStat(executedStats), ClientStatsUtil.TO_GSID, + ClientStatsUtil.IDENTITY), + ClientStatsUtil.windowSetConverter(valueStat(executeLatencyStats), + ClientStatsUtil.TO_GSID, ClientStatsUtil.IDENTITY)); ret.set_specific(ExecutorSpecificStats.bolt(boltStats)); return ret; diff --git a/storm-client/src/jvm/org/apache/storm/stats/ClientStatsUtil.java b/storm-client/src/jvm/org/apache/storm/stats/ClientStatsUtil.java index 5eca654635d..9dcf984a9ca 100644 --- a/storm-client/src/jvm/org/apache/storm/stats/ClientStatsUtil.java +++ b/storm-client/src/jvm/org/apache/storm/stats/ClientStatsUtil.java @@ -50,6 +50,7 @@ public static List convertExecutor(List executor) { /** * Make an map of executors to empty stats, in preparation for doing a heartbeat. + * * @param executors the executors as keys of the map * @return and empty map of executors to stats */ @@ -65,7 +66,8 @@ public static Map, ExecutorStats> mkEmptyExecutorZkHbs(Set, ExecutorStats> convertExecutorZkHbs(Map, ExecutorStats> executorBeats) { + public static Map, ExecutorStats> convertExecutorZkHbs(Map, + ExecutorStats> executorBeats) { Map, ExecutorStats> ret = new HashMap<>(); for (Map.Entry, ExecutorStats> entry : executorBeats.entrySet()) { ret.put(convertExecutor(entry.getKey()), entry.getValue()); @@ -75,12 +77,14 @@ public static Map, ExecutorStats> convertExecutorZkHbs(Map mkZkWorkerHb(String topoId, Map, ExecutorStats> executorStats, Integer uptime) { + public static Map mkZkWorkerHb(String topoId, Map, + ExecutorStats> executorStats, Integer uptime) { Map ret = new HashMap<>(); ret.put("storm-id", topoId); ret.put(EXECUTOR_STATS, executorStats); @@ -104,6 +108,7 @@ private static Number getByKeyOr0(Map m, String k) { /** * Get a sub-map by a given key. + * * @param map the original map * @param key the key to get it from * @return the map stored under key @@ -140,6 +145,7 @@ public static ClusterWorkerHeartbeat thriftifyZkWorkerHb(Map hea /** * Converts stats to be over given windows of time. + * * @param stats the stats * @param secKeyFunc transform the sub-key * @param firstKeyFunc transform the main key diff --git a/storm-client/src/jvm/org/apache/storm/stats/CommonStats.java b/storm-client/src/jvm/org/apache/storm/stats/CommonStats.java index 320ab57d50d..7a8d9626219 100644 --- a/storm-client/src/jvm/org/apache/storm/stats/CommonStats.java +++ b/storm-client/src/jvm/org/apache/storm/stats/CommonStats.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/stats/SpoutExecutorStats.java b/storm-client/src/jvm/org/apache/storm/stats/SpoutExecutorStats.java index e13de34c5ea..dd214691536 100644 --- a/storm-client/src/jvm/org/apache/storm/stats/SpoutExecutorStats.java +++ b/storm-client/src/jvm/org/apache/storm/stats/SpoutExecutorStats.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/streams/Edge.java b/storm-client/src/jvm/org/apache/storm/streams/Edge.java index 168918cd66d..a61b836ae64 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/Edge.java +++ b/storm-client/src/jvm/org/apache/storm/streams/Edge.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/streams/GroupingInfo.java b/storm-client/src/jvm/org/apache/storm/streams/GroupingInfo.java index b98e7292730..23947d7e026 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/GroupingInfo.java +++ b/storm-client/src/jvm/org/apache/storm/streams/GroupingInfo.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -30,7 +36,8 @@ private GroupingInfo(Fields fields) { public static GroupingInfo shuffle() { return new GroupingInfo() { @Override - public void declareGrouping(BoltDeclarer declarer, String componentId, String streamId, Fields fields) { + public void declareGrouping(BoltDeclarer declarer, String componentId, String streamId, + Fields fields) { declarer.shuffleGrouping(componentId, streamId); } }; @@ -39,7 +46,8 @@ public void declareGrouping(BoltDeclarer declarer, String componentId, String st public static GroupingInfo fields(Fields fields) { return new GroupingInfo(fields) { @Override - public void declareGrouping(BoltDeclarer declarer, String componentId, String streamId, Fields fields) { + public void declareGrouping(BoltDeclarer declarer, String componentId, String streamId, + Fields fields) { declarer.fieldsGrouping(componentId, streamId, fields); } }; @@ -48,7 +56,8 @@ public void declareGrouping(BoltDeclarer declarer, String componentId, String st public static GroupingInfo global() { return new GroupingInfo() { @Override - public void declareGrouping(BoltDeclarer declarer, String componentId, String streamId, Fields fields) { + public void declareGrouping(BoltDeclarer declarer, String componentId, String streamId, + Fields fields) { declarer.globalGrouping(componentId, streamId); } }; @@ -57,13 +66,15 @@ public void declareGrouping(BoltDeclarer declarer, String componentId, String st public static GroupingInfo all() { return new GroupingInfo() { @Override - public void declareGrouping(BoltDeclarer declarer, String componentId, String streamId, Fields fields) { + public void declareGrouping(BoltDeclarer declarer, String componentId, String streamId, + Fields fields) { declarer.allGrouping(componentId, streamId); } }; } - public abstract void declareGrouping(BoltDeclarer declarer, String componentId, String streamId, Fields fields); + public abstract void declareGrouping(BoltDeclarer declarer, String componentId, String streamId, + Fields fields); public Fields getFields() { return fields; diff --git a/storm-client/src/jvm/org/apache/storm/streams/Node.java b/storm-client/src/jvm/org/apache/storm/streams/Node.java index 9857536c1fd..ec7d95f40f5 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/Node.java +++ b/storm-client/src/jvm/org/apache/storm/streams/Node.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -49,8 +55,10 @@ abstract class Node implements Serializable { this.groupingInfo = groupingInfo; } - Node(String outputStream, Fields outputFields, String componentId, int parallelism, GroupingInfo groupingInfo) { - this(Collections.singleton(outputStream), outputFields, componentId, parallelism, groupingInfo); + Node(String outputStream, Fields outputFields, String componentId, int parallelism, + GroupingInfo groupingInfo) { + this(Collections.singleton(outputStream), outputFields, componentId, parallelism, + groupingInfo); } Node(String outputStream, Fields outputFields, String componentId, GroupingInfo groupingInfo) { @@ -128,7 +136,8 @@ Collection getParentStreams(Node parent) { } Set getParents(String stream) { - Multimap rev = Multimaps.invertFrom(parentStreams, ArrayListMultimap.create()); + Multimap rev = Multimaps.invertFrom(parentStreams, ArrayListMultimap.create()); return new HashSet<>(rev.get(stream)); } diff --git a/storm-client/src/jvm/org/apache/storm/streams/Pair.java b/storm-client/src/jvm/org/apache/storm/streams/Pair.java index bd513ac1e29..4960f48b769 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/Pair.java +++ b/storm-client/src/jvm/org/apache/storm/streams/Pair.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/streams/PairStream.java b/storm-client/src/jvm/org/apache/storm/streams/PairStream.java index d2305564d16..5b3c9117fe2 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/PairStream.java +++ b/storm-client/src/jvm/org/apache/storm/streams/PairStream.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -56,29 +62,34 @@ public class PairStream extends Stream> { } /** - * Returns a new stream by applying a {@link Function} to the value of each key-value pairs in this stream. + * Returns a new stream by applying a {@link Function} to the value of each key-value pairs in + * this stream. * * @param function the mapping function * @return the new stream */ public PairStream mapValues(Function function) { return new PairStream<>(streamBuilder, - addProcessorNode(new MapValuesProcessor<>(function), KEY_VALUE, true)); + addProcessorNode(new MapValuesProcessor<>(function), KEY_VALUE, + true)); } /** - * Return a new stream by applying a {@link FlatMapFunction} function to the value of each key-value pairs in this stream. + * Return a new stream by applying a {@link FlatMapFunction} function to the value of each + * key-value pairs in this stream. * * @param function the flatmap function * @return the new stream */ public PairStream flatMapValues(FlatMapFunction function) { return new PairStream<>(streamBuilder, - addProcessorNode(new FlatMapValuesProcessor<>(function), KEY_VALUE, true)); + addProcessorNode(new FlatMapValuesProcessor<>(function), KEY_VALUE, + true)); } /** - * Aggregates the values for each key of this stream using the given initial value, accumulator and combiner. + * Aggregates the values for each key of this stream using the given initial value, accumulator + * and combiner. * * @param initialValue the initial value of the result * @param accumulator the accumulator @@ -86,8 +97,10 @@ public PairStream flatMapValues(FlatMapFunction PairStream aggregateByKey(R initialValue, - BiFunction accumulator, - BiFunction combiner) { + BiFunction accumulator, + BiFunction combiner) { return combineByKey(CombinerAggregator.of(initialValue, accumulator, combiner)); } @@ -97,7 +110,8 @@ public PairStream aggregateByKey(R initialValue, * @param aggregator the combiner aggregator * @return the new stream */ - public PairStream aggregateByKey(CombinerAggregator aggregator) { + public PairStream aggregateByKey(CombinerAggregator aggregator) { return combineByKey(aggregator); } @@ -111,7 +125,8 @@ public PairStream countByKey() { } /** - * Performs a reduction on the values for each key of this stream by repeatedly applying the reducer. + * Performs a reduction on the values for each key of this stream by repeatedly applying the + * reducer. * * @param reducer the reducer * @return the new stream @@ -130,7 +145,8 @@ public PairStream> groupByKey() { } /** - * Returns a new stream where the values are grouped by keys and the given window. The values that arrive within a window having the + * Returns a new stream where the values are grouped by keys and the given window. The values + * that arrive within a window having the * same key will be merged together and returned as an Iterable of values mapped to the key. * * @param window the window configuration @@ -141,7 +157,8 @@ public PairStream> groupByKeyAndWindow(Window window) { } /** - * Returns a new stream where the values that arrive within a window having the same key will be reduced by repeatedly applying the + * Returns a new stream where the values that arrive within a window having the same key will be + * reduced by repeatedly applying the * reducer. * * @param reducer the reducer @@ -170,8 +187,8 @@ public PairStream filter(Predicate> predicate) { /** * Join the values of this stream with the values having the same key from the other stream. - *

    - * Note: The parallelism of this stream is carried forward to the joined stream. + * + *

    Note: The parallelism of this stream is carried forward to the joined stream. *

    * * @param otherStream the other stream @@ -183,8 +200,8 @@ public PairStream> join(PairStream otherStream) { /** * Join the values of this stream with the values having the same key from the other stream. - *

    - * Note: The parallelism of this stream is carried forward to the joined stream. + * + *

    Note: The parallelism of this stream is carried forward to the joined stream. *

    * * @param otherStream the other stream @@ -202,9 +219,10 @@ public PairStream join(PairStream otherStream, } /** - * Does a left outer join of the values of this stream with the values having the same key from the other stream. - *

    - * Note: The parallelism of this stream is carried forward to the joined stream. + * Does a left outer join of the values of this stream with the values having the same key from + * the other stream. + * + *

    Note: The parallelism of this stream is carried forward to the joined stream. *

    * * @param otherStream the other stream @@ -215,9 +233,10 @@ public PairStream> leftOuterJoin(PairStream otherStre } /** - * Does a left outer join of the values of this stream with the values having the same key from the other stream. - *

    - * Note: The parallelism of this stream is carried forward to the joined stream. + * Does a left outer join of the values of this stream with the values having the same key from + * the other stream. + * + *

    Note: The parallelism of this stream is carried forward to the joined stream. *

    * * @param otherStream the other stream @@ -235,9 +254,10 @@ public PairStream leftOuterJoin(PairStream otherStream, } /** - * Does a right outer join of the values of this stream with the values having the same key from the other stream. - *

    - * Note: The parallelism of this stream is carried forward to the joined stream. + * Does a right outer join of the values of this stream with the values having the same key from + * the other stream. + * + *

    Note: The parallelism of this stream is carried forward to the joined stream. *

    * * @param otherStream the other stream @@ -248,9 +268,10 @@ public PairStream> rightOuterJoin(PairStream otherStr } /** - * Does a right outer join of the values of this stream with the values having the same key from the other stream. - *

    - * Note: The parallelism of this stream is carried forward to the joined stream. + * Does a right outer join of the values of this stream with the values having the same key from + * the other stream. + * + *

    Note: The parallelism of this stream is carried forward to the joined stream. *

    * * @param otherStream the other stream @@ -268,9 +289,10 @@ public PairStream rightOuterJoin(PairStream otherStream, } /** - * Does a full outer join of the values of this stream with the values having the same key from the other stream. - *

    - * Note: The parallelism of this stream is carried forward to the joined stream. + * Does a full outer join of the values of this stream with the values having the same key from + * the other stream. + * + *

    Note: The parallelism of this stream is carried forward to the joined stream. *

    * * @param otherStream the other stream @@ -281,9 +303,10 @@ public PairStream> fullOuterJoin(PairStream otherStre } /** - * Does a full outer join of the values of this stream with the values having the same key from the other stream. - *

    - * Note: The parallelism of this stream is carried forward to the joined stream. + * Does a full outer join of the values of this stream with the values having the same key from + * the other stream. + * + *

    Note: The parallelism of this stream is carried forward to the joined stream. *

    * * @param otherStream the other stream @@ -330,57 +353,70 @@ public PairStream[] branch(Predicate>... predicates) { } /** - * Update the state by applying the given state update function to the previous state of the key and the new value for the key. This - * internally uses {@link org.apache.storm.topology.IStatefulBolt} to save the state. Use {@link Config#TOPOLOGY_STATE_PROVIDER} to + * Update the state by applying the given state update function to the previous state of the key + * and the new value for the key. This + * internally uses {@link org.apache.storm.topology.IStatefulBolt} to save the state. Use {@link + * Config#TOPOLOGY_STATE_PROVIDER} to * choose the state implementation. * * @param stateUpdateFn the state update function * @return the {@link StreamState} which can be used to query the state */ public StreamState updateStateByKey(R initialValue, - BiFunction stateUpdateFn) { + BiFunction stateUpdateFn) { return updateStateByKey(StateUpdater.of(initialValue, stateUpdateFn)); } /** - * Update the state by applying the given state update function to the previous state of the key and the new value for the key. This - * internally uses {@link org.apache.storm.topology.IStatefulBolt} to save the state. Use {@link Config#TOPOLOGY_STATE_PROVIDER} to + * Update the state by applying the given state update function to the previous state of the key + * and the new value for the key. This + * internally uses {@link org.apache.storm.topology.IStatefulBolt} to save the state. Use {@link + * Config#TOPOLOGY_STATE_PROVIDER} to * choose the state implementation. * * @param stateUpdater the state updater * @return the {@link StreamState} which can be used to query the state */ - public StreamState updateStateByKey(StateUpdater stateUpdater) { - // repartition so that state query fields grouping works correctly. this can be optimized further + public StreamState updateStateByKey(StateUpdater stateUpdater) { + // repartition so that state query fields grouping works correctly. this can be optimized + // further return partitionBy(KEY).updateStateByKeyPartition(stateUpdater); } /** * Groups the values of this stream with the values having the same key from the other stream. - *

    - * If stream1 has values - (k1, v1), (k2, v2), (k2, v3)
    and stream2 has values - (k1, x1), (k1, x2), (k3, x3)
    The the - * co-grouped stream would contain - (k1, ([v1], [x1, x2]), (k2, ([v2, v3], [])), (k3, ([], [x3])) + * + *

    If stream1 has values - (k1, v1), (k2, v2), (k2, v3)
    and stream2 has values - (k1, + * x1), + * (k1, x2), (k3, x3)
    The the + * co-grouped stream would contain - (k1, ([v1], [x1, x2]), (k2, ([v2, v3], [])), (k3, ([], + * [x3])) *

    - *

    - * Note: The parallelism of this stream is carried forward to the co-grouped stream. + * + *

    Note: The parallelism of this stream is carried forward to the co-grouped stream. *

    * * @param otherStream the other stream * @return the new stream */ - public PairStream, Iterable>> coGroupByKey(PairStream otherStream) { + public PairStream, Iterable>> coGroupByKey(PairStream otherStream) { return partitionByKey().coGroupByKeyPartition(otherStream); } - - private StreamState updateStateByKeyPartition(StateUpdater stateUpdater) { + private StreamState updateStateByKeyPartition(StateUpdater stateUpdater) { return new StreamState<>( new PairStream<>(streamBuilder, - addProcessorNode(new UpdateStateByKeyProcessor<>(stateUpdater), KEY_VALUE, true))); + addProcessorNode(new UpdateStateByKeyProcessor<>(stateUpdater), + KEY_VALUE, true))); } private PairStream joinPartition(PairStream otherStream, - ValueJoiner valueJoiner, + ValueJoiner valueJoiner, JoinProcessor.JoinType leftType, JoinProcessor.JoinType rightType) { String leftStream = stream; @@ -434,28 +470,35 @@ private PairStream toPairStream(Stream> stream) { return new PairStream<>(stream.streamBuilder, stream.node); } - private PairStream aggregatePartition(CombinerAggregator aggregator) { + private PairStream aggregatePartition(CombinerAggregator aggregator) { return new PairStream<>(streamBuilder, - addProcessorNode(new AggregateByKeyProcessor<>(aggregator), KEY_VALUE, true)); + addProcessorNode(new AggregateByKeyProcessor<>(aggregator), + KEY_VALUE, true)); } private PairStream combinePartition(CombinerAggregator aggregator) { return new PairStream<>(streamBuilder, - addProcessorNode(new AggregateByKeyProcessor<>(aggregator, true), KEY_VALUE, true)); + addProcessorNode(new AggregateByKeyProcessor<>(aggregator, true), + KEY_VALUE, true)); } private PairStream merge(CombinerAggregator aggregator) { return new PairStream<>(streamBuilder, - addProcessorNode(new MergeAggregateByKeyProcessor<>(aggregator), KEY_VALUE, true)); + addProcessorNode(new MergeAggregateByKeyProcessor<>(aggregator), + KEY_VALUE, true)); } private PairStream reducePartition(Reducer reducer) { return new PairStream<>(streamBuilder, - addProcessorNode(new ReduceByKeyProcessor<>(reducer), KEY_VALUE, true)); + addProcessorNode(new ReduceByKeyProcessor<>(reducer), KEY_VALUE, + true)); } - // if re-partitioning is involved, does a per-partition aggregate by key before emitting the results downstream - private PairStream combineByKey(CombinerAggregator aggregator) { + // if re-partitioning is involved, does a per-partition aggregate by key before emitting the + // results downstream + private PairStream combineByKey(CombinerAggregator aggregator) { if (shouldPartitionByKey()) { if (node instanceof ProcessorNode) { if (node.isWindowed()) { @@ -463,7 +506,8 @@ private PairStream combineByKey(CombinerAggregator parents = node.getParents(); - Optional nonWindowed = parents.stream().filter(p -> !p.isWindowed()).findAny(); + Optional nonWindowed = parents.stream().filter(p -> !p.isWindowed()) + .findAny(); if (!nonWindowed.isPresent()) { parents.forEach(p -> { Node localAggregateNode = makeProcessorNode( @@ -479,7 +523,8 @@ private PairStream combineByKey(CombinerAggregator combineByKey(Reducer reducer) { if (shouldPartitionByKey()) { if (node instanceof ProcessorNode) { @@ -489,7 +534,9 @@ private PairStream combineByKey(Reducer reducer) { } else if (node instanceof WindowNode) { for (Node p : node.getParents()) { if (p.isWindowed()) { - Node localReduceNode = makeProcessorNode(new ReduceByKeyProcessor<>(reducer), KEY_VALUE, true); + Node localReduceNode = + makeProcessorNode(new ReduceByKeyProcessor<>(reducer), KEY_VALUE, + true); streamBuilder.insert(p, localReduceNode); } } diff --git a/storm-client/src/jvm/org/apache/storm/streams/PartitionNode.java b/storm-client/src/jvm/org/apache/storm/streams/PartitionNode.java index 372cfc0b3ec..c7e88f2b331 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/PartitionNode.java +++ b/storm-client/src/jvm/org/apache/storm/streams/PartitionNode.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -15,7 +21,8 @@ import org.apache.storm.tuple.Fields; /** - * Node that holds partitioning/grouping information. This is used for operations like groupBy (fields grouping), global aggregate/reduce + * Node that holds partitioning/grouping information. This is used for operations like groupBy + * (fields grouping), global aggregate/reduce * (global grouping), state query (all grouping). */ class PartitionNode extends Node { diff --git a/storm-client/src/jvm/org/apache/storm/streams/ProcessorBolt.java b/storm-client/src/jvm/org/apache/storm/streams/ProcessorBolt.java index 5c8fc13cf78..786e69c8cdd 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/ProcessorBolt.java +++ b/storm-client/src/jvm/org/apache/storm/streams/ProcessorBolt.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -33,7 +39,8 @@ class ProcessorBolt extends BaseRichBolt implements StreamBolt { } @Override - public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) { + public void prepare(Map topoConf, TopologyContext context, + OutputCollector collector) { delegate.prepare(topoConf, context, collector); } @@ -47,7 +54,6 @@ public void declareOutputFields(OutputFieldsDeclarer declarer) { delegate.declareOutputFields(declarer); } - @Override public void setTimestampField(String fieldName) { delegate.setTimestampField(fieldName); diff --git a/storm-client/src/jvm/org/apache/storm/streams/ProcessorBoltDelegate.java b/storm-client/src/jvm/org/apache/storm/streams/ProcessorBoltDelegate.java index ccb6511003e..0074c7eaaef 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/ProcessorBoltDelegate.java +++ b/storm-client/src/jvm/org/apache/storm/streams/ProcessorBoltDelegate.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -98,9 +104,11 @@ void prepare(Map topoConf, TopologyContext context, OutputCollec streamToChildren.put(stream, child); } } - ForwardingProcessorContext forwardingContext = new ForwardingProcessorContext(processorNode, streamToChildren); + ForwardingProcessorContext forwardingContext = + new ForwardingProcessorContext(processorNode, streamToChildren); if (hasOutgoingChild(processorNode, new HashSet<>(children))) { - processorContext = new ChainedProcessorContext(processorNode, forwardingContext, createEmittingContext(processorNode)); + processorContext = new ChainedProcessorContext(processorNode, forwardingContext, + createEmittingContext(processorNode)); } else { processorContext = forwardingContext; } @@ -129,11 +137,14 @@ void declareOutputFields(OutputFieldsDeclarer declarer) { declarer.declareStream(stream, new Fields(fields)); } /* - * Declare a separate 'punctuation' stream per output stream so that the receiving bolt - * can subscribe to this stream with 'ALL' grouping and process the punctuation once it + * Declare a separate 'punctuation' stream per output stream so that the receiving + * bolt + * can subscribe to this stream with 'ALL' grouping and process the punctuation once + * it * receives from all upstream tasks. */ - declarer.declareStream(StreamUtil.getPunctuationStream(stream), StreamUtil.getPunctuationFields()); + declarer.declareStream(StreamUtil.getPunctuationStream(stream), StreamUtil + .getPunctuationFields()); } } } @@ -151,7 +162,8 @@ Pair getValueAndStream(Tuple input) { // otherwise the value is in the first field of the tuple if (input.getSourceComponent().startsWith("spout")) { value = input; - stream = input.getSourceGlobalStreamId().get_componentId() + input.getSourceGlobalStreamId().get_streamId(); + stream = input.getSourceGlobalStreamId().get_componentId() + input + .getSourceGlobalStreamId().get_streamId(); } else if (isPair(input)) { value = Pair.of(input.getValue(0), input.getValue(1)); stream = input.getSourceStreamId(); @@ -243,7 +255,8 @@ private void ack(RefCountedTuple tuple) { private ProcessorContext createEmittingContext(ProcessorNode processorNode) { List emittingContexts = new ArrayList<>(); for (String stream : processorNode.getOutputStreams()) { - EmittingProcessorContext emittingContext = new EmittingProcessorContext(processorNode, outputCollector, stream); + EmittingProcessorContext emittingContext = new EmittingProcessorContext(processorNode, + outputCollector, stream); emittingContexts.add(emittingContext); } emittingProcessorContexts.addAll(emittingContexts); @@ -273,26 +286,31 @@ private Set getChildNodes(Node node) { return children; } - // for the given processor node, if we received punctuation from all tasks of its parent windowed streams + // for the given processor node, if we received punctuation from all tasks of its parent + // windowed streams private boolean shouldPunctuate(ProcessorNode processorNode, String sourceStreamId) { if (!processorNode.getWindowedParentStreams().isEmpty()) { updateCount(processorNode, sourceStreamId); - if (punctuationState.row(processorNode).size() != processorNode.getWindowedParentStreams().size()) { + if (punctuationState.row(processorNode).size() != processorNode + .getWindowedParentStreams().size()) { return false; } // size matches, check if the streams are expected Set receivedStreams = punctuationState.row(processorNode).keySet(); if (!receivedStreams.equals(processorNode.getWindowedParentStreams())) { - throw new IllegalStateException("Received punctuation from streams " + receivedStreams + " expected " + throw new IllegalStateException("Received punctuation from streams " + + receivedStreams + " expected " + processorNode.getWindowedParentStreams()); } for (String receivedStream : receivedStreams) { Integer expected = streamToInputTaskCount.get(receivedStream); if (expected == null) { - throw new IllegalStateException("Punctuation received on unexpected stream '" + receivedStream + throw new IllegalStateException("Punctuation received on unexpected stream '" + + receivedStream + "' for which input task count is not set."); } - if (punctuationState.get(processorNode, receivedStream) < streamToInputTaskCount.get(receivedStream)) { + if (punctuationState.get(processorNode, receivedStream) < streamToInputTaskCount + .get(receivedStream)) { return false; } } diff --git a/storm-client/src/jvm/org/apache/storm/streams/ProcessorNode.java b/storm-client/src/jvm/org/apache/storm/streams/ProcessorNode.java index a92b5fe9886..53b76b702ca 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/ProcessorNode.java +++ b/storm-client/src/jvm/org/apache/storm/streams/ProcessorNode.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -30,7 +36,8 @@ public class ProcessorNode extends Node { // Windowed parent streams private Set windowedParentStreams = Collections.emptySet(); - public ProcessorNode(Processor processor, String outputStream, Fields outputFields, boolean preservesKey) { + public ProcessorNode(Processor processor, String outputStream, Fields outputFields, + boolean preservesKey) { super(outputStream, outputFields); this.isBatch = processor instanceof BatchProcessor; this.processor = processor; diff --git a/storm-client/src/jvm/org/apache/storm/streams/RefCountedTuple.java b/storm-client/src/jvm/org/apache/storm/streams/RefCountedTuple.java index f8874491f46..069858ad3db 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/RefCountedTuple.java +++ b/storm-client/src/jvm/org/apache/storm/streams/RefCountedTuple.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -15,7 +21,8 @@ import org.apache.storm.tuple.Tuple; /** - * Provides reference counting of tuples. Used when operations that operate on a batch of tuples are involved (e.g. aggregation, join etc). + * Provides reference counting of tuples. Used when operations that operate on a batch of tuples are + * involved (e.g. aggregation, join etc). * The input tuples are acked once the result is emitted downstream. */ public class RefCountedTuple { diff --git a/storm-client/src/jvm/org/apache/storm/streams/SinkNode.java b/storm-client/src/jvm/org/apache/storm/streams/SinkNode.java index c12b643d3cd..efebce0e1fe 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/SinkNode.java +++ b/storm-client/src/jvm/org/apache/storm/streams/SinkNode.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/streams/SpoutNode.java b/storm-client/src/jvm/org/apache/storm/streams/SpoutNode.java index d59fe921860..058a1af86c3 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/SpoutNode.java +++ b/storm-client/src/jvm/org/apache/storm/streams/SpoutNode.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/streams/StatefulProcessorBolt.java b/storm-client/src/jvm/org/apache/storm/streams/StatefulProcessorBolt.java index d86008acad7..ed75a037888 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/StatefulProcessorBolt.java +++ b/storm-client/src/jvm/org/apache/storm/streams/StatefulProcessorBolt.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -41,7 +47,8 @@ class StatefulProcessorBolt extends BaseStatefulBolt> } @Override - public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) { + public void prepare(Map topoConf, TopologyContext context, + OutputCollector collector) { delegate.prepare(topoConf, context, collector); } @@ -98,7 +105,8 @@ private Set> getStatefulProcessors(List n statefulProcessors.add((StatefulProcessor) node.getProcessor()); if (node.getProcessor() instanceof UpdateStateByKeyProcessor) { if (++updateStateByKeyCount > 1) { - throw new IllegalArgumentException("Cannot have more than one updateStateByKey processor " + throw new IllegalArgumentException("Cannot have more than one " + + "updateStateByKey processor " + "in a StatefulProcessorBolt"); } } diff --git a/storm-client/src/jvm/org/apache/storm/streams/Stream.java b/storm-client/src/jvm/org/apache/storm/streams/Stream.java index 117b756a425..12994f8abd2 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/Stream.java +++ b/storm-client/src/jvm/org/apache/storm/streams/Stream.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -83,11 +89,13 @@ private Stream(StreamBuilder streamBuilder, Node node, String stream) { * @return the new stream */ public Stream filter(Predicate predicate) { - return new Stream<>(streamBuilder, addProcessorNode(new FilterProcessor<>(predicate), VALUE, true)); + return new Stream<>(streamBuilder, addProcessorNode(new FilterProcessor<>(predicate), VALUE, + true)); } /** - * Returns a stream consisting of the result of applying the given mapping function to the values of this stream. + * Returns a stream consisting of the result of applying the given mapping function to the + * values of this stream. * * @param function a mapping function to be applied to each value in this stream. * @return the new stream @@ -97,34 +105,43 @@ public Stream map(Function function) { } /** - * Returns a stream of key-value pairs by applying a {@link PairFunction} on each value of this stream. + * Returns a stream of key-value pairs by applying a {@link PairFunction} on each value of this + * stream. * * @param function the mapping function to be applied to each value in this stream * @param the key type * @param the value type * @return the new stream of key-value pairs */ - public PairStream mapToPair(PairFunction function) { - return new PairStream<>(streamBuilder, addProcessorNode(new MapProcessor<>(function), KEY_VALUE)); + public PairStream mapToPair(PairFunction function) { + return new PairStream<>(streamBuilder, addProcessorNode(new MapProcessor<>(function), + KEY_VALUE)); } /** - * Returns a stream consisting of the results of replacing each value of this stream with the contents produced by applying the provided - * mapping function to each value. This has the effect of applying a one-to-many transformation to the values of the stream, and then + * Returns a stream consisting of the results of replacing each value of this stream with the + * contents produced by applying the provided + * mapping function to each value. This has the effect of applying a one-to-many transformation + * to the values of the stream, and then * flattening the resulting elements into a new stream. * - * @param function a mapping function to be applied to each value in this stream which produces new values. + * @param function a mapping function to be applied to each value in this stream which produces + * new values. * @return the new stream */ public Stream flatMap(FlatMapFunction function) { - return new Stream<>(streamBuilder, addProcessorNode(new FlatMapProcessor<>(function), VALUE)); + return new Stream<>(streamBuilder, addProcessorNode(new FlatMapProcessor<>(function), + VALUE)); } /** - * Returns a stream consisting of the results of replacing each value of this stream with the key-value pairs produced by applying the + * Returns a stream consisting of the results of replacing each value of this stream with the + * key-value pairs produced by applying the * provided mapping function to each value. * - * @param function the mapping function to be applied to each value in this stream which produces new key-value pairs. + * @param function the mapping function to be applied to each value in this stream which + * produces new key-value pairs. * @param the key type * @param the value type * @return the new stream of key-value pairs @@ -132,13 +149,17 @@ public Stream flatMap(FlatMapFunction function) { * @see #flatMap(FlatMapFunction) * @see #mapToPair(PairFunction) */ - public PairStream flatMapToPair(PairFlatMapFunction function) { - return new PairStream<>(streamBuilder, addProcessorNode(new FlatMapProcessor<>(function), KEY_VALUE)); + public PairStream flatMapToPair(PairFlatMapFunction function) { + return new PairStream<>(streamBuilder, addProcessorNode(new FlatMapProcessor<>(function), + KEY_VALUE)); } /** - * Returns a new stream consisting of the elements that fall within the window as specified by the window parameter. The {@link Window} - * specification could be used to specify sliding or tumbling windows based on time duration or event count. For example, + * Returns a new stream consisting of the elements that fall within the window as specified by + * the window parameter. The {@link Window} + * specification could be used to specify sliding or tumbling windows based on time duration or + * event count. For example, *

          * // time duration based sliding window
          * stream.window(SlidingWindows.of(Duration.minutes(10), Duration.minutes(1));
    @@ -156,7 +177,8 @@ public  PairStream flatMapToPair(PairFlatMapFunction window(Window window) {
    -        return new Stream<>(streamBuilder, addNode(new WindowNode(window, stream, node.getOutputFields())));
    +        return new Stream<>(streamBuilder, addNode(new WindowNode(window, stream, node
    +                .getOutputFields())));
         }
     
         /**
    @@ -169,21 +191,27 @@ public void forEach(Consumer action) {
         }
     
         /**
    -     * Returns a stream consisting of the elements of this stream, additionally performing the provided action on each element as they are
    +     * Returns a stream consisting of the elements of this stream, additionally performing the
    +     * provided action on each element as they are
          * consumed from the resulting stream.
          *
          * @param action the action to perform on the element as they are consumed from the stream
          * @return the new stream
          */
         public Stream peek(Consumer action) {
    -        return new Stream<>(streamBuilder, addProcessorNode(new PeekProcessor<>(action), node.getOutputFields(), true));
    +        return new Stream<>(streamBuilder, addProcessorNode(new PeekProcessor<>(action), node
    +                .getOutputFields(), true));
         }
     
         /**
    -     * Aggregates the values in this stream using the aggregator. This does a global aggregation of values across all partitions.
    -     * 

    - * If the stream is windowed, the aggregate result is emitted after each window activation and represents the aggregate of elements that - * fall within that window. If the stream is not windowed, the aggregate result is emitted as each new element in the stream is + * Aggregates the values in this stream using the aggregator. This does a global aggregation of + * values across all partitions. + * + *

    If the stream is windowed, the aggregate result is emitted after each window activation + * and + * represents the aggregate of elements that + * fall within that window. If the stream is not windowed, the aggregate result is emitted as + * each new element in the stream is * processed. *

    * @@ -197,11 +225,15 @@ public Stream aggregate(CombinerAggregator } /** - * Aggregates the values in this stream using the given initial value, accumulator and combiner. This does a global aggregation of + * Aggregates the values in this stream using the given initial value, accumulator and combiner. + * This does a global aggregation of * values across all partitions. - *

    - * If the stream is windowed, the aggregate result is emitted after each window activation and represents the aggregate of elements that - * fall within that window. If the stream is not windowed, the aggregate result is emitted as each new element in the stream is + * + *

    If the stream is windowed, the aggregate result is emitted after each window activation + * and + * represents the aggregate of elements that + * fall within that window. If the stream is not windowed, the aggregate result is emitted as + * each new element in the stream is * processed. *

    * @@ -218,10 +250,14 @@ public Stream aggregate(R initialValue, } /** - * Counts the number of values in this stream. This does a global count of values across all partitions. - *

    - * If the stream is windowed, the counts are emitted after each window activation and represents the count of elements that fall within - * that window. If the stream is not windowed, the count is emitted as each new element in the stream is processed. + * Counts the number of values in this stream. This does a global count of values across all + * partitions. + * + *

    If the stream is windowed, the counts are emitted after each window activation and + * represents + * the count of elements that fall within + * that window. If the stream is not windowed, the count is emitted as each new element in the + * stream is processed. *

    * * @return the new stream @@ -231,11 +267,15 @@ public Stream count() { } /** - * Performs a reduction on the elements of this stream, by repeatedly applying the reducer. This does a global reduction of values + * Performs a reduction on the elements of this stream, by repeatedly applying the reducer. This + * does a global reduction of values * across all partitions. - *

    - * If the stream is windowed, the result is emitted after each window activation and represents the reduction of elements that fall - * within that window. If the stream is not windowed, the result is emitted as each new element in the stream is processed. + * + *

    If the stream is windowed, the result is emitted after each window activation and + * represents + * the reduction of elements that fall + * within that window. If the stream is not windowed, the result is emitted as each new element + * in the stream is processed. *

    * * @param reducer the reducer @@ -246,7 +286,8 @@ public Stream reduce(Reducer reducer) { } /** - * Returns a new stream with the given value of parallelism. Further operations on this stream would execute at this level of + * Returns a new stream with the given value of parallelism. Further operations on this stream + * would execute at this level of * parallelism. * * @param parallelism the parallelism value @@ -257,19 +298,23 @@ public Stream repartition(int parallelism) { throw new IllegalArgumentException("Parallelism should be >= 1"); } if (node.getParallelism() == parallelism) { - LOG.debug("Node's current parallelism {}, new parallelism {}", node.getParallelism(), parallelism); + LOG.debug("Node's current parallelism {}, new parallelism {}", node.getParallelism(), + parallelism); return this; } - Node partitionNode = addNode(node, new PartitionNode(stream, node.getOutputFields()), parallelism); + Node partitionNode = addNode(node, new PartitionNode(stream, node.getOutputFields()), + parallelism); return new Stream<>(streamBuilder, partitionNode); } /** - * Returns an array of streams by splitting the given stream into multiple branches based on the given predicates. The predicates are - * applied in the given order to the values of this stream and the result is forwarded to the corresponding (index based) result stream + * Returns an array of streams by splitting the given stream into multiple branches based on the + * given predicates. The predicates are + * applied in the given order to the values of this stream and the result is forwarded to the + * corresponding (index based) result stream * based on the (index of) predicate that matches. - *

    - * Note: If none of the predicates match a value, that value is dropped. + * + *

    Note: If none of the predicates match a value, that value is dropped. *

    * * @param predicates the predicates @@ -283,7 +328,9 @@ public Stream[] branch(Predicate... predicates) { Node branchNode = addProcessorNode(branchProcessor, VALUE); for (Predicate predicate : predicates) { // create a child node (identity) per branch - ProcessorNode child = makeProcessorNode(new MapProcessor<>(new IdentityFunction<>()), node.getOutputFields()); + ProcessorNode child = + makeProcessorNode(new MapProcessor<>(new IdentityFunction<>()), node + .getOutputFields()); String branchStream = child.getOutputStreams().iterator().next() + "-branch"; // branchStream is the parent stream that connects branch processor to this child branchNode.addOutputStream(branchStream); @@ -305,8 +352,8 @@ public void print() { /** * Sends the elements of this stream to a bolt. This could be used to plug in existing bolts as sinks in the stream, for e.g. a {@code * RedisStoreBolt}. The bolt would have a parallelism of 1. - *

    - * Note: This would provide guarantees only based on what the bolt provides. + * + *

    Note: This would provide guarantees only based on what the bolt provides. *

    * * @param bolt the bolt @@ -318,8 +365,8 @@ public void to(IRichBolt bolt) { /** * Sends the elements of this stream to a bolt. This could be used to plug in existing bolts as sinks in the stream, for e.g. a {@code * RedisStoreBolt}. - *

    - * Note: This would provide guarantees only based on what the bolt provides. + * + *

    Note: This would provide guarantees only based on what the bolt provides. *

    * * @param bolt the bolt @@ -332,8 +379,8 @@ public void to(IRichBolt bolt, int parallelism) { /** * Sends the elements of this stream to a bolt. This could be used to plug in existing bolts as sinks in the stream, for e.g. a {@code * RedisStoreBolt}. The bolt would have a parallelism of 1. - *

    - * Note: This would provide guarantees only based on what the bolt provides. + * + *

    Note: This would provide guarantees only based on what the bolt provides. *

    * * @param bolt the bolt @@ -345,8 +392,8 @@ public void to(IBasicBolt bolt) { /** * Sends the elements of this stream to a bolt. This could be used to plug in existing bolts as sinks in the stream, for e.g. a {@code * RedisStoreBolt}. - *

    - * Note: This would provide guarantees only based on what the bolt provides. + * + *

    Note: This would provide guarantees only based on what the bolt provides. *

    * * @param bolt the bolt @@ -365,7 +412,8 @@ public void to(IBasicBolt bolt, int parallelism) { */ public PairStream stateQuery(StreamState streamState) { // need field grouping for state query so that the query is routed to the correct task - Node newNode = partitionBy(VALUE, node.getParallelism()).addProcessorNode(new StateQueryProcessor<>(streamState), KEY_VALUE); + Node newNode = partitionBy(VALUE, node.getParallelism()) + .addProcessorNode(new StateQueryProcessor<>(streamState), KEY_VALUE); return new PairStream<>(streamBuilder, newNode); } @@ -409,8 +457,10 @@ private ProcessorNode makeProcessorNode(Processor processor, Fields outputFie return makeProcessorNode(processor, outputFields, false); } - ProcessorNode makeProcessorNode(Processor processor, Fields outputFields, boolean preservesKey) { - return new ProcessorNode(processor, UniqueIdGen.getInstance().getUniqueStreamId(), outputFields, preservesKey); + ProcessorNode makeProcessorNode(Processor processor, Fields outputFields, + boolean preservesKey) { + return new ProcessorNode(processor, UniqueIdGen.getInstance().getUniqueStreamId(), + outputFields, preservesKey); } private void addSinkNode(SinkNode sinkNode, int parallelism) { @@ -425,14 +475,16 @@ private void addSinkNode(SinkNode sinkNode, int parallelism) { } private Stream global() { - Node partitionNode = addNode(new PartitionNode(stream, node.getOutputFields(), GroupingInfo.global())); + Node partitionNode = addNode(new PartitionNode(stream, node.getOutputFields(), GroupingInfo + .global())); return new Stream<>(streamBuilder, partitionNode); } protected Stream partitionBy(Fields fields, int parallelism) { return new Stream<>( streamBuilder, - addNode(node, new PartitionNode(stream, node.getOutputFields(), GroupingInfo.fields(fields)), parallelism)); + addNode(node, new PartitionNode(stream, node.getOutputFields(), GroupingInfo + .fields(fields)), parallelism)); } private boolean shouldPartition() { @@ -441,7 +493,8 @@ private boolean shouldPartition() { private
    Stream combinePartition(CombinerAggregator aggregator) { return new Stream<>(streamBuilder, - addProcessorNode(new AggregateProcessor<>(aggregator, true), VALUE, true)); + addProcessorNode(new AggregateProcessor<>(aggregator, true), VALUE, + true)); } private Stream merge(CombinerAggregator aggregator) { @@ -449,15 +502,18 @@ private Stream merge(CombinerAggregator aggregator) { addProcessorNode(new MergeAggregateProcessor<>(aggregator), VALUE)); } - private Stream aggregatePartition(CombinerAggregator aggregator) { - return new Stream<>(streamBuilder, addProcessorNode(new AggregateProcessor<>(aggregator), VALUE)); + private Stream aggregatePartition(CombinerAggregator aggregator) { + return new Stream<>(streamBuilder, addProcessorNode(new AggregateProcessor<>(aggregator), + VALUE)); } private Stream reducePartition(Reducer reducer) { return new Stream<>(streamBuilder, addProcessorNode(new ReduceProcessor<>(reducer), VALUE)); } - // if re-partitioning is involved, does a per-partition aggregate before emitting the results downstream + // if re-partitioning is involved, does a per-partition aggregate before emitting the results + // downstream private Stream combine(CombinerAggregator aggregator) { if (shouldPartition()) { if (node instanceof ProcessorNode) { @@ -466,7 +522,8 @@ private Stream combine(CombinerAggregator a } } else if (node instanceof WindowNode) { Set parents = node.getParents(); - Optional nonWindowed = parents.stream().filter(p -> !p.isWindowed()).findAny(); + Optional nonWindowed = parents.stream().filter(p -> !p.isWindowed()) + .findAny(); if (!nonWindowed.isPresent()) { parents.forEach(p -> { Node localAggregateNode = makeProcessorNode( @@ -482,7 +539,8 @@ private Stream combine(CombinerAggregator a } } - // if re-partitioning is involved, does a per-partition reduce before emitting the results downstream + // if re-partitioning is involved, does a per-partition reduce before emitting the results + // downstream private Stream combine(Reducer reducer) { if (shouldPartition()) { if (node instanceof ProcessorNode) { @@ -492,7 +550,8 @@ private Stream combine(Reducer reducer) { } else if (node instanceof WindowNode) { for (Node p : node.getParents()) { if (p.isWindowed()) { - Node localReduceNode = makeProcessorNode(new ReduceProcessor<>(reducer), VALUE); + Node localReduceNode = makeProcessorNode(new ReduceProcessor<>(reducer), + VALUE); streamBuilder.insert(p, localReduceNode); } } diff --git a/storm-client/src/jvm/org/apache/storm/streams/StreamBolt.java b/storm-client/src/jvm/org/apache/storm/streams/StreamBolt.java index afcf1340e9f..e69d82ea60f 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/StreamBolt.java +++ b/storm-client/src/jvm/org/apache/storm/streams/StreamBolt.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/streams/StreamBuilder.java b/storm-client/src/jvm/org/apache/storm/streams/StreamBuilder.java index 32192652b6c..5fda0601418 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/StreamBuilder.java +++ b/storm-client/src/jvm/org/apache/storm/streams/StreamBuilder.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -81,7 +87,8 @@ public Stream newStream(IRichSpout spout) { } /** - * Creates a new {@link Stream} of tuples from the given {@link IRichSpout} with the given parallelism. + * Creates a new {@link Stream} of tuples from the given {@link IRichSpout} with the given + * parallelism. * * @param spout the spout * @param parallelism the parallelism of the stream @@ -97,7 +104,8 @@ public Stream newStream(IRichSpout spout, int parallelism) { } /** - * Creates a new {@link Stream} of values from the given {@link IRichSpout} by extracting field(s) from tuples via the supplied {@link + * Creates a new {@link Stream} of values from the given {@link IRichSpout} by extracting + * field(s) from tuples via the supplied {@link * TupleValueMapper}. * * @param spout the spout @@ -108,9 +116,9 @@ public Stream newStream(IRichSpout spout, TupleValueMapper valueMapper return newStream(spout).map(valueMapper); } - /** - * Creates a new {@link Stream} of values from the given {@link IRichSpout} by extracting field(s) from tuples via the supplied {@link + * Creates a new {@link Stream} of values from the given {@link IRichSpout} by extracting + * field(s) from tuples via the supplied {@link * TupleValueMapper} with the given parallelism. * * @param spout the spout @@ -118,24 +126,28 @@ public Stream newStream(IRichSpout spout, TupleValueMapper valueMapper * @param parallelism the parallelism of the stream * @return the new stream */ - public Stream newStream(IRichSpout spout, TupleValueMapper valueMapper, int parallelism) { + public Stream newStream(IRichSpout spout, TupleValueMapper valueMapper, + int parallelism) { return newStream(spout, parallelism).map(valueMapper); } /** - * Creates a new {@link PairStream} of key-value pairs from the given {@link IRichSpout} by extracting key and value from tuples via the + * Creates a new {@link PairStream} of key-value pairs from the given {@link IRichSpout} by + * extracting key and value from tuples via the * supplied {@link PairValueMapper}. * * @param spout the spout * @param pairValueMapper the pair value mapper * @return the new stream of key-value pairs */ - public PairStream newStream(IRichSpout spout, PairValueMapper pairValueMapper) { + public PairStream newStream(IRichSpout spout, PairValueMapper pairValueMapper) { return newStream(spout).mapToPair(pairValueMapper); } /** - * Creates a new {@link PairStream} of key-value pairs from the given {@link IRichSpout} by extracting key and value from tuples via the + * Creates a new {@link PairStream} of key-value pairs from the given {@link IRichSpout} by + * extracting key and value from tuples via the * supplied {@link PairValueMapper} and with the given value of parallelism. * * @param spout the spout @@ -143,11 +155,11 @@ public PairStream newStream(IRichSpout spout, PairValueMapper * @param parallelism the parallelism of the stream * @return the new stream of key-value pairs */ - public PairStream newStream(IRichSpout spout, PairValueMapper pairValueMapper, int parallelism) { + public PairStream newStream(IRichSpout spout, PairValueMapper pairValueMapper, int parallelism) { return newStream(spout, parallelism).mapToPair(pairValueMapper); } - /** * Builds a new {@link StormTopology} for the computation expressed via the stream api. * @@ -157,7 +169,8 @@ public StormTopology build() { nodeGroupingInfo.clear(); windowInfo.clear(); curGroup.clear(); - TopologicalOrderIterator iterator = new TopologicalOrderIterator<>(graph, priorityComparator()); + TopologicalOrderIterator iterator = new TopologicalOrderIterator<>(graph, + priorityComparator()); TopologyBuilder topologyBuilder = new TopologyBuilder(); while (iterator.hasNext()) { Node node = iterator.next(); @@ -182,7 +195,8 @@ public StormTopology build() { } Node addNode(Node parent, Node child) { - return addNode(parent, child, parent.getOutputStreams().iterator().next(), parent.getParallelism()); + return addNode(parent, child, parent.getOutputStreams().iterator().next(), parent + .getParallelism()); } Node addNode(Node parent, Node child, int parallelism) { @@ -240,8 +254,10 @@ private Comparator priorityComparator() { /* * Nodes in the descending order of priority. * ProcessorNode has higher priority than partition and window nodes - * so that the topological order iterator will group as many processor nodes together as possible. - * UpdateStateByKeyProcessor has a higher priority than StateQueryProcessor so that StateQueryProcessor + * so that the topological order iterator will group as many processor nodes together as + * possible. + * UpdateStateByKeyProcessor has a higher priority than StateQueryProcessor so that + * StateQueryProcessor * can be mapped to the same StatefulBolt that UpdateStateByKeyProcessor is part of. */ Map, Integer> map = new HashMap<>(); @@ -305,7 +321,8 @@ private void handleProcessorNode(ProcessorNode processorNode, TopologyBuilder to * force create a windowed bolt with identity nodes so that we don't * have a stateful processor inside a windowed bolt. */ - private void splitStatefulProcessor(ProcessorNode processorNode, TopologyBuilder topologyBuilder) { + private void splitStatefulProcessor(ProcessorNode processorNode, + TopologyBuilder topologyBuilder) { for (Node parent : StreamUtil.getParents(graph, processorNode)) { ProcessorNode identity = new ProcessorNode(new MapProcessor<>(new IdentityFunction<>()), @@ -354,7 +371,8 @@ private void updateWindowInfo(WindowNode windowNode) { Node parentNode(Node curNode) { Set parentNode = parentNodes(curNode); if (parentNode.size() > 1) { - throw new IllegalArgumentException("Node " + curNode + " has more than one parent node."); + throw new IllegalArgumentException("Node " + curNode + + " has more than one parent node."); } if (parentNode.isEmpty()) { throw new IllegalArgumentException("Node " + curNode + " has no parent."); @@ -375,7 +393,8 @@ private Set parentNodes(Node curNode) { } private Collection> parallelismGroups(List processorNodes) { - return processorNodes.stream().collect(Collectors.groupingBy(Node::getParallelism)).values(); + return processorNodes.stream().collect(Collectors.groupingBy(Node::getParallelism)) + .values(); } private void processCurGroup(TopologyBuilder topologyBuilder) { @@ -400,9 +419,11 @@ private void doProcessCurGroup(TopologyBuilder topologyBuilder, List processorNodes) { } private int getParallelism(List group) { - Set parallelisms = group.stream().map(Node::getParallelism).collect(Collectors.toSet()); + Set parallelisms = group.stream().map(Node::getParallelism).collect(Collectors + .toSet()); if (parallelisms.size() > 1) { - throw new IllegalStateException("Current group does not have same parallelism " + group); + throw new IllegalStateException("Current group does not have same parallelism " + + group); } return parallelisms.isEmpty() ? 1 : parallelisms.iterator().next(); @@ -448,9 +471,11 @@ private void addSink(TopologyBuilder topologyBuilder, SinkNode sinkNode) { IComponent bolt = sinkNode.getBolt(); BoltDeclarer boltDeclarer; if (bolt instanceof IRichBolt) { - boltDeclarer = topologyBuilder.setBolt(sinkNode.getComponentId(), (IRichBolt) bolt, sinkNode.getParallelism()); + boltDeclarer = topologyBuilder.setBolt(sinkNode.getComponentId(), (IRichBolt) bolt, + sinkNode.getParallelism()); } else if (bolt instanceof IBasicBolt) { - boltDeclarer = topologyBuilder.setBolt(sinkNode.getComponentId(), (IBasicBolt) bolt, sinkNode.getParallelism()); + boltDeclarer = topologyBuilder.setBolt(sinkNode.getComponentId(), (IBasicBolt) bolt, + sinkNode.getParallelism()); } else { throw new IllegalArgumentException("Expect IRichBolt or IBasicBolt in addBolt"); } @@ -480,18 +505,21 @@ private StreamBolt addStatefulBolt(TopologyBuilder topologyBuilder, StatefulProcessorBolt bolt; if (stateQueryProcessor == null) { bolt = new StatefulProcessorBolt<>(boltId, graph, group); - BoltDeclarer boltDeclarer = topologyBuilder.setBolt(boltId, bolt, getParallelism(group)); + BoltDeclarer boltDeclarer = topologyBuilder.setBolt(boltId, bolt, + getParallelism(group)); bolt.setStreamToInitialProcessors(wireBolt(group, boltDeclarer, initialProcessors)); streamBolts.put(bolt, boltDeclarer); } else { // state query is added to the existing stateful bolt - ProcessorNode updateStateNode = stateQueryProcessor.getStreamState().getUpdateStateNode(); + ProcessorNode updateStateNode = stateQueryProcessor.getStreamState() + .getUpdateStateNode(); bolt = findStatefulProcessorBolt(updateStateNode); for (ProcessorNode node : group) { node.setComponentId(bolt.getId()); } bolt.addNodes(group); - bolt.addStreamToInitialProcessors(wireBolt(bolt.getNodes(), streamBolts.get(bolt), initialProcessors)); + bolt.addStreamToInitialProcessors(wireBolt(bolt.getNodes(), streamBolts.get(bolt), + initialProcessors)); } return bolt; } @@ -526,7 +554,8 @@ private StreamBolt addWindowedBolt(TopologyBuilder topologyBuilder, } } } - throw new IllegalArgumentException("Could not find Stateful bolt for node " + updateStateNode); + throw new IllegalArgumentException("Could not find Stateful bolt for node " + + updateStateNode); } private Set getWindowedParentStreams(ProcessorNode processorNode) { @@ -542,7 +571,8 @@ private Set getWindowedParentStreams(ProcessorNode processorNode) { private Multimap wireBolt(List group, BoltDeclarer boltDeclarer, Set initialProcessors) { - LOG.debug("Wiring bolt with boltDeclarer {}, group {}, initialProcessors {}, nodeGroupingInfo {}", + LOG.debug("Wiring bolt with boltDeclarer {}, group {}, initialProcessors {}, " + + "nodeGroupingInfo {}", boltDeclarer, group, initialProcessors, nodeGroupingInfo); Multimap streamToInitialProcessor = ArrayListMultimap.create(); Set curSet = new HashSet<>(group); @@ -552,14 +582,16 @@ private Multimap wireBolt(List group, LOG.debug("Parent {} of curNode {} is in group {}", parent, curNode, group); } else { for (String stream : curNode.getParentStreams(parent)) { - declareGrouping(boltDeclarer, parent, stream, nodeGroupingInfo.get(parent, stream)); + declareGrouping(boltDeclarer, parent, stream, nodeGroupingInfo.get(parent, + stream)); // put global stream id for spouts if (parent.getComponentId().startsWith("spout")) { stream = parent.getComponentId() + stream; } else { // subscribe to parent's punctuation stream String punctuationStream = StreamUtil.getPunctuationStream(stream); - declareGrouping(boltDeclarer, parent, punctuationStream, GroupingInfo.all()); + declareGrouping(boltDeclarer, parent, punctuationStream, GroupingInfo + .all()); } streamToInitialProcessor.put(stream, curNode); } @@ -569,11 +601,13 @@ private Multimap wireBolt(List group, return streamToInitialProcessor; } - private void declareGrouping(BoltDeclarer boltDeclarer, Node parent, String streamId, GroupingInfo grouping) { + private void declareGrouping(BoltDeclarer boltDeclarer, Node parent, String streamId, + GroupingInfo grouping) { if (grouping == null) { boltDeclarer.shuffleGrouping(parent.getComponentId(), streamId); } else { - grouping.declareGrouping(boltDeclarer, parent.getComponentId(), streamId, grouping.getFields()); + grouping.declareGrouping(boltDeclarer, parent.getComponentId(), streamId, grouping + .getFields()); } } diff --git a/storm-client/src/jvm/org/apache/storm/streams/StreamState.java b/storm-client/src/jvm/org/apache/storm/streams/StreamState.java index 2f798a1ebec..728fa111374 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/StreamState.java +++ b/storm-client/src/jvm/org/apache/storm/streams/StreamState.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -15,7 +21,8 @@ import java.io.Serializable; /** - * A wrapper for the stream state which can be used to query the state via {@link Stream#stateQuery(StreamState)}. + * A wrapper for the stream state which can be used to query the state via {@link + * Stream#stateQuery(StreamState)}. * * @param the key type * @param the value type diff --git a/storm-client/src/jvm/org/apache/storm/streams/StreamUtil.java b/storm-client/src/jvm/org/apache/storm/streams/StreamUtil.java index a6ca3f09a79..9f5f45fb136 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/StreamUtil.java +++ b/storm-client/src/jvm/org/apache/storm/streams/StreamUtil.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/streams/UniqueIdGen.java b/storm-client/src/jvm/org/apache/storm/streams/UniqueIdGen.java index bd18456f46c..c5345301598 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/UniqueIdGen.java +++ b/storm-client/src/jvm/org/apache/storm/streams/UniqueIdGen.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/streams/WindowNode.java b/storm-client/src/jvm/org/apache/storm/streams/WindowNode.java index 0d769d8acf0..800ebfa0fad 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/WindowNode.java +++ b/storm-client/src/jvm/org/apache/storm/streams/WindowNode.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/streams/WindowedProcessorBolt.java b/storm-client/src/jvm/org/apache/storm/streams/WindowedProcessorBolt.java index 1a6df0524f5..63674411843 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/WindowedProcessorBolt.java +++ b/storm-client/src/jvm/org/apache/storm/streams/WindowedProcessorBolt.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -48,7 +54,8 @@ class WindowedProcessorBolt extends BaseWindowedBolt implements StreamBolt { } @Override - public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) { + public void prepare(Map topoConf, TopologyContext context, + OutputCollector collector) { delegate.prepare(topoConf, context, collector); } diff --git a/storm-client/src/jvm/org/apache/storm/streams/operations/BiFunction.java b/storm-client/src/jvm/org/apache/storm/streams/operations/BiFunction.java index 690b768579f..189fb743c97 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/operations/BiFunction.java +++ b/storm-client/src/jvm/org/apache/storm/streams/operations/BiFunction.java @@ -1,19 +1,25 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.streams.operations; /** - * a function that accepts two arguments and produces a result. + * A function that accepts two arguments and produces a result. * * @param the type of the first argument to the function * @param the type of the second argument to the function diff --git a/storm-client/src/jvm/org/apache/storm/streams/operations/CombinerAggregator.java b/storm-client/src/jvm/org/apache/storm/streams/operations/CombinerAggregator.java index bb36fbe67f2..778f407d9af 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/operations/CombinerAggregator.java +++ b/storm-client/src/jvm/org/apache/storm/streams/operations/CombinerAggregator.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -17,7 +23,8 @@ */ public interface CombinerAggregator extends Operation { /** - * A static factory to create a {@link CombinerAggregator} based on initial value, accumulator and combiner. + * A static factory to create a {@link CombinerAggregator} based on initial value, accumulator + * and combiner. * * @param initialValue the initial value of the result to start with * @param accumulator a function that accumulates values into a partial result @@ -25,8 +32,10 @@ public interface CombinerAggregator extends Operation { * @return the {@link CombinerAggregator} */ static CombinerAggregator of(R initialValue, - BiFunction accumulator, - BiFunction combiner) { + BiFunction accumulator, + BiFunction combiner) { return new CombinerAggregator() { @Override public R init() { diff --git a/storm-client/src/jvm/org/apache/storm/streams/operations/Consumer.java b/storm-client/src/jvm/org/apache/storm/streams/operations/Consumer.java index 5b5a9318067..a84626da936 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/operations/Consumer.java +++ b/storm-client/src/jvm/org/apache/storm/streams/operations/Consumer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/streams/operations/FlatMapFunction.java b/storm-client/src/jvm/org/apache/storm/streams/operations/FlatMapFunction.java index d4f220a858f..b3974ee1b1d 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/operations/FlatMapFunction.java +++ b/storm-client/src/jvm/org/apache/storm/streams/operations/FlatMapFunction.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/streams/operations/Function.java b/storm-client/src/jvm/org/apache/storm/streams/operations/Function.java index 09d45a9c2ac..204d9a7646f 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/operations/Function.java +++ b/storm-client/src/jvm/org/apache/storm/streams/operations/Function.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/streams/operations/IdentityFunction.java b/storm-client/src/jvm/org/apache/storm/streams/operations/IdentityFunction.java index 9d27e2e5aa1..d72ded8983d 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/operations/IdentityFunction.java +++ b/storm-client/src/jvm/org/apache/storm/streams/operations/IdentityFunction.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/streams/operations/Operation.java b/storm-client/src/jvm/org/apache/storm/streams/operations/Operation.java index 8b7df3f4a86..bb03dcfe176 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/operations/Operation.java +++ b/storm-client/src/jvm/org/apache/storm/streams/operations/Operation.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/streams/operations/PairFlatMapFunction.java b/storm-client/src/jvm/org/apache/storm/streams/operations/PairFlatMapFunction.java index 53852d6b2da..e6072a23b84 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/operations/PairFlatMapFunction.java +++ b/storm-client/src/jvm/org/apache/storm/streams/operations/PairFlatMapFunction.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -15,7 +21,8 @@ import org.apache.storm.streams.Pair; /** - * A function that accepts one argument and returns an {@link Iterable} of {@link Pair} as its result. + * A function that accepts one argument and returns an {@link Iterable} of {@link Pair} as its + * result. * * @param the type of the input to the function * @param the key type of the key-value pairs produced as a result diff --git a/storm-client/src/jvm/org/apache/storm/streams/operations/PairFunction.java b/storm-client/src/jvm/org/apache/storm/streams/operations/PairFunction.java index 51c1ff956b4..d3e0a1a28e0 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/operations/PairFunction.java +++ b/storm-client/src/jvm/org/apache/storm/streams/operations/PairFunction.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/streams/operations/PairValueJoiner.java b/storm-client/src/jvm/org/apache/storm/streams/operations/PairValueJoiner.java index 5d854f33202..1242520d87a 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/operations/PairValueJoiner.java +++ b/storm-client/src/jvm/org/apache/storm/streams/operations/PairValueJoiner.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -15,7 +21,8 @@ import org.apache.storm.streams.Pair; /** - * A {@link ValueJoiner} that joins two values to produce a {@link Pair} of the two values as the result. + * A {@link ValueJoiner} that joins two values to produce a {@link Pair} of the two values as the + * result. * * @param the type of the first value * @param the type of the second value diff --git a/storm-client/src/jvm/org/apache/storm/streams/operations/Predicate.java b/storm-client/src/jvm/org/apache/storm/streams/operations/Predicate.java index 5a93c66f142..2e80471b6da 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/operations/Predicate.java +++ b/storm-client/src/jvm/org/apache/storm/streams/operations/Predicate.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/streams/operations/PrintConsumer.java b/storm-client/src/jvm/org/apache/storm/streams/operations/PrintConsumer.java index ae8825a8352..5f073d9b5a2 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/operations/PrintConsumer.java +++ b/storm-client/src/jvm/org/apache/storm/streams/operations/PrintConsumer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/streams/operations/Reducer.java b/storm-client/src/jvm/org/apache/storm/streams/operations/Reducer.java index 8f2f4bbc48f..08aab77aa47 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/operations/Reducer.java +++ b/storm-client/src/jvm/org/apache/storm/streams/operations/Reducer.java @@ -1,19 +1,26 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.streams.operations; /** - * The {@link Reducer} performs an operation on two values of the same type producing a result of the same type. + * The {@link Reducer} performs an operation on two values of the same type producing a result of + * the same type. * * @param the type of the arguments and the result */ diff --git a/storm-client/src/jvm/org/apache/storm/streams/operations/StateUpdater.java b/storm-client/src/jvm/org/apache/storm/streams/operations/StateUpdater.java index bbdf08ef531..7398ee6b6fb 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/operations/StateUpdater.java +++ b/storm-client/src/jvm/org/apache/storm/streams/operations/StateUpdater.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -17,14 +23,16 @@ */ public interface StateUpdater extends Operation { /** - * A static factory to create a {@link StateUpdater} based on an initial value of the state and a state update function. + * A static factory to create a {@link StateUpdater} based on an initial value of the state and + * a state update function. * * @param initialValue the intial value of the state * @param stateUpdateFn the state update function * @return the {@link StateUpdater} */ static StateUpdater of(S initialValue, - BiFunction stateUpdateFn) { + BiFunction stateUpdateFn) { return new StateUpdater() { @Override public S init() { diff --git a/storm-client/src/jvm/org/apache/storm/streams/operations/ValueJoiner.java b/storm-client/src/jvm/org/apache/storm/streams/operations/ValueJoiner.java index c4adc173c47..d6a78fbfea9 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/operations/ValueJoiner.java +++ b/storm-client/src/jvm/org/apache/storm/streams/operations/ValueJoiner.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/streams/operations/aggregators/Count.java b/storm-client/src/jvm/org/apache/storm/streams/operations/aggregators/Count.java index c91b538279a..8aa66a001ad 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/operations/aggregators/Count.java +++ b/storm-client/src/jvm/org/apache/storm/streams/operations/aggregators/Count.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/streams/operations/aggregators/LongSum.java b/storm-client/src/jvm/org/apache/storm/streams/operations/aggregators/LongSum.java index 344a842347a..aeb2a55ed22 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/operations/aggregators/LongSum.java +++ b/storm-client/src/jvm/org/apache/storm/streams/operations/aggregators/LongSum.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/streams/operations/mappers/PairValueMapper.java b/storm-client/src/jvm/org/apache/storm/streams/operations/mappers/PairValueMapper.java index 7d86826de60..697ea5755e5 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/operations/mappers/PairValueMapper.java +++ b/storm-client/src/jvm/org/apache/storm/streams/operations/mappers/PairValueMapper.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -27,7 +33,8 @@ public class PairValueMapper implements TupleValueMapper>, Pair private final int valueIndex; /** - * Constructs a new {@link PairValueMapper} that constructs a pair from a tuple based on the key and value index. + * Constructs a new {@link PairValueMapper} that constructs a pair from a tuple based on the key + * and value index. * * @param keyIndex the key index * @param valueIndex the value index diff --git a/storm-client/src/jvm/org/apache/storm/streams/operations/mappers/TupleValueMapper.java b/storm-client/src/jvm/org/apache/storm/streams/operations/mappers/TupleValueMapper.java index d008e049f36..cbae83bfa29 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/operations/mappers/TupleValueMapper.java +++ b/storm-client/src/jvm/org/apache/storm/streams/operations/mappers/TupleValueMapper.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/streams/operations/mappers/TupleValueMappers.java b/storm-client/src/jvm/org/apache/storm/streams/operations/mappers/TupleValueMappers.java index 50c09822ce0..48a7deb04e4 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/operations/mappers/TupleValueMappers.java +++ b/storm-client/src/jvm/org/apache/storm/streams/operations/mappers/TupleValueMappers.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

    - * http://www.apache.org/licenses/LICENSE-2.0 - *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

    http://www.apache.org/licenses/LICENSE-2.0 + * + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/streams/operations/mappers/ValueMapper.java b/storm-client/src/jvm/org/apache/storm/streams/operations/mappers/ValueMapper.java index b0d30602403..dd54cb6c17b 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/operations/mappers/ValueMapper.java +++ b/storm-client/src/jvm/org/apache/storm/streams/operations/mappers/ValueMapper.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/streams/operations/mappers/ValuesMapper.java b/storm-client/src/jvm/org/apache/storm/streams/operations/mappers/ValuesMapper.java index 40838dfd5fb..9610c2fb19d 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/operations/mappers/ValuesMapper.java +++ b/storm-client/src/jvm/org/apache/storm/streams/operations/mappers/ValuesMapper.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -22,7 +28,8 @@ public class ValuesMapper implements TupleValueMapper { private final int[] indices; /** - * Constructs a new {@link ValuesMapper} that extracts value from a {@link Tuple} at specified indices. + * Constructs a new {@link ValuesMapper} that extracts value from a {@link Tuple} at specified + * indices. * * @param indices the indices */ diff --git a/storm-client/src/jvm/org/apache/storm/streams/processors/AggregateByKeyProcessor.java b/storm-client/src/jvm/org/apache/storm/streams/processors/AggregateByKeyProcessor.java index 78307299fe6..8f28118b9c7 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/processors/AggregateByKeyProcessor.java +++ b/storm-client/src/jvm/org/apache/storm/streams/processors/AggregateByKeyProcessor.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/streams/processors/AggregateProcessor.java b/storm-client/src/jvm/org/apache/storm/streams/processors/AggregateProcessor.java index 952d7cea724..2c4e33811ac 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/processors/AggregateProcessor.java +++ b/storm-client/src/jvm/org/apache/storm/streams/processors/AggregateProcessor.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/streams/processors/BaseProcessor.java b/storm-client/src/jvm/org/apache/storm/streams/processors/BaseProcessor.java index 06486ac1718..9adfd28815b 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/processors/BaseProcessor.java +++ b/storm-client/src/jvm/org/apache/storm/streams/processors/BaseProcessor.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -19,7 +25,8 @@ import java.util.function.Supplier; /** - * Base implementation of the {@link Processor} interface that provides convenience methods {@link #execute(Object)} and {@link #finish()}. + * Base implementation of the {@link Processor} interface that provides convenience methods {@link + * #execute(Object)} and {@link #finish()}. */ abstract class BaseProcessor implements Processor { private final Set punctuationState = new HashSet<>(); @@ -34,7 +41,8 @@ public void init(ProcessorContext context) { } /** - * {@inheritDoc} Processors that do not care about the source stream should override {@link BaseProcessor#execute(Object)} instead. + * {@inheritDoc} Processors that do not care about the source stream should override {@link + * BaseProcessor#execute(Object)} instead. */ @Override public void execute(T input, String streamId) { @@ -42,7 +50,8 @@ public void execute(T input, String streamId) { } /** - * Execute some operation on the input value. Sub classes can override this when then don't care about the source stream from where the + * Execute some operation on the input value. Sub classes can override this when then don't care + * about the source stream from where the * input is received. * * @param input the input @@ -64,8 +73,10 @@ public void punctuate(String stream) { } /** - * This is triggered to signal the end of the current batch of values. Sub classes can override this to emit the result of a batch of - * values, for e.g. to emit the result of an aggregate or join operation on a batch of values. If a processor does per-value operation + * This is triggered to signal the end of the current batch of values. Sub classes can override + * this to emit the result of a batch of + * values, for e.g. to emit the result of an aggregate or join operation on a batch of values. + * If a processor does per-value operation * like filter, map etc, they can choose to ignore this. */ protected void finish() { @@ -73,7 +84,8 @@ protected void finish() { } /** - * Forwards the result update to downstream processors. Processors that operate on a batch of tuples, like aggregation, join etc can use + * Forwards the result update to downstream processors. Processors that operate on a batch of + * tuples, like aggregation, join etc can use * this to emit the partial results on each input if they are operating in non-windowed mode. * * @param result the result function diff --git a/storm-client/src/jvm/org/apache/storm/streams/processors/BatchProcessor.java b/storm-client/src/jvm/org/apache/storm/streams/processors/BatchProcessor.java index 3bfdcbb5288..289bf65a536 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/processors/BatchProcessor.java +++ b/storm-client/src/jvm/org/apache/storm/streams/processors/BatchProcessor.java @@ -1,19 +1,26 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.streams.processors; /** - * Top level marker interface for processors that computes results for a batch of tuples like Aggregate, Join etc. + * Top level marker interface for processors that computes results for a batch of tuples like + * Aggregate, Join etc. */ public interface BatchProcessor { } diff --git a/storm-client/src/jvm/org/apache/storm/streams/processors/BranchProcessor.java b/storm-client/src/jvm/org/apache/storm/streams/processors/BranchProcessor.java index 13060042598..d2e11affa52 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/processors/BranchProcessor.java +++ b/storm-client/src/jvm/org/apache/storm/streams/processors/BranchProcessor.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/streams/processors/ChainedProcessorContext.java b/storm-client/src/jvm/org/apache/storm/streams/processors/ChainedProcessorContext.java index c79ced8d256..2b05073b7e9 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/processors/ChainedProcessorContext.java +++ b/storm-client/src/jvm/org/apache/storm/streams/processors/ChainedProcessorContext.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -25,7 +31,8 @@ public class ChainedProcessorContext implements ProcessorContext { private final ProcessorNode processorNode; private final List contexts; - public ChainedProcessorContext(ProcessorNode processorNode, List contexts) { + public ChainedProcessorContext(ProcessorNode processorNode, + List contexts) { this.processorNode = processorNode; this.contexts = new ArrayList<>(contexts); } diff --git a/storm-client/src/jvm/org/apache/storm/streams/processors/CoGroupByKeyProcessor.java b/storm-client/src/jvm/org/apache/storm/streams/processors/CoGroupByKeyProcessor.java index af638cb1dbb..194a6c64e42 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/processors/CoGroupByKeyProcessor.java +++ b/storm-client/src/jvm/org/apache/storm/streams/processors/CoGroupByKeyProcessor.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -26,7 +32,6 @@ public class CoGroupByKeyProcessor extends BaseProcessor> private final Multimap firstMap = ArrayListMultimap.create(); private final Multimap secondMap = ArrayListMultimap.create(); - public CoGroupByKeyProcessor(String firstStream, String secondStream) { this.firstStream = firstStream; this.secondStream = secondStream; @@ -57,11 +62,13 @@ public void finish() { private void forwardValues() { firstMap.asMap().forEach((key, values) -> { - context.forward(Pair.of(key, Pair.of(new ArrayList<>(values), secondMap.removeAll(key)))); + context.forward(Pair.of(key, Pair.of(new ArrayList<>(values), secondMap + .removeAll(key)))); }); secondMap.asMap().forEach((key, values) -> { - context.forward(Pair.of(key, Pair.of(firstMap.removeAll(key), new ArrayList<>(values)))); + context.forward(Pair.of(key, Pair.of(firstMap.removeAll(key), + new ArrayList<>(values)))); }); } diff --git a/storm-client/src/jvm/org/apache/storm/streams/processors/EmittingProcessorContext.java b/storm-client/src/jvm/org/apache/storm/streams/processors/EmittingProcessorContext.java index 079db436dae..e96a046aac5 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/processors/EmittingProcessorContext.java +++ b/storm-client/src/jvm/org/apache/storm/streams/processors/EmittingProcessorContext.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -45,7 +51,8 @@ public class EmittingProcessorContext implements ProcessorContext { private long eventTimestamp; private String timestampField; - public EmittingProcessorContext(ProcessorNode processorNode, OutputCollector collector, String outputStreamId) { + public EmittingProcessorContext(ProcessorNode processorNode, OutputCollector collector, + String outputStreamId) { this.processorNode = processorNode; this.outputStreamId = outputStreamId; this.collector = collector; @@ -100,7 +107,8 @@ public void setAnchor(RefCountedTuple anchor) { } /* * track punctuation in non-batch mode so that the - * punctuation is acked after all the processors have emitted the punctuation downstream. + * punctuation is acked after all the processors have emitted the punctuation + * downstream. */ if (StreamUtil.isPunctuation(anchor.tuple().getValue(0))) { anchor.increment(); @@ -139,7 +147,8 @@ private void emit(Values values, String outputStreamId) { LOG.debug("Emit un-anchored, outputStreamId: {}, values: {}", outputStreamId, values); collector.emit(outputStreamId, values); } else { - LOG.debug("Emit, outputStreamId: {}, anchors: {}, values: {}", outputStreamId, anchors, values); + LOG.debug("Emit, outputStreamId: {}, anchors: {}, values: {}", outputStreamId, anchors, + values); collector.emit(outputStreamId, tuples(anchors), values); } } diff --git a/storm-client/src/jvm/org/apache/storm/streams/processors/FilterProcessor.java b/storm-client/src/jvm/org/apache/storm/streams/processors/FilterProcessor.java index beda18236a0..48b0c3df857 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/processors/FilterProcessor.java +++ b/storm-client/src/jvm/org/apache/storm/streams/processors/FilterProcessor.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/streams/processors/FlatMapProcessor.java b/storm-client/src/jvm/org/apache/storm/streams/processors/FlatMapProcessor.java index 364b729cfc9..20c80a94449 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/processors/FlatMapProcessor.java +++ b/storm-client/src/jvm/org/apache/storm/streams/processors/FlatMapProcessor.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/streams/processors/FlatMapValuesProcessor.java b/storm-client/src/jvm/org/apache/storm/streams/processors/FlatMapValuesProcessor.java index aeadaa9b784..aa0708d3737 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/processors/FlatMapValuesProcessor.java +++ b/storm-client/src/jvm/org/apache/storm/streams/processors/FlatMapValuesProcessor.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/streams/processors/ForEachProcessor.java b/storm-client/src/jvm/org/apache/storm/streams/processors/ForEachProcessor.java index 571737f84ec..00a504455ba 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/processors/ForEachProcessor.java +++ b/storm-client/src/jvm/org/apache/storm/streams/processors/ForEachProcessor.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/streams/processors/ForwardingProcessorContext.java b/storm-client/src/jvm/org/apache/storm/streams/processors/ForwardingProcessorContext.java index b436df2f19a..7752466c459 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/processors/ForwardingProcessorContext.java +++ b/storm-client/src/jvm/org/apache/storm/streams/processors/ForwardingProcessorContext.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -29,7 +35,8 @@ public class ForwardingProcessorContext implements ProcessorContext { private final Multimap streamToChildren; private final Set streams; - public ForwardingProcessorContext(ProcessorNode processorNode, Multimap streamToChildren) { + public ForwardingProcessorContext(ProcessorNode processorNode, Multimap streamToChildren) { this.processorNode = processorNode; this.streamToChildren = streamToChildren; this.streams = streamToChildren.keySet(); diff --git a/storm-client/src/jvm/org/apache/storm/streams/processors/JoinProcessor.java b/storm-client/src/jvm/org/apache/storm/streams/processors/JoinProcessor.java index 6295c26be80..bfb7f8a3a26 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/processors/JoinProcessor.java +++ b/storm-client/src/jvm/org/apache/storm/streams/processors/JoinProcessor.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -35,7 +41,8 @@ public class JoinProcessor extends BaseProcessor> imple private final JoinType leftType; private final JoinType rightType; - public JoinProcessor(String leftStream, String rightStream, ValueJoiner valueJoiner) { + public JoinProcessor(String leftStream, String rightStream, ValueJoiner valueJoiner) { this(leftStream, rightStream, valueJoiner, JoinType.INNER, JoinType.INNER); } @@ -89,11 +96,13 @@ public String getRightStream() { */ private void joinAndForward(List> leftRows, List> rightRows) { if (leftRows.size() < rightRows.size()) { - for (Tuple3 res : join(getJoinTable(leftRows), rightRows, leftType, rightType)) { + for (Tuple3 res : join(getJoinTable(leftRows), rightRows, leftType, + rightType)) { context.forward(Pair.of(res.value1, valueJoiner.apply(res.value2, res.value3))); } } else { - for (Tuple3 res : join(getJoinTable(rightRows), leftRows, rightType, leftType)) { + for (Tuple3 res : join(getJoinTable(rightRows), leftRows, rightType, + leftType)) { context.forward(Pair.of(res.value1, valueJoiner.apply(res.value3, res.value2))); } } diff --git a/storm-client/src/jvm/org/apache/storm/streams/processors/MapProcessor.java b/storm-client/src/jvm/org/apache/storm/streams/processors/MapProcessor.java index eb51855ea35..093690da110 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/processors/MapProcessor.java +++ b/storm-client/src/jvm/org/apache/storm/streams/processors/MapProcessor.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/streams/processors/MapValuesProcessor.java b/storm-client/src/jvm/org/apache/storm/streams/processors/MapValuesProcessor.java index c3e061e8c0c..b97a5bec92a 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/processors/MapValuesProcessor.java +++ b/storm-client/src/jvm/org/apache/storm/streams/processors/MapValuesProcessor.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/streams/processors/MergeAggregateByKeyProcessor.java b/storm-client/src/jvm/org/apache/storm/streams/processors/MergeAggregateByKeyProcessor.java index ab27cf5f0ac..462c3e2aec4 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/processors/MergeAggregateByKeyProcessor.java +++ b/storm-client/src/jvm/org/apache/storm/streams/processors/MergeAggregateByKeyProcessor.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/streams/processors/MergeAggregateProcessor.java b/storm-client/src/jvm/org/apache/storm/streams/processors/MergeAggregateProcessor.java index d388e1a63d1..cce2caf2d8c 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/processors/MergeAggregateProcessor.java +++ b/storm-client/src/jvm/org/apache/storm/streams/processors/MergeAggregateProcessor.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/streams/processors/PeekProcessor.java b/storm-client/src/jvm/org/apache/storm/streams/processors/PeekProcessor.java index f5ecec428e6..6a3b269e726 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/processors/PeekProcessor.java +++ b/storm-client/src/jvm/org/apache/storm/streams/processors/PeekProcessor.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/streams/processors/Processor.java b/storm-client/src/jvm/org/apache/storm/streams/processors/Processor.java index 8fb0248b7f9..c02577d93fd 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/processors/Processor.java +++ b/storm-client/src/jvm/org/apache/storm/streams/processors/Processor.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -23,7 +29,8 @@ @InterfaceStability.Unstable public interface Processor extends Serializable { /** - * Initializes the processor. This is typically invoked from the underlying storm bolt's prepare method. + * Initializes the processor. This is typically invoked from the underlying storm bolt's prepare + * method. * * @param context the processor context */ @@ -38,7 +45,8 @@ public interface Processor extends Serializable { void execute(T input, String streamId); /** - * Punctuation marks end of a batch which can be used to compute and pass the results of one stage in the pipeline to the next. For e.g. + * Punctuation marks end of a batch which can be used to compute and pass the results of one + * stage in the pipeline to the next. For e.g. * emit the results of an aggregation. * * @param stream the stream id on which the punctuation arrived diff --git a/storm-client/src/jvm/org/apache/storm/streams/processors/ProcessorContext.java b/storm-client/src/jvm/org/apache/storm/streams/processors/ProcessorContext.java index 91bd424c577..5f540cb1628 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/processors/ProcessorContext.java +++ b/storm-client/src/jvm/org/apache/storm/streams/processors/ProcessorContext.java @@ -1,12 +1,17 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -37,7 +42,8 @@ public interface ProcessorContext extends Serializable { void forward(T input, String stream); /** - * Returns true if the processing is in a windowed context and should wait for punctuation before emitting results. + * Returns true if the processing is in a windowed context and should wait for punctuation + * before emitting results. * * @return whether this is a windowed context or not */ diff --git a/storm-client/src/jvm/org/apache/storm/streams/processors/ReduceByKeyProcessor.java b/storm-client/src/jvm/org/apache/storm/streams/processors/ReduceByKeyProcessor.java index 0a412d4ced4..3a2957a7074 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/processors/ReduceByKeyProcessor.java +++ b/storm-client/src/jvm/org/apache/storm/streams/processors/ReduceByKeyProcessor.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/streams/processors/ReduceProcessor.java b/storm-client/src/jvm/org/apache/storm/streams/processors/ReduceProcessor.java index 4f3ba69ec78..5c4a82b01b0 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/processors/ReduceProcessor.java +++ b/storm-client/src/jvm/org/apache/storm/streams/processors/ReduceProcessor.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/streams/processors/StateQueryProcessor.java b/storm-client/src/jvm/org/apache/storm/streams/processors/StateQueryProcessor.java index 310d84c98e0..dcaa5603516 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/processors/StateQueryProcessor.java +++ b/storm-client/src/jvm/org/apache/storm/streams/processors/StateQueryProcessor.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/streams/processors/StatefulProcessor.java b/storm-client/src/jvm/org/apache/storm/streams/processors/StatefulProcessor.java index 68ae103fe9e..caf9a1deea7 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/processors/StatefulProcessor.java +++ b/storm-client/src/jvm/org/apache/storm/streams/processors/StatefulProcessor.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/streams/processors/UpdateStateByKeyProcessor.java b/storm-client/src/jvm/org/apache/storm/streams/processors/UpdateStateByKeyProcessor.java index 73a87b44316..f419517d037 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/processors/UpdateStateByKeyProcessor.java +++ b/storm-client/src/jvm/org/apache/storm/streams/processors/UpdateStateByKeyProcessor.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/streams/tuple/Tuple10.java b/storm-client/src/jvm/org/apache/storm/streams/tuple/Tuple10.java index 38d08f24c82..8cd4da22509 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/tuple/Tuple10.java +++ b/storm-client/src/jvm/org/apache/storm/streams/tuple/Tuple10.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -53,7 +59,8 @@ public class Tuple10 { * @param value9 the ninth element * @param value10 the tenth element */ - public Tuple10(T1 value1, T2 value2, T3 value3, T4 value4, T5 value5, T6 value6, T7 value7, T8 value8, T9 value9, T10 value10) { + public Tuple10(T1 value1, T2 value2, T3 value3, T4 value4, T5 value5, T6 value6, T7 value7, + T8 value8, T9 value9, T10 value10) { this.value1 = value1; this.value2 = value2; this.value3 = value3; diff --git a/storm-client/src/jvm/org/apache/storm/streams/tuple/Tuple3.java b/storm-client/src/jvm/org/apache/storm/streams/tuple/Tuple3.java index ebf1477de26..bdd1d388270 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/tuple/Tuple3.java +++ b/storm-client/src/jvm/org/apache/storm/streams/tuple/Tuple3.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/streams/tuple/Tuple4.java b/storm-client/src/jvm/org/apache/storm/streams/tuple/Tuple4.java index e4577151996..3acccd61cc0 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/tuple/Tuple4.java +++ b/storm-client/src/jvm/org/apache/storm/streams/tuple/Tuple4.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/streams/tuple/Tuple5.java b/storm-client/src/jvm/org/apache/storm/streams/tuple/Tuple5.java index 21d52713e94..d35f7fd0ffe 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/tuple/Tuple5.java +++ b/storm-client/src/jvm/org/apache/storm/streams/tuple/Tuple5.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/streams/tuple/Tuple6.java b/storm-client/src/jvm/org/apache/storm/streams/tuple/Tuple6.java index 6891f433a39..2a7398225fb 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/tuple/Tuple6.java +++ b/storm-client/src/jvm/org/apache/storm/streams/tuple/Tuple6.java @@ -1,18 +1,23 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.streams.tuple; - /** * A tuple of six elements along the lines of Scala's Tuple. * @@ -93,6 +98,7 @@ public int hashCode() { @Override public String toString() { - return "(" + value1 + "," + value2 + "," + value3 + "," + value4 + "," + value5 + "," + value6 + ")"; + return "(" + value1 + "," + value2 + "," + value3 + "," + value4 + "," + value5 + "," + + value6 + ")"; } } diff --git a/storm-client/src/jvm/org/apache/storm/streams/tuple/Tuple7.java b/storm-client/src/jvm/org/apache/storm/streams/tuple/Tuple7.java index c2aab4268b2..df7e4d71f76 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/tuple/Tuple7.java +++ b/storm-client/src/jvm/org/apache/storm/streams/tuple/Tuple7.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -100,6 +106,7 @@ public int hashCode() { @Override public String toString() { - return "(" + value1 + "," + value2 + "," + value3 + "," + value4 + "," + value5 + "," + value6 + "," + value7 + ")"; + return "(" + value1 + "," + value2 + "," + value3 + "," + value4 + "," + value5 + "," + + value6 + "," + value7 + ")"; } } diff --git a/storm-client/src/jvm/org/apache/storm/streams/tuple/Tuple8.java b/storm-client/src/jvm/org/apache/storm/streams/tuple/Tuple8.java index 11a85f83ee9..4556c243b1c 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/tuple/Tuple8.java +++ b/storm-client/src/jvm/org/apache/storm/streams/tuple/Tuple8.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -46,7 +52,8 @@ public class Tuple8 { * @param value7 the seventh element * @param value8 the eighth element */ - public Tuple8(T1 value1, T2 value2, T3 value3, T4 value4, T5 value5, T6 value6, T7 value7, T8 value8) { + public Tuple8(T1 value1, T2 value2, T3 value3, T4 value4, T5 value5, T6 value6, T7 value7, + T8 value8) { this.value1 = value1; this.value2 = value2; this.value3 = value3; @@ -108,6 +115,7 @@ public int hashCode() { @Override public String toString() { - return "(" + value1 + "," + value2 + "," + value3 + "," + value4 + "," + value5 + "," + value6 + "," + value7 + "," + value8 + ")"; + return "(" + value1 + "," + value2 + "," + value3 + "," + value4 + "," + value5 + "," + + value6 + "," + value7 + "," + value8 + ")"; } } diff --git a/storm-client/src/jvm/org/apache/storm/streams/tuple/Tuple9.java b/storm-client/src/jvm/org/apache/storm/streams/tuple/Tuple9.java index 9f64b183350..60f0d556473 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/tuple/Tuple9.java +++ b/storm-client/src/jvm/org/apache/storm/streams/tuple/Tuple9.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -49,7 +55,8 @@ public class Tuple9 { * @param value8 the eighth element * @param value9 the ninth element */ - public Tuple9(T1 value1, T2 value2, T3 value3, T4 value4, T5 value5, T6 value6, T7 value7, T8 value8, T9 value9) { + public Tuple9(T1 value1, T2 value2, T3 value3, T4 value4, T5 value5, T6 value6, T7 value7, + T8 value8, T9 value9) { this.value1 = value1; this.value2 = value2; this.value3 = value3; diff --git a/storm-client/src/jvm/org/apache/storm/streams/windowing/BaseWindow.java b/storm-client/src/jvm/org/apache/storm/streams/windowing/BaseWindow.java index dd66c2f6723..9990b257492 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/windowing/BaseWindow.java +++ b/storm-client/src/jvm/org/apache/storm/streams/windowing/BaseWindow.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -45,10 +51,12 @@ public boolean equals(Object o) { BaseWindow that = (BaseWindow) o; - if (timestampField != null ? !timestampField.equals(that.timestampField) : that.timestampField != null) { + if (timestampField != null ? !timestampField + .equals(that.timestampField) : that.timestampField != null) { return false; } - if (lateTupleStream != null ? !lateTupleStream.equals(that.lateTupleStream) : that.lateTupleStream != null) { + if (lateTupleStream != null ? !lateTupleStream + .equals(that.lateTupleStream) : that.lateTupleStream != null) { return false; } return lag != null ? lag.equals(that.lag) : that.lag == null; diff --git a/storm-client/src/jvm/org/apache/storm/streams/windowing/SlidingWindows.java b/storm-client/src/jvm/org/apache/storm/streams/windowing/SlidingWindows.java index 254c3e18c8e..87794429c8e 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/windowing/SlidingWindows.java +++ b/storm-client/src/jvm/org/apache/storm/streams/windowing/SlidingWindows.java @@ -1,20 +1,26 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.streams.windowing; -import org.apache.storm.topology.base.BaseWindowedBolt; import org.apache.storm.topology.base.BaseWindowedBolt.Count; import org.apache.storm.topology.base.BaseWindowedBolt.Duration; +import org.apache.storm.topology.base.BaseWindowedBolt; /** * A sliding window specification based on a window length and sliding interval. @@ -47,7 +53,8 @@ public static SlidingWindows of(Count windowLength, Count slidingI * @param windowLength the time duration of the window * @param slidingInterval the time duration after which the window slides */ - public static SlidingWindows of(Duration windowLength, Duration slidingInterval) { + public static SlidingWindows of(Duration windowLength, + Duration slidingInterval) { return new SlidingWindows<>(windowLength, slidingInterval); } @@ -88,8 +95,10 @@ public I getSlidingInterval() { } /** - * The name of the field in the tuple that contains the timestamp when the event occurred as a long value. This is used of event-time - * based processing. If this config is set and the field is not present in the incoming tuple, an {@link IllegalArgumentException} will + * The name of the field in the tuple that contains the timestamp when the event occurred as a + * long value. This is used of event-time + * based processing. If this config is set and the field is not present in the incoming tuple, + * an {@link IllegalArgumentException} will * be thrown. * * @param fieldName the name of the field that contains the timestamp @@ -100,12 +109,17 @@ public SlidingWindows withTimestampField(String fieldName) { } /** - * Specify a stream id on which late tuples are going to be emitted. They are going to be accessible via the {@link - * org.apache.storm.topology.WindowedBoltExecutor#LATE_TUPLE_FIELD} field. It must be defined on a per-component basis, and in - * conjunction with the {@link BaseWindowedBolt#withTimestampField}, otherwise {@link IllegalArgumentException} will be thrown. + * Specify a stream id on which late tuples are going to be emitted. They are going to be + * accessible via the {@link + * org.apache.storm.topology.WindowedBoltExecutor#LATE_TUPLE_FIELD} field. It must be defined on + * a per-component basis, and in + * conjunction with the {@link BaseWindowedBolt#withTimestampField}, otherwise {@link + * IllegalArgumentException} will be thrown. * - *

    The late tuple is emitted as a {@link org.apache.storm.tuple.DetachedTuple}, a serializable copy of the original - * tuple detached from the topology context, so it can be consumed by bolts running in other workers. + *

    The late tuple is emitted as a {@link org.apache.storm.tuple.DetachedTuple}, a + * serializable copy of the original + * tuple detached from the topology context, so it can be consumed by bolts running in other + * workers. * * @param streamId the name of the stream used to emit late tuples on */ @@ -115,7 +129,8 @@ public SlidingWindows withLateTupleStream(String streamId) { } /** - * Specify the maximum time lag of the tuple timestamp in milliseconds. It means that the tuple timestamps cannot be out of order by + * Specify the maximum time lag of the tuple timestamp in milliseconds. It means that the tuple + * timestamps cannot be out of order by * more than this amount. * * @param duration the max lag duration @@ -139,10 +154,12 @@ public boolean equals(Object o) { SlidingWindows that = (SlidingWindows) o; - if (windowLength != null ? !windowLength.equals(that.windowLength) : that.windowLength != null) { + if (windowLength != null ? !windowLength + .equals(that.windowLength) : that.windowLength != null) { return false; } - return slidingInterval != null ? slidingInterval.equals(that.slidingInterval) : that.slidingInterval == null; + return slidingInterval != null ? slidingInterval + .equals(that.slidingInterval) : that.slidingInterval == null; } diff --git a/storm-client/src/jvm/org/apache/storm/streams/windowing/TumblingWindows.java b/storm-client/src/jvm/org/apache/storm/streams/windowing/TumblingWindows.java index c06f29d9cc8..c1d67a80ae3 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/windowing/TumblingWindows.java +++ b/storm-client/src/jvm/org/apache/storm/streams/windowing/TumblingWindows.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -58,8 +64,10 @@ public L getSlidingInterval() { } /** - * The name of the field in the tuple that contains the timestamp when the event occurred as a long value. This is used of event-time - * based processing. If this config is set and the field is not present in the incoming tuple, an {@link IllegalArgumentException} will + * The name of the field in the tuple that contains the timestamp when the event occurred as a + * long value. This is used of event-time + * based processing. If this config is set and the field is not present in the incoming tuple, + * an {@link IllegalArgumentException} will * be thrown. * * @param fieldName the name of the field that contains the timestamp @@ -70,12 +78,17 @@ public TumblingWindows withTimestampField(String fieldName) { } /** - * Specify a stream id on which late tuples are going to be emitted. They are going to be accessible via the {@link - * org.apache.storm.topology.WindowedBoltExecutor#LATE_TUPLE_FIELD} field. It must be defined on a per-component basis, and in - * conjunction with the {@link BaseWindowedBolt#withTimestampField}, otherwise {@link IllegalArgumentException} will be thrown. + * Specify a stream id on which late tuples are going to be emitted. They are going to be + * accessible via the {@link + * org.apache.storm.topology.WindowedBoltExecutor#LATE_TUPLE_FIELD} field. It must be defined on + * a per-component basis, and in + * conjunction with the {@link BaseWindowedBolt#withTimestampField}, otherwise {@link + * IllegalArgumentException} will be thrown. * - *

    The late tuple is emitted as a {@link org.apache.storm.tuple.DetachedTuple}, a serializable copy of the original - * tuple detached from the topology context, so it can be consumed by bolts running in other workers. + *

    The late tuple is emitted as a {@link org.apache.storm.tuple.DetachedTuple}, a + * serializable copy of the original + * tuple detached from the topology context, so it can be consumed by bolts running in other + * workers. * * @param streamId the name of the stream used to emit late tuples on */ @@ -85,7 +98,8 @@ public TumblingWindows withLateTupleStream(String streamId) { } /** - * Specify the maximum time lag of the tuple timestamp in milliseconds. It means that the tuple timestamps cannot be out of order by + * Specify the maximum time lag of the tuple timestamp in milliseconds. It means that the tuple + * timestamps cannot be out of order by * more than this amount. * * @param duration the max lag duration @@ -109,7 +123,8 @@ public boolean equals(Object o) { TumblingWindows that = (TumblingWindows) o; - return windowLength != null ? windowLength.equals(that.windowLength) : that.windowLength == null; + return windowLength != null ? windowLength + .equals(that.windowLength) : that.windowLength == null; } diff --git a/storm-client/src/jvm/org/apache/storm/streams/windowing/Window.java b/storm-client/src/jvm/org/apache/storm/streams/windowing/Window.java index 986426f3aac..356fd322f77 100644 --- a/storm-client/src/jvm/org/apache/storm/streams/windowing/Window.java +++ b/storm-client/src/jvm/org/apache/storm/streams/windowing/Window.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -39,8 +45,10 @@ public interface Window extends Serializable { I getSlidingInterval(); /** - * The name of the field in the tuple that contains the timestamp when the event occurred as a long value. This is used of event-time - * based processing. If this config is set and the field is not present in the incoming tuple, an {@link IllegalArgumentException} will + * The name of the field in the tuple that contains the timestamp when the event occurred as a + * long value. This is used of event-time + * based processing. If this config is set and the field is not present in the incoming tuple, + * an {@link IllegalArgumentException} will * be thrown. * * @return the timestamp field. @@ -48,14 +56,16 @@ public interface Window extends Serializable { String getTimestampField(); /** - * The name of the stream where late arriving tuples should be emitted. If this is not provided, the late tuples would be discarded. + * The name of the stream where late arriving tuples should be emitted. If this is not provided, + * the late tuples would be discarded. * * @return the name of the stream used to emit late tuples on */ String getLateTupleStream(); /** - * The maximum time lag of the tuple timestamp in milliseconds. It means that the tuple timestamps cannot be out of order by more than + * The maximum time lag of the tuple timestamp in milliseconds. It means that the tuple + * timestamps cannot be out of order by more than * this amount. * * @return the lag diff --git a/storm-client/src/jvm/org/apache/storm/task/GeneralTopologyContext.java b/storm-client/src/jvm/org/apache/storm/task/GeneralTopologyContext.java index 05ce547cf5e..477b5e9265f 100644 --- a/storm-client/src/jvm/org/apache/storm/task/GeneralTopologyContext.java +++ b/storm-client/src/jvm/org/apache/storm/task/GeneralTopologyContext.java @@ -60,7 +60,8 @@ public GeneralTopologyContext(StormTopology topology, Map topoCo } /** - * Gets the unique id assigned to this topology. The id is the storm name with a unique nonce appended to it. + * Gets the unique id assigned to this topology. The id is the storm name with a unique nonce + * appended to it. * * @return the storm id */ @@ -78,7 +79,8 @@ public StormTopology getRawTopology() { } /** - * Gets the component id for the specified task id. The component id maps to a component id specified for a Spout or Bolt in the + * Gets the component id for the specified task id. The component id maps to a component id + * specified for a Spout or Bolt in the * topology definition. * * @param taskId the task id @@ -100,7 +102,8 @@ public Set getComponentStreams(String componentId) { } /** - * Gets the task ids allocated for the given component id. The task ids are always returned in ascending order. + * Gets the task ids allocated for the given component id. The task ids are always returned in + * ascending order. */ public List getComponentTasks(String componentId) { List ret = componentToTasks.get(componentId); @@ -117,7 +120,8 @@ public List getComponentTasks(String componentId) { public Fields getComponentOutputFields(String componentId, String streamId) { Fields ret = componentToStreamToFields.get(componentId).get(streamId); if (ret == null) { - throw new IllegalArgumentException("No output fields defined for component:stream " + componentId + ":" + streamId); + throw new IllegalArgumentException("No output fields defined for component:stream " + + componentId + ":" + streamId); } return ret; } @@ -146,7 +150,8 @@ public Map getSources(String componentId) { public Map> getTargets(String componentId) { Map> ret = new HashMap<>(); for (String otherComponentId : getComponentIds()) { - Map inputs = getComponentCommon(otherComponentId).get_inputs(); + Map inputs = getComponentCommon(otherComponentId) + .get_inputs(); for (Map.Entry entry : inputs.entrySet()) { GlobalStreamId id = entry.getKey(); if (id.get_componentId().equals(componentId)) { diff --git a/storm-client/src/jvm/org/apache/storm/task/IBolt.java b/storm-client/src/jvm/org/apache/storm/task/IBolt.java index ad47ea5a1c4..c64164b193c 100644 --- a/storm-client/src/jvm/org/apache/storm/task/IBolt.java +++ b/storm-client/src/jvm/org/apache/storm/task/IBolt.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -18,58 +24,79 @@ import org.apache.storm.tuple.Tuple; /** - * An IBolt represents a component that takes tuples as input and produces tuples as output. An IBolt can do everything from filtering to - * joining to functions to aggregations. It does not have to process a tuple immediately and may hold onto tuples to process later. + * An IBolt represents a component that takes tuples as input and produces tuples as output. An + * IBolt can do everything from filtering to + * joining to functions to aggregations. It does not have to process a tuple immediately and may + * hold onto tuples to process later. * *

    A bolt's lifecycle is as follows: * - *

    IBolt object created on client machine. The IBolt is serialized into the topology (using Java serialization) and - * submitted to the master machine of the cluster (Nimbus). Nimbus then launches workers which deserialize the object, + *

    IBolt object created on client machine. The IBolt is serialized into the topology (using Java + * serialization) and + * submitted to the master machine of the cluster (Nimbus). Nimbus then launches workers which + * deserialize the object, * call prepare on it, and then start processing tuples. * - *

    If you want to parameterize an IBolt, you should set the parameters through its constructor and save the parameterization state as - * instance variables (which will then get serialized and shipped to every task executing this bolt across the cluster). + *

    If you want to parameterize an IBolt, you should set the parameters through its constructor + * and save the parameterization state as + * instance variables (which will then get serialized and shipped to every task executing this bolt + * across the cluster). * - *

    When defining bolts in Java, you should use the IRichBolt interface which adds necessary methods for using the + *

    When defining bolts in Java, you should use the IRichBolt interface which adds necessary + * methods for using the * Java TopologyBuilder API. */ public interface IBolt extends Serializable { /** - * Called when a task for this component is initialized within a worker on the cluster. It provides the bolt with the environment in + * Called when a task for this component is initialized within a worker on the cluster. It + * provides the bolt with the environment in * which the bolt executes. * *

    This includes the: * - * @param topoConf The Storm configuration for this bolt. This is the configuration provided to the topology merged in with cluster + * @param topoConf The Storm configuration for this bolt. This is the configuration provided to + * the topology merged in with cluster * configuration on this machine. - * @param context This object can be used to get information about this task's place within the topology, including the task id and + * @param context This object can be used to get information about this task's place within the + * topology, including the task id and * component id of this task, input and output information, etc. - * @param collector The collector is used to emit tuples from this bolt. Tuples can be emitted at any time, including the prepare and - * cleanup methods. The collector is thread-safe and should be saved as an instance variable of this bolt object. + * @param collector The collector is used to emit tuples from this bolt. Tuples can be emitted + * at any time, including the prepare and + * cleanup methods. The collector is thread-safe and should be saved as an instance variable + * of this bolt object. */ void prepare(Map topoConf, TopologyContext context, OutputCollector collector); /** - * Process a single tuple of input. The Tuple object contains metadata on it about which component/stream/task it came from. The values - * of the Tuple can be accessed using Tuple#getValue. The IBolt does not have to process the Tuple immediately. It is perfectly fine to + * Process a single tuple of input. The Tuple object contains metadata on it about which + * component/stream/task it came from. The values + * of the Tuple can be accessed using Tuple#getValue. The IBolt does not have to process the + * Tuple immediately. It is perfectly fine to * hang onto a tuple and process it later (for instance, to do an aggregation or join). * - *

    Tuples should be emitted using the OutputCollector provided through the prepare method. It is required that all input tuples are - * acked or failed at some point using the OutputCollector. Otherwise, Storm will be unable to determine when tuples coming off the + *

    Tuples should be emitted using the OutputCollector provided through the prepare method. It + * is required that all input tuples are + * acked or failed at some point using the OutputCollector. Otherwise, Storm will be unable to + * determine when tuples coming off the * spouts have been completed. * - *

    For the common case of acking an input tuple at the end of the execute method, see IBasicBolt which automates this. + *

    For the common case of acking an input tuple at the end of the execute method, see + * IBasicBolt which automates this. * * @param input The input tuple to be processed. */ void execute(Tuple input); /** - * Called when an IBolt is going to be shutdown. Storm will make a best-effort attempt to call this if the worker shutdown is orderly. - * The {@link Config#SUPERVISOR_WORKER_SHUTDOWN_SLEEP_SECS} setting controls how long orderly shutdown is allowed to take. - * There is no guarantee that cleanup will be called if shutdown is not orderly, or if the shutdown exceeds the time limit. + * Called when an IBolt is going to be shutdown. Storm will make a best-effort attempt to call + * this if the worker shutdown is orderly. + * The {@link Config#SUPERVISOR_WORKER_SHUTDOWN_SLEEP_SECS} setting controls how long orderly + * shutdown is allowed to take. + * There is no guarantee that cleanup will be called if shutdown is not orderly, or if the + * shutdown exceeds the time limit. * - *

    The one context where cleanup is guaranteed to be called is when a topology is killed when running Storm in local mode. + *

    The one context where cleanup is guaranteed to be called is when a topology is killed when + * running Storm in local mode. */ void cleanup(); } diff --git a/storm-client/src/jvm/org/apache/storm/task/IErrorReporter.java b/storm-client/src/jvm/org/apache/storm/task/IErrorReporter.java index 71b6a390ded..41b403beb3d 100644 --- a/storm-client/src/jvm/org/apache/storm/task/IErrorReporter.java +++ b/storm-client/src/jvm/org/apache/storm/task/IErrorReporter.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/task/IMetricsContext.java b/storm-client/src/jvm/org/apache/storm/task/IMetricsContext.java index 802ec5af6e7..0359a555988 100644 --- a/storm-client/src/jvm/org/apache/storm/task/IMetricsContext.java +++ b/storm-client/src/jvm/org/apache/storm/task/IMetricsContext.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -24,10 +30,10 @@ import org.apache.storm.metric.api.IReducer; import org.apache.storm.metric.api.ReducedMetric; - public interface IMetricsContext { /** * Register metric. + * * @deprecated in favor of metrics v2 (the non-deprecated methods on this class) */ @Deprecated @@ -35,6 +41,7 @@ public interface IMetricsContext { /** * Register metric. + * * @deprecated in favor of metrics v2 (the non-deprecated methods on this class) */ @Deprecated @@ -42,6 +49,7 @@ public interface IMetricsContext { /** * Register metric. + * * @deprecated in favor of metrics v2 (the non-deprecated methods on this class) */ @Deprecated diff --git a/storm-client/src/jvm/org/apache/storm/task/IOutputCollector.java b/storm-client/src/jvm/org/apache/storm/task/IOutputCollector.java index 02b80858ba7..7c01704eeef 100644 --- a/storm-client/src/jvm/org/apache/storm/task/IOutputCollector.java +++ b/storm-client/src/jvm/org/apache/storm/task/IOutputCollector.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/task/OutputCollector.java b/storm-client/src/jvm/org/apache/storm/task/OutputCollector.java index 57ab27988d6..eb19b818aa5 100644 --- a/storm-client/src/jvm/org/apache/storm/task/OutputCollector.java +++ b/storm-client/src/jvm/org/apache/storm/task/OutputCollector.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -19,19 +25,20 @@ import org.apache.storm.utils.Utils; /** - * This output collector exposes the API for emitting tuples from an IRichBolt. This is the core API for emitting tuples. For a simpler API, + * This output collector exposes the API for emitting tuples from an IRichBolt. This is the core API + * for emitting tuples. For a simpler API, * and a more restricted form of stream processing, see IBasicBolt and BasicOutputCollector. */ public class OutputCollector implements IOutputCollector { private IOutputCollector delegate; - public OutputCollector(IOutputCollector delegate) { this.delegate = delegate; } /** - * Emits a new tuple to a specific stream with a single anchor. The emitted values must be immutable. + * Emits a new tuple to a specific stream with a single anchor. The emitted values must be + * immutable. * * @param streamId the stream to emit to * @param anchor the tuple to anchor to @@ -43,8 +50,10 @@ public List emit(String streamId, Tuple anchor, List tuple) { } /** - * Emits a new unanchored tuple to the specified stream. Because it's unanchored, if a failure happens downstream, this new tuple won't - * affect whether any spout tuples are considered failed or not. The emitted values must be immutable. + * Emits a new unanchored tuple to the specified stream. Because it's unanchored, if a failure + * happens downstream, this new tuple won't + * affect whether any spout tuples are considered failed or not. The emitted values must be + * immutable. * * @param streamId the stream to emit to * @param tuple the new output tuple from this bolt @@ -55,7 +64,8 @@ public List emit(String streamId, List tuple) { } /** - * Emits a new tuple to the default stream anchored on a group of input tuples. The emitted values must be immutable. + * Emits a new tuple to the default stream anchored on a group of input tuples. The emitted + * values must be immutable. * * @param anchors the tuples to anchor to * @param tuple the new output tuple from this bolt @@ -65,9 +75,9 @@ public List emit(Collection anchors, List tuple) { return emit(Utils.DEFAULT_STREAM_ID, anchors, tuple); } - /** - * Emits a new tuple to the default stream anchored on a single tuple. The emitted values must be immutable. + * Emits a new tuple to the default stream anchored on a single tuple. The emitted values must + * be immutable. * * @param anchor the tuple to anchor to * @param tuple the new output tuple from this bolt @@ -78,8 +88,10 @@ public List emit(Tuple anchor, List tuple) { } /** - * Emits a new unanchored tuple to the default stream. Beacuse it's unanchored, if a failure happens downstream, this new tuple won't - * affect whether any spout tuples are considered failed or not. The emitted values must be immutable. + * Emits a new unanchored tuple to the default stream. Beacuse it's unanchored, if a failure + * happens downstream, this new tuple won't + * affect whether any spout tuples are considered failed or not. The emitted values must be + * immutable. * * @param tuple the new output tuple from this bolt * @return the list of task ids that this new tuple was sent to @@ -94,12 +106,16 @@ public List emit(String streamId, Collection anchors, List t } /** - * Emits a tuple directly to the specified task id on the specified stream. If the target bolt does not subscribe to this bolt using a - * direct grouping, the tuple will not be sent. If the specified output stream is not declared as direct, or the target bolt subscribes - * with a non-direct grouping, an error will occur at runtime. Note that this method does not use anchors, so downstream failures won't + * Emits a tuple directly to the specified task id on the specified stream. If the target bolt + * does not subscribe to this bolt using a + * direct grouping, the tuple will not be sent. If the specified output stream is not declared + * as direct, or the target bolt subscribes + * with a non-direct grouping, an error will occur at runtime. Note that this method does not + * use anchors, so downstream failures won't * affect the failure status of any spout tuples. The emitted values must be immutable. * * @param taskId the taskId to send the new tuple to - * @param streamId the stream to send the tuple on. It must be declared as a direct stream in the topology definition. + * @param streamId the stream to send the tuple on. It must be declared as a direct stream in + * the topology definition. * @param tuple the new output tuple from this bolt */ public void emitDirect(int taskId, String streamId, List tuple) { @@ -122,11 +142,15 @@ public void emitDirect(int taskId, String streamId, List tuple) { } /** - * Emits a tuple directly to the specified task id on the default stream. If the target bolt does not subscribe to this bolt using a - * direct grouping, the tuple will not be sent. If the specified output stream is not declared as direct, or the target bolt subscribes - * with a non-direct grouping, an error will occur at runtime. The emitted values must be immutable. + * Emits a tuple directly to the specified task id on the default stream. If the target bolt + * does not subscribe to this bolt using a + * direct grouping, the tuple will not be sent. If the specified output stream is not declared + * as direct, or the target bolt subscribes + * with a non-direct grouping, an error will occur at runtime. The emitted values must be + * immutable. * - *

    The default stream must be declared as direct in the topology definition. See OutputDeclarer#declare for how this is done when + *

    The default stream must be declared as direct in the topology definition. See + * OutputDeclarer#declare for how this is done when * defining topologies in Java. * * @param taskId the taskId to send the new tuple to @@ -138,11 +162,15 @@ public void emitDirect(int taskId, Collection anchors, List tuple } /** - * Emits a tuple directly to the specified task id on the default stream. If the target bolt does not subscribe to this bolt using a - * direct grouping, the tuple will not be sent. If the specified output stream is not declared as direct, or the target bolt subscribes - * with a non-direct grouping, an error will occur at runtime. The emitted values must be immutable. + * Emits a tuple directly to the specified task id on the default stream. If the target bolt + * does not subscribe to this bolt using a + * direct grouping, the tuple will not be sent. If the specified output stream is not declared + * as direct, or the target bolt subscribes + * with a non-direct grouping, an error will occur at runtime. The emitted values must be + * immutable. * - *

    The default stream must be declared as direct in the topology definition. See OutputDeclarer#declare for how this is done when + *

    The default stream must be declared as direct in the topology definition. See + * OutputDeclarer#declare for how this is done when * defining topologies in Java. * * @param taskId the taskId to send the new tuple to @@ -153,16 +181,20 @@ public void emitDirect(int taskId, Tuple anchor, List tuple) { emitDirect(taskId, Utils.DEFAULT_STREAM_ID, anchor, tuple); } - /** - * Emits a tuple directly to the specified task id on the default stream. If the target bolt does not subscribe to this bolt using a - * direct grouping, the tuple will not be sent. If the specified output stream is not declared as direct, or the target bolt subscribes - * with a non-direct grouping, an error will occur at runtime. The emitted values must be immutable. + * Emits a tuple directly to the specified task id on the default stream. If the target bolt + * does not subscribe to this bolt using a + * direct grouping, the tuple will not be sent. If the specified output stream is not declared + * as direct, or the target bolt subscribes + * with a non-direct grouping, an error will occur at runtime. The emitted values must be + * immutable. * - *

    The default stream must be declared as direct in the topology definition. See OutputDeclarer#declare for how this is done when + *

    The default stream must be declared as direct in the topology definition. See + * OutputDeclarer#declare for how this is done when * defining topologies in Java.< * - *

    Note that this method does not use anchors, so downstream failures won't affect the failure status of any spout tuples. + *

    Note that this method does not use anchors, so downstream failures won't affect the + * failure status of any spout tuples. * * @param taskId the taskId to send the new tuple to * @param tuple the new output tuple from this bolt @@ -172,7 +204,8 @@ public void emitDirect(int taskId, List tuple) { } @Override - public void emitDirect(int taskId, String streamId, Collection anchors, List tuple) { + public void emitDirect(int taskId, String streamId, Collection anchors, + List tuple) { delegate.emitDirect(taskId, streamId, anchors, tuple); } @@ -187,8 +220,10 @@ public void fail(Tuple input) { } /** - * Resets the message timeout for any tuple trees to which the given tuple belongs. The timeout is reset to - * Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS. Note that this is an expensive operation, and should be used sparingly. + * Resets the message timeout for any tuple trees to which the given tuple belongs. The timeout + * is reset to + * Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS. Note that this is an expensive operation, and should be + * used sparingly. * * @param input the tuple to reset timeout for */ diff --git a/storm-client/src/jvm/org/apache/storm/task/ShellBolt.java b/storm-client/src/jvm/org/apache/storm/task/ShellBolt.java index 6800e625acd..53eb3b88365 100644 --- a/storm-client/src/jvm/org/apache/storm/task/ShellBolt.java +++ b/storm-client/src/jvm/org/apache/storm/task/ShellBolt.java @@ -52,20 +52,27 @@ import org.slf4j.LoggerFactory; /** - * A bolt that shells out to another process to process tuples. ShellBolt communicates with that process over stdio using a special - * protocol. An ~100 line library is required to implement that protocol, and adapter libraries currently exist for Ruby and Python. + * A bolt that shells out to another process to process tuples. ShellBolt communicates with that + * process over stdio using a special + * protocol. An ~100 line library is required to implement that protocol, and adapter libraries + * currently exist for Ruby and Python. * - *

    To run a ShellBolt on a cluster, the scripts that are shelled out to must be in the resources directory within the - * jar submitted to the master. During development/testing on a local machine, that resources directory just needs to be + *

    To run a ShellBolt on a cluster, the scripts that are shelled out to must be in the resources + * directory within the + * jar submitted to the master. During development/testing on a local machine, that resources + * directory just needs to be * on the classpath. * - *

    When creating topologies using the Java API, subclass this bolt and implement the IRichBolt interface to create components for the + *

    When creating topologies using the Java API, subclass this bolt and implement the IRichBolt + * interface to create components for the * topology that use other languages. For example: * * - *

    ```java public class MyBolt extends ShellBolt implements IRichBolt { public MyBolt() { super("python3", "mybolt.py"); } + *

    ```java public class MyBolt extends ShellBolt implements IRichBolt { public MyBolt() { + * super("python3", "mybolt.py"); } * - *

    public void declareOutputFields(OutputFieldsDeclarer declarer) { declarer.declare(new Fields("field1", "field2")); } } ``` + *

    public void declareOutputFields(OutputFieldsDeclarer declarer) { declarer.declare(new + * Fields("field1", "field2")); } } ``` */ public class ShellBolt implements IBolt { public static final String HEARTBEAT_STREAM_ID = "__heartbeat"; @@ -115,10 +122,12 @@ public boolean shouldChangeChildCWD() { } /** - * Set if the current working directory of the child process should change to the resources dir from extracted from the jar, or if it + * Set if the current working directory of the child process should change to the resources dir + * from extracted from the jar, or if it * should stay the same as the worker process to access things from the blob store. * - * @param changeDirectory true change the directory (default) false leave the directory the same as the worker process. + * @param changeDirectory true change the directory (default) false leave the directory the same + * as the worker process. */ @SuppressWarnings("checkstyle:AbbreviationAsWordInName") public void changeChildCWD(boolean changeDirectory) { @@ -142,9 +151,11 @@ public void prepare(Map topoConf, TopologyContext context, this.context = context; if (topoConf.containsKey(Config.TOPOLOGY_SUBPROCESS_TIMEOUT_SECS)) { - workerTimeoutMills = 1000 * ObjectReader.getInt(topoConf.get(Config.TOPOLOGY_SUBPROCESS_TIMEOUT_SECS)); + workerTimeoutMills = 1000 * ObjectReader.getInt(topoConf + .get(Config.TOPOLOGY_SUBPROCESS_TIMEOUT_SECS)); } else { - workerTimeoutMills = 1000 * ObjectReader.getInt(topoConf.get(Config.SUPERVISOR_WORKER_TIMEOUT_SECS)); + workerTimeoutMills = 1000 * ObjectReader.getInt(topoConf + .get(Config.SUPERVISOR_WORKER_TIMEOUT_SECS)); } process = new ShellProcess(command); @@ -152,7 +163,7 @@ public void prepare(Map topoConf, TopologyContext context, process.setEnv(env); } - //subprocesses must send their pid first thing + // subprocesses must send their pid first thing Number subpid = process.launch(topoConf, context, changeDirectory); LOG.info("Launched subprocess with pid " + subpid); @@ -169,8 +180,10 @@ public void prepare(Map topoConf, TopologyContext context, LOG.info("Start checking heartbeat..."); setHeartbeat(); - heartBeatExecutorService = MoreExecutors.getExitingScheduledExecutorService(new ScheduledThreadPoolExecutor(1)); - heartBeatExecutorService.scheduleAtFixedRate(new BoltHeartbeatTimerTask(this), 1, 1, TimeUnit.SECONDS); + heartBeatExecutorService = MoreExecutors + .getExitingScheduledExecutorService(new ScheduledThreadPoolExecutor(1)); + heartBeatExecutorService.scheduleAtFixedRate(new BoltHeartbeatTimerTask(this), 1, 1, + TimeUnit.SECONDS); } @Override @@ -179,7 +192,7 @@ public void execute(Tuple input) { throw new RuntimeException(exception); } - //just need an id + // just need an id String genId = Long.toString(rand.nextLong()); inputs.put(genId, input); try { @@ -246,7 +259,8 @@ private void handleEmit(ShellMsg shellMsg) throws InterruptedException { } if (shellMsg.getTask() == 0) { - List outtasks = collector.emit(shellMsg.getStream(), anchors, shellMsg.getTuple()); + List outtasks = collector.emit(shellMsg.getStream(), anchors, shellMsg + .getTuple()); if (shellMsg.areTaskIdsNeeded()) { pendingWrites.putTaskIds(outtasks); } @@ -257,23 +271,24 @@ private void handleEmit(ShellMsg shellMsg) throws InterruptedException { } private void handleMetrics(ShellMsg shellMsg) { - //get metric name + // get metric name String name = shellMsg.getMetricName(); if (name.isEmpty()) { throw new RuntimeException("Receive Metrics name is empty"); } - //get metric by name + // get metric by name IMetric metric = context.getRegisteredMetricByName(name); if (metric == null) { throw new RuntimeException("Could not find metric by name[" + name + "] "); } if (!(metric instanceof IShellMetric)) { - throw new RuntimeException("Metric[" + name + "] is not IShellMetric, can not call by RPC"); + throw new RuntimeException("Metric[" + name + + "] is not IShellMetric, can not call by RPC"); } IShellMetric shellMetric = (IShellMetric) metric; - //call updateMetricFromRPC with params + // call updateMetricFromRPC with params Object paramsObj = shellMsg.getMetricParams(); try { shellMetric.updateMetricFromRPC(paramsObj); @@ -293,14 +308,17 @@ private long getLastHeartbeat() { } private void die(Throwable exception) { - String processInfo = process.getProcessInfoString() + process.getProcessTerminationInfoString(); + String processInfo = process.getProcessInfoString() + process + .getProcessTerminationInfoString(); this.exception = new RuntimeException(processInfo, exception); - String message = String.format("Halting process: ShellBolt died. Command: %s, ProcessInfo %s", + String message = String + .format("Halting process: ShellBolt died. Command: %s, ProcessInfo %s", Arrays.toString(command), processInfo); LOG.error(message, exception); collector.reportError(exception); - if (!isLocalMode && (running || (exception instanceof Error))) { //don't exit if not running, unless it is an Error + if (!isLocalMode && (running + || (exception instanceof Error))) { // don't exit if not running, unless it is an Error System.exit(11); } } @@ -337,7 +355,8 @@ public void run() { String command = shellMsg.getCommand(); if (command == null) { - throw new IllegalArgumentException("Command not found in bolt message: " + shellMsg); + throw new IllegalArgumentException("Command not found in bolt message: " + + shellMsg); } setHeartbeat(); diff --git a/storm-client/src/jvm/org/apache/storm/task/TopologyContext.java b/storm-client/src/jvm/org/apache/storm/task/TopologyContext.java index ab028bbef85..fac0f803c72 100644 --- a/storm-client/src/jvm/org/apache/storm/task/TopologyContext.java +++ b/storm-client/src/jvm/org/apache/storm/task/TopologyContext.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -42,10 +48,13 @@ import org.apache.storm.utils.Utils; /** - * A `TopologyContext` is given to bolts and spouts in their `prepare()` and `open()` methods, respectively. This object provides - * information about the component's place within the topology, such as task ids, inputs and outputs, etc. + * A `TopologyContext` is given to bolts and spouts in their `prepare()` and `open()` methods, + * respectively. This object provides + * information about the component's place within the topology, such as task ids, inputs and + * outputs, etc. * - *

    The `TopologyContext` is also used to declare `ISubscribedState` objects to synchronize state with StateSpouts + *

    The `TopologyContext` is also used to declare `ISubscribedState` objects to synchronize state + * with StateSpouts * this object is subscribed to. */ public class TopologyContext extends WorkerTopologyContext implements IMetricsContext { @@ -101,7 +110,8 @@ private static Map groupingToJSONableMap(Grouping grouping) { /** * All state from all subscribed state spouts streams will be synced with the provided object. * - *

    It is recommended that your ISubscribedState object is kept as an instance variable of this object. The recommended usage of this + *

    It is recommended that your ISubscribedState object is kept as an instance variable of + * this object. The recommended usage of this * method is as follows: * *

    ```java _myState = context.setAllSubscribedState(new MyState()); ``` @@ -110,13 +120,14 @@ private static Map groupingToJSONableMap(Grouping grouping) { * @return Returns the ISubscribedState object provided */ public T setAllSubscribedState(T obj) { - //check that only subscribed to one component/stream for statespout - //setsubscribedstate appropriately + // check that only subscribed to one component/stream for statespout + // setsubscribedstate appropriately throw new NotImplementedException(); } /** - * Synchronizes the default stream from the specified state spout component id with the provided ISubscribedState object. + * Synchronizes the default stream from the specified state spout component id with the provided + * ISubscribedState object. * *

    The recommended usage of this method is as follows: * @@ -131,7 +142,8 @@ public T setSubscribedState(String componentId, T o } /** - * Synchronizes the specified stream from the specified state spout component id with the provided ISubscribedState object. + * Synchronizes the specified stream from the specified state spout component id with the + * provided ISubscribedState object. * *

    The recommended usage of this method is as follows: * @@ -142,7 +154,8 @@ public T setSubscribedState(String componentId, T o * @param obj Provided ISubscribedState implementation * @return Returns the ISubscribedState object provided */ - public T setSubscribedState(String componentId, String streamId, T obj) { + public T setSubscribedState(String componentId, String streamId, + T obj) { throw new NotImplementedException(); } @@ -161,7 +174,9 @@ public int getThisTaskId() { /** * Get component id. - * @return the component id for this task. The component id maps to a component id specified for a Spout or Bolt in the topology + * + * @return the component id for this task. The component id maps to a component id specified for + * a Spout or Bolt in the topology * definition. */ public String getThisComponentId() { @@ -169,7 +184,8 @@ public String getThisComponentId() { } /** - * Gets the declared output fields for the specified stream id for the component this task is a part of. + * Gets the declared output fields for the specified stream id for the component this task is a + * part of. */ public Fields getThisOutputFields(String streamId) { return getComponentOutputFields(getThisComponentId(), streamId); @@ -194,7 +210,8 @@ public Set getThisStreams() { } /** - * Gets the index of this task id in getComponentTasks(getThisComponentId()). An example use case for this method is determining which + * Gets the index of this task id in getComponentTasks(getThisComponentId()). An example use + * case for this method is determining which * task accesses which resource in a distributed resource to ensure an even distribution. */ public int getThisTaskIndex() { @@ -249,7 +266,8 @@ public Map> getThisTargets() { } /** - * Sets the task-level data for the given name. This data is shared amongst the task and its corresponding task hooks. + * Sets the task-level data for the given name. This data is shared amongst the task and its + * corresponding task hooks. * * @param name name of the task-level data to be set * @param data task-level data @@ -259,7 +277,8 @@ public void setTaskData(String name, Object data) { } /** - * Fetches the task-level data for the given name. This data is shared amongst the task and its corresponding task hooks. + * Fetches the task-level data for the given name. This data is shared amongst the task and its + * corresponding task hooks. * * @param name name of the task-level data to be fetched * @return Associated task-level data @@ -269,7 +288,8 @@ public Object getTaskData(String name) { } /** - * Sets the executor-level data for the given name. This data is shared amongst tasks and corresponding task hooks managed by the + * Sets the executor-level data for the given name. This data is shared amongst tasks and + * corresponding task hooks managed by the * given executor. * * @param name name of the executor-level data to be set @@ -280,7 +300,8 @@ public void setExecutorData(String name, Object data) { } /** - * Fetches the executor-level data for the given name. This data is shared across tasks and corresponding task hook managed by the + * Fetches the executor-level data for the given name. This data is shared across tasks and + * corresponding task hook managed by the * given executor. * * @param name name of the executor-level data to be fetched @@ -314,7 +335,8 @@ public String toJSONString() { for (Map.Entry> entry : this.getThisTargets().entrySet()) { Map stringTargetMap = new HashMap<>(); for (Map.Entry innerEntry : entry.getValue().entrySet()) { - stringTargetMap.put(innerEntry.getKey(), groupingToJSONableMap(innerEntry.getValue())); + stringTargetMap.put(innerEntry.getKey(), groupingToJSONableMap(innerEntry + .getValue())); } stringTargets.put(entry.getKey(), stringTargetMap); } @@ -348,7 +370,8 @@ public String toJSONString() { @Override public T registerMetric(String name, T metric, int timeBucketSizeInSecs) { if (openOrPrepareWasCalled.get()) { - throw new RuntimeException("TopologyContext.registerMetric can only be called from within overridden " + throw new RuntimeException("TopologyContext.registerMetric can only be called from " + + "within overridden " + "IBolt::prepare() or ISpout::open() method."); } @@ -357,7 +380,8 @@ public T registerMetric(String name, T metric, int timeBucke } if (timeBucketSizeInSecs <= 0) { - throw new IllegalArgumentException("TopologyContext.registerMetric can only be called with " + throw new IllegalArgumentException("TopologyContext.registerMetric can only be called " + + "with " + "timeBucketSizeInSecs greater than or equal to 1 second."); } @@ -399,12 +423,14 @@ public ReducedMetric registerMetric(String name, IReducer reducer, int timeBucke */ @Deprecated @Override - public CombinedMetric registerMetric(String name, ICombiner combiner, int timeBucketSizeInSecs) { + public CombinedMetric registerMetric(String name, ICombiner combiner, + int timeBucketSizeInSecs) { return registerMetric(name, new CombinedMetric(combiner), timeBucketSizeInSecs); } /** - * Get component's metric from registered metrics by name. Notice: Normally, one component can only register one metric name once. But + * Get component's metric from registered metrics by name. Notice: Normally, one component can + * only register one metric name once. But * now registerMetric has a bug(https://issues.apache.org/jira/browse/STORM-254) cause the same metric name can register twice. So we * just return the first metric we meet. */ @@ -417,7 +443,7 @@ public IMetric getRegisteredMetricByName(String name) { if (nameToMetric != null) { metric = nameToMetric.get(name); if (metric != null) { - //we just return the first metric we meet + // we just return the first metric we meet break; } } diff --git a/storm-client/src/jvm/org/apache/storm/task/WorkerTopologyContext.java b/storm-client/src/jvm/org/apache/storm/task/WorkerTopologyContext.java index 6b0312df431..064aec9fef6 100644 --- a/storm-client/src/jvm/org/apache/storm/task/WorkerTopologyContext.java +++ b/storm-client/src/jvm/org/apache/storm/task/WorkerTopologyContext.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -52,7 +58,8 @@ public WorkerTopologyContext( String assignmentId, AtomicReference> nodeToHost ) { - super(topology, topoConf, taskToComponent, componentToSortedTasks, componentToStreamToFields, stormId); + super(topology, topoConf, taskToComponent, componentToSortedTasks, + componentToStreamToFields, stormId); this.codeDir = codeDir; this.defaultResources = defaultResources; this.userResources = userResources; @@ -86,12 +93,14 @@ public WorkerTopologyContext( List workerTasks, Map defaultResources, Map userResources) { - this(topology, topoConf, taskToComponent, componentToSortedTasks, componentToStreamToFields, stormId, + this(topology, topoConf, taskToComponent, componentToSortedTasks, componentToStreamToFields, + stormId, codeDir, pidDir, workerPort, workerTasks, defaultResources, userResources, null, null, null); } /** - * Gets all the task ids that are running in this worker process (including the task for this task). + * Gets all the task ids that are running in this worker process (including the task for this + * task). */ public List getThisWorkerTasks() { return workerTasks; @@ -116,6 +125,7 @@ public AtomicReference> getTaskToNodePort() { /** * Get a map from nodeId to hostname. + * * @return a map from nodeId to hostname */ public AtomicReference> getNodeToHost() { @@ -123,7 +133,8 @@ public AtomicReference> getNodeToHost() { } /** - * Gets the location of the external resources for this worker on the local filesystem. These external resources typically include bolts + * Gets the location of the external resources for this worker on the local filesystem. These + * external resources typically include bolts * implemented in other languages, such as Ruby or Python. */ public String getCodeDir() { @@ -131,7 +142,8 @@ public String getCodeDir() { } /** - * If this task spawns any subprocesses, those subprocesses must immediately write their PID to this directory on the local filesystem + * If this task spawns any subprocesses, those subprocesses must immediately write their PID to + * this directory on the local filesystem * to ensure that Storm properly destroys that process when the worker is shutdown. */ @SuppressWarnings("checkstyle:AbbreviationAsWordInName") @@ -140,8 +152,10 @@ public String getPIDDir() { } /** - * Fetches the worker-level data for the given name. The corresponding data needs to be first set in an implementation of - * {@link IWorkerHook#start(Map, WorkerUserContext)} via {@link WorkerUserContext#setResource(String, Object)} + * Fetches the worker-level data for the given name. The corresponding data needs to be first + * set in an implementation of + * {@link IWorkerHook#start(Map, WorkerUserContext)} via {@link + * WorkerUserContext#setResource(String, Object)} * * @param name name of the worker-level data to be fetched * @return Associated worker-level data diff --git a/storm-client/src/jvm/org/apache/storm/task/WorkerUserContext.java b/storm-client/src/jvm/org/apache/storm/task/WorkerUserContext.java index d69b6be6e84..688513788aa 100644 --- a/storm-client/src/jvm/org/apache/storm/task/WorkerUserContext.java +++ b/storm-client/src/jvm/org/apache/storm/task/WorkerUserContext.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -37,12 +43,14 @@ public WorkerUserContext( AtomicReference> taskToNodePort, String assignmentId, AtomicReference> nodeToHost) { - super(topology, topoConf, taskToComponent, componentToSortedTasks, componentToStreamToFields, stormId, codeDir, pidDir, workerPort, + super(topology, topoConf, taskToComponent, componentToSortedTasks, + componentToStreamToFields, stormId, codeDir, pidDir, workerPort, workerTasks, defaultResources, userResources, taskToNodePort, assignmentId, nodeToHost); } /** - * Sets the worker-level data for the given name. This data can then be read by all components running on the same worker, + * Sets the worker-level data for the given name. This data can then be read by all components + * running on the same worker, * i.e. tasks (spouts, bolts), task hooks and worker hooks. * * @param name name of the worker-level data to be set diff --git a/storm-client/src/jvm/org/apache/storm/testing/AckFailDelegate.java b/storm-client/src/jvm/org/apache/storm/testing/AckFailDelegate.java index 19bf202299b..c64dc193278 100644 --- a/storm-client/src/jvm/org/apache/storm/testing/AckFailDelegate.java +++ b/storm-client/src/jvm/org/apache/storm/testing/AckFailDelegate.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/testing/AckFailMapTracker.java b/storm-client/src/jvm/org/apache/storm/testing/AckFailMapTracker.java index 2bdff61e9b9..98acf1950e2 100644 --- a/storm-client/src/jvm/org/apache/storm/testing/AckFailMapTracker.java +++ b/storm-client/src/jvm/org/apache/storm/testing/AckFailMapTracker.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/testing/AckTracker.java b/storm-client/src/jvm/org/apache/storm/testing/AckTracker.java index e62dee27151..fd1edd8050c 100644 --- a/storm-client/src/jvm/org/apache/storm/testing/AckTracker.java +++ b/storm-client/src/jvm/org/apache/storm/testing/AckTracker.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/testing/AlternateRackDNSToSwitchMapping.java b/storm-client/src/jvm/org/apache/storm/testing/AlternateRackDNSToSwitchMapping.java index 411c382345b..4f9e0f09185 100644 --- a/storm-client/src/jvm/org/apache/storm/testing/AlternateRackDNSToSwitchMapping.java +++ b/storm-client/src/jvm/org/apache/storm/testing/AlternateRackDNSToSwitchMapping.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -21,7 +27,8 @@ import org.apache.storm.networktopography.DNSToSwitchMapping; /** - * This class implements the {@link DNSToSwitchMapping} interface It alternates bewteen RACK1 and RACK2 for the hosts. + * This class implements the {@link DNSToSwitchMapping} interface It alternates bewteen RACK1 and + * RACK2 for the hosts. */ @SuppressWarnings("checkstyle:AbbreviationAsWordInName") public final class AlternateRackDNSToSwitchMapping extends AbstractDNSToSwitchMapping { @@ -33,7 +40,7 @@ public Map resolve(List names) { TreeSet sortedNames = new TreeSet(names); Map m = new HashMap(); if (names.isEmpty()) { - //name list is empty, return an empty map + // name list is empty, return an empty map return m; } diff --git a/storm-client/src/jvm/org/apache/storm/testing/BoltTracker.java b/storm-client/src/jvm/org/apache/storm/testing/BoltTracker.java index d6ff164b293..0dec75c1277 100644 --- a/storm-client/src/jvm/org/apache/storm/testing/BoltTracker.java +++ b/storm-client/src/jvm/org/apache/storm/testing/BoltTracker.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -17,7 +23,6 @@ import org.apache.storm.topology.IRichBolt; import org.apache.storm.topology.OutputFieldsDeclarer; - public class BoltTracker extends NonRichBoltTracker implements IRichBolt { IRichBolt richDelegate; diff --git a/storm-client/src/jvm/org/apache/storm/testing/CompletableSpout.java b/storm-client/src/jvm/org/apache/storm/testing/CompletableSpout.java index 57d24bced83..f57ebe39586 100644 --- a/storm-client/src/jvm/org/apache/storm/testing/CompletableSpout.java +++ b/storm-client/src/jvm/org/apache/storm/testing/CompletableSpout.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -15,6 +21,7 @@ public interface CompletableSpout { /** * Check whether spout is exhausted. + * * @return true if all the tuples have been completed else false. */ boolean isExhausted(); @@ -23,13 +30,13 @@ public interface CompletableSpout { * Cleanup any global state kept. */ default void clean() { - //NOOP + // NOOP } /** * Prepare the spout (globally) before starting the topology. */ default void startup() { - //NOOP + // NOOP } } diff --git a/storm-client/src/jvm/org/apache/storm/testing/FeederSpout.java b/storm-client/src/jvm/org/apache/storm/testing/FeederSpout.java index 32a0b2526ed..41d82dceb74 100644 --- a/storm-client/src/jvm/org/apache/storm/testing/FeederSpout.java +++ b/storm-client/src/jvm/org/apache/storm/testing/FeederSpout.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -24,7 +30,6 @@ import org.apache.storm.tuple.Values; import org.apache.storm.utils.InprocMessaging; - public class FeederSpout extends BaseRichSpout { private int id; private Fields outFields; @@ -61,7 +66,8 @@ public void waitForReader() { } @Override - public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { + public void open(Map conf, TopologyContext context, + SpoutOutputCollector collector) { this.collector = collector; } diff --git a/storm-client/src/jvm/org/apache/storm/testing/FixedTuple.java b/storm-client/src/jvm/org/apache/storm/testing/FixedTuple.java index b512594ebb7..3c339fb128e 100644 --- a/storm-client/src/jvm/org/apache/storm/testing/FixedTuple.java +++ b/storm-client/src/jvm/org/apache/storm/testing/FixedTuple.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/testing/FixedTupleSpout.java b/storm-client/src/jvm/org/apache/storm/testing/FixedTupleSpout.java index c58dd932ec6..d88078c69cb 100644 --- a/storm-client/src/jvm/org/apache/storm/testing/FixedTupleSpout.java +++ b/storm-client/src/jvm/org/apache/storm/testing/FixedTupleSpout.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -105,7 +111,8 @@ public void cleanup() { } @Override - public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { + public void open(Map conf, TopologyContext context, + SpoutOutputCollector collector) { this.context = context; List tasks = context.getComponentTasks(context.getThisComponentId()); int startIndex; diff --git a/storm-client/src/jvm/org/apache/storm/testing/ForwardingMetricsConsumer.java b/storm-client/src/jvm/org/apache/storm/testing/ForwardingMetricsConsumer.java index 7fc863ebe42..fe7067baa87 100644 --- a/storm-client/src/jvm/org/apache/storm/testing/ForwardingMetricsConsumer.java +++ b/storm-client/src/jvm/org/apache/storm/testing/ForwardingMetricsConsumer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -26,7 +32,8 @@ * To use, add this to your topology's configuration: * * ```java - * conf.registerMetricsConsumer(org.apache.storm.testing.ForwardingMetricsConsumer.class, ":", 1); + * conf.registerMetricsConsumer(org.apache.storm.testing.ForwardingMetricsConsumer.class, + * ":", 1); * ``` * * Or edit the storm.yaml config file: @@ -46,7 +53,8 @@ public class ForwardingMetricsConsumer implements IMetricsConsumer { OutputStream out; @Override - public void prepare(Map topoConf, Object registrationArgument, TopologyContext context, IErrorReporter errorReporter) { + public void prepare(Map topoConf, Object registrationArgument, + TopologyContext context, IErrorReporter errorReporter) { String[] parts = ((String) registrationArgument).split(":", 2); host = parts[0]; port = Integer.valueOf(parts[1]); diff --git a/storm-client/src/jvm/org/apache/storm/testing/IdentityBolt.java b/storm-client/src/jvm/org/apache/storm/testing/IdentityBolt.java index 93c5e95494b..67e86bebbc7 100644 --- a/storm-client/src/jvm/org/apache/storm/testing/IdentityBolt.java +++ b/storm-client/src/jvm/org/apache/storm/testing/IdentityBolt.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/testing/MkClusterParam.java b/storm-client/src/jvm/org/apache/storm/testing/MkClusterParam.java index 55612fd9ddf..9beb9b4a96c 100644 --- a/storm-client/src/jvm/org/apache/storm/testing/MkClusterParam.java +++ b/storm-client/src/jvm/org/apache/storm/testing/MkClusterParam.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -15,19 +21,20 @@ import java.util.Map; /** - * The param arg for `Testing.withSimulatedTimeCluster`, `Testing.withTrackedCluster` and `Testing.withLocalCluster`. + * The param arg for `Testing.withSimulatedTimeCluster`, `Testing.withTrackedCluster` and + * `Testing.withLocalCluster`. */ public class MkClusterParam { /** - * count of supervisors for the cluster. + * Count of supervisors for the cluster. */ private Integer supervisors; /** - * count of port for each supervisor. + * Count of port for each supervisor. */ private Integer portsPerSupervisor; /** - * cluster config. + * Cluster config. */ private Map daemonConf; @@ -62,7 +69,8 @@ public void setDaemonConf(Map daemonConf) { } /** - * When nimbusDaemon is true, the local cluster will be started with a Nimbus Thrift server, allowing communication through for example + * When nimbusDaemon is true, the local cluster will be started with a Nimbus Thrift server, + * allowing communication through for example * org.apache.storm.utils.NimbusClient. */ public void setNimbusDaemon(Boolean nimbusDaemon) { diff --git a/storm-client/src/jvm/org/apache/storm/testing/MkTupleParam.java b/storm-client/src/jvm/org/apache/storm/testing/MkTupleParam.java index 26c208e90fd..f50a40fb5d5 100644 --- a/storm-client/src/jvm/org/apache/storm/testing/MkTupleParam.java +++ b/storm-client/src/jvm/org/apache/storm/testing/MkTupleParam.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/testing/MockedSources.java b/storm-client/src/jvm/org/apache/storm/testing/MockedSources.java index fedfb661165..2661bd1a959 100644 --- a/storm-client/src/jvm/org/apache/storm/testing/MockedSources.java +++ b/storm-client/src/jvm/org/apache/storm/testing/MockedSources.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -21,12 +27,12 @@ public class MockedSources { /** - * mocked spout sources for the [spout, stream] pair. + * Mocked spout sources for the [spout, stream] pair. */ private Map> data = new HashMap>(); public MockedSources() { - //Empty + // Empty } public MockedSources(Map> data) { @@ -34,7 +40,7 @@ public MockedSources(Map> data) { } /** - * add mock data for the spout. + * Add mock data for the spout. * * @param spoutId the spout to be mocked * @param streamId the stream of the spout to be mocked diff --git a/storm-client/src/jvm/org/apache/storm/testing/NGrouping.java b/storm-client/src/jvm/org/apache/storm/testing/NGrouping.java index 3135b326d32..7d6acf02174 100644 --- a/storm-client/src/jvm/org/apache/storm/testing/NGrouping.java +++ b/storm-client/src/jvm/org/apache/storm/testing/NGrouping.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -28,7 +34,8 @@ public NGrouping(Integer n) { } @Override - public void prepare(WorkerTopologyContext context, GlobalStreamId stream, List targetTasks) { + public void prepare(WorkerTopologyContext context, GlobalStreamId stream, + List targetTasks) { targetTasks = new ArrayList(targetTasks); Collections.sort(targetTasks); outTasks = new ArrayList(); diff --git a/storm-client/src/jvm/org/apache/storm/testing/NonRichBoltTracker.java b/storm-client/src/jvm/org/apache/storm/testing/NonRichBoltTracker.java index 815ba71fdc3..9266fb2e4f3 100644 --- a/storm-client/src/jvm/org/apache/storm/testing/NonRichBoltTracker.java +++ b/storm-client/src/jvm/org/apache/storm/testing/NonRichBoltTracker.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -20,7 +26,6 @@ import org.apache.storm.tuple.Tuple; import org.apache.storm.utils.RegisteredGlobalState; - public class NonRichBoltTracker implements IBolt { IBolt delegate; String trackId; @@ -31,7 +36,8 @@ public NonRichBoltTracker(IBolt delegate, String id) { } @Override - public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) { + public void prepare(Map topoConf, TopologyContext context, + OutputCollector collector) { delegate.prepare(topoConf, context, collector); } diff --git a/storm-client/src/jvm/org/apache/storm/testing/PrepareBatchBolt.java b/storm-client/src/jvm/org/apache/storm/testing/PrepareBatchBolt.java index 35c1b091473..7dd6bd868a1 100644 --- a/storm-client/src/jvm/org/apache/storm/testing/PrepareBatchBolt.java +++ b/storm-client/src/jvm/org/apache/storm/testing/PrepareBatchBolt.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -21,7 +27,6 @@ import org.apache.storm.tuple.Tuple; import org.apache.storm.utils.Utils; - public class PrepareBatchBolt extends BaseBasicBolt { Fields outFields; diff --git a/storm-client/src/jvm/org/apache/storm/testing/PythonShellMetricsBolt.java b/storm-client/src/jvm/org/apache/storm/testing/PythonShellMetricsBolt.java index bdd4edfcfcb..bbb58b26fc7 100644 --- a/storm-client/src/jvm/org/apache/storm/testing/PythonShellMetricsBolt.java +++ b/storm-client/src/jvm/org/apache/storm/testing/PythonShellMetricsBolt.java @@ -38,7 +38,8 @@ public PythonShellMetricsBolt(String command, String file) { } @Override - public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) { + public void prepare(Map topoConf, TopologyContext context, + OutputCollector collector) { super.prepare(topoConf, context, collector); CountShellMetric countShellMetric = new CountShellMetric(); diff --git a/storm-client/src/jvm/org/apache/storm/testing/PythonShellMetricsSpout.java b/storm-client/src/jvm/org/apache/storm/testing/PythonShellMetricsSpout.java index f7ecb51cce5..dd7296de499 100644 --- a/storm-client/src/jvm/org/apache/storm/testing/PythonShellMetricsSpout.java +++ b/storm-client/src/jvm/org/apache/storm/testing/PythonShellMetricsSpout.java @@ -39,7 +39,8 @@ public PythonShellMetricsSpout(String command, String file) { } @Override - public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { + public void open(Map conf, TopologyContext context, + SpoutOutputCollector collector) { super.open(conf, context, collector); CountShellMetric countShellMetric = new CountShellMetric(); diff --git a/storm-client/src/jvm/org/apache/storm/testing/SingleUserSimpleTransport.java b/storm-client/src/jvm/org/apache/storm/testing/SingleUserSimpleTransport.java index d385de53347..61463bedd32 100644 --- a/storm-client/src/jvm/org/apache/storm/testing/SingleUserSimpleTransport.java +++ b/storm-client/src/jvm/org/apache/storm/testing/SingleUserSimpleTransport.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -17,7 +23,6 @@ import javax.security.auth.Subject; import org.apache.storm.security.auth.SimpleTransportPlugin; - public class SingleUserSimpleTransport extends SimpleTransportPlugin { @Override protected Subject getDefaultSubject() { diff --git a/storm-client/src/jvm/org/apache/storm/testing/SpoutTracker.java b/storm-client/src/jvm/org/apache/storm/testing/SpoutTracker.java index 00ab28d50c1..2f918e7b5a8 100644 --- a/storm-client/src/jvm/org/apache/storm/testing/SpoutTracker.java +++ b/storm-client/src/jvm/org/apache/storm/testing/SpoutTracker.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -23,20 +29,19 @@ import org.apache.storm.topology.base.BaseRichSpout; import org.apache.storm.utils.RegisteredGlobalState; - public class SpoutTracker extends BaseRichSpout { IRichSpout delegate; SpoutTrackOutputCollector tracker; String trackId; - public SpoutTracker(IRichSpout delegate, String trackId) { this.delegate = delegate; this.trackId = trackId; } @Override - public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { + public void open(Map conf, TopologyContext context, + SpoutOutputCollector collector) { tracker = new SpoutTrackOutputCollector(collector); delegate.open(conf, context, new SpoutOutputCollector(tracker)); } @@ -80,7 +85,8 @@ private class SpoutTrackOutputCollector implements ISpoutOutputCollector { } private void recordSpoutEmit() { - Map stats = (Map) RegisteredGlobalState.getState(trackId); + Map stats = (Map) RegisteredGlobalState + .getState(trackId); ((AtomicInteger) stats.get("spout-emitted")).incrementAndGet(); } diff --git a/storm-client/src/jvm/org/apache/storm/testing/TestAggregatesCounter.java b/storm-client/src/jvm/org/apache/storm/testing/TestAggregatesCounter.java index 47fdae53f51..6ee16693c86 100644 --- a/storm-client/src/jvm/org/apache/storm/testing/TestAggregatesCounter.java +++ b/storm-client/src/jvm/org/apache/storm/testing/TestAggregatesCounter.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -25,7 +31,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; - public class TestAggregatesCounter extends BaseRichBolt { public static Logger LOG = LoggerFactory.getLogger(TestWordCounter.class); @@ -33,7 +38,8 @@ public class TestAggregatesCounter extends BaseRichBolt { OutputCollector collector; @Override - public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) { + public void prepare(Map topoConf, TopologyContext context, + OutputCollector collector) { this.collector = collector; counts = new HashMap(); } diff --git a/storm-client/src/jvm/org/apache/storm/testing/TestConfBolt.java b/storm-client/src/jvm/org/apache/storm/testing/TestConfBolt.java index 51cb45e106f..95317dec5f4 100644 --- a/storm-client/src/jvm/org/apache/storm/testing/TestConfBolt.java +++ b/storm-client/src/jvm/org/apache/storm/testing/TestConfBolt.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -21,7 +27,6 @@ import org.apache.storm.tuple.Tuple; import org.apache.storm.tuple.Values; - public class TestConfBolt extends BaseBasicBolt { Map componentConf; Map conf; diff --git a/storm-client/src/jvm/org/apache/storm/testing/TestEventLogSpout.java b/storm-client/src/jvm/org/apache/storm/testing/TestEventLogSpout.java index 4f4df3d4ffa..06f4495d99d 100644 --- a/storm-client/src/jvm/org/apache/storm/testing/TestEventLogSpout.java +++ b/storm-client/src/jvm/org/apache/storm/testing/TestEventLogSpout.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -63,7 +69,8 @@ public static int getNumFailed(String stormId) { } @Override - public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { + public void open(Map conf, TopologyContext context, + SpoutOutputCollector collector) { this.collector = collector; this.source = context.getThisTaskId(); long taskCount = context.getComponentTasks(context.getThisComponentId()).size(); diff --git a/storm-client/src/jvm/org/apache/storm/testing/TestEventOrderCheckBolt.java b/storm-client/src/jvm/org/apache/storm/testing/TestEventOrderCheckBolt.java index 3511e4e5995..3046b508530 100644 --- a/storm-client/src/jvm/org/apache/storm/testing/TestEventOrderCheckBolt.java +++ b/storm-client/src/jvm/org/apache/storm/testing/TestEventOrderCheckBolt.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -31,7 +37,8 @@ public class TestEventOrderCheckBolt extends BaseRichBolt { private int count; @Override - public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) { + public void prepare(Map topoConf, TopologyContext context, + OutputCollector collector) { this.collector = collector; count = 0; } @@ -44,7 +51,8 @@ public void execute(Tuple input) { if (null != recentEvent && eventId <= recentEvent) { String error = "Error: event id is not in strict order! event source Id: " - + sourceId + ", last event Id: " + recentEvent + ", current event Id: " + eventId; + + sourceId + ", last event Id: " + recentEvent + ", current event Id: " + + eventId; collector.emit(input, new Values(error)); } diff --git a/storm-client/src/jvm/org/apache/storm/testing/TestGlobalCount.java b/storm-client/src/jvm/org/apache/storm/testing/TestGlobalCount.java index 7acc4e16b75..aae6d0da952 100644 --- a/storm-client/src/jvm/org/apache/storm/testing/TestGlobalCount.java +++ b/storm-client/src/jvm/org/apache/storm/testing/TestGlobalCount.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -23,14 +29,14 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; - public class TestGlobalCount extends BaseRichBolt { public static Logger LOG = LoggerFactory.getLogger(TestWordCounter.class); OutputCollector collector; private int count; @Override - public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) { + public void prepare(Map topoConf, TopologyContext context, + OutputCollector collector) { this.collector = collector; count = 0; } diff --git a/storm-client/src/jvm/org/apache/storm/testing/TestKryoDecorator.java b/storm-client/src/jvm/org/apache/storm/testing/TestKryoDecorator.java index 7af7ebafaf9..56140ea5b4b 100644 --- a/storm-client/src/jvm/org/apache/storm/testing/TestKryoDecorator.java +++ b/storm-client/src/jvm/org/apache/storm/testing/TestKryoDecorator.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/testing/TestPlannerBolt.java b/storm-client/src/jvm/org/apache/storm/testing/TestPlannerBolt.java index d99d405f822..dab8d82e489 100644 --- a/storm-client/src/jvm/org/apache/storm/testing/TestPlannerBolt.java +++ b/storm-client/src/jvm/org/apache/storm/testing/TestPlannerBolt.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -20,10 +26,10 @@ import org.apache.storm.tuple.Fields; import org.apache.storm.tuple.Tuple; - public class TestPlannerBolt extends BaseRichBolt { @Override - public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) { + public void prepare(Map topoConf, TopologyContext context, + OutputCollector collector) { } diff --git a/storm-client/src/jvm/org/apache/storm/testing/TestPlannerSpout.java b/storm-client/src/jvm/org/apache/storm/testing/TestPlannerSpout.java index bec73af5737..e2a81618346 100644 --- a/storm-client/src/jvm/org/apache/storm/testing/TestPlannerSpout.java +++ b/storm-client/src/jvm/org/apache/storm/testing/TestPlannerSpout.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -22,7 +28,6 @@ import org.apache.storm.tuple.Fields; import org.apache.storm.utils.Utils; - public class TestPlannerSpout extends BaseRichSpout { boolean isDistributed; Fields outFields; @@ -44,9 +49,9 @@ public Fields getOutputFields() { return outFields; } - @Override - public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { + public void open(Map conf, TopologyContext context, + SpoutOutputCollector collector) { } diff --git a/storm-client/src/jvm/org/apache/storm/testing/TestSerObject.java b/storm-client/src/jvm/org/apache/storm/testing/TestSerObject.java index 42e7200c445..ac9a39ced2c 100644 --- a/storm-client/src/jvm/org/apache/storm/testing/TestSerObject.java +++ b/storm-client/src/jvm/org/apache/storm/testing/TestSerObject.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/testing/TestWordBytesCounter.java b/storm-client/src/jvm/org/apache/storm/testing/TestWordBytesCounter.java index d29320cc84d..a8d55ac896a 100644 --- a/storm-client/src/jvm/org/apache/storm/testing/TestWordBytesCounter.java +++ b/storm-client/src/jvm/org/apache/storm/testing/TestWordBytesCounter.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/testing/TestWordCounter.java b/storm-client/src/jvm/org/apache/storm/testing/TestWordCounter.java index 1f88c514ad5..ff8c84f6572 100644 --- a/storm-client/src/jvm/org/apache/storm/testing/TestWordCounter.java +++ b/storm-client/src/jvm/org/apache/storm/testing/TestWordCounter.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -25,7 +31,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; - public class TestWordCounter extends BaseBasicBolt { public static Logger LOG = LoggerFactory.getLogger(TestWordCounter.class); diff --git a/storm-client/src/jvm/org/apache/storm/testing/TestWordSpout.java b/storm-client/src/jvm/org/apache/storm/testing/TestWordSpout.java index 0d179ce1b12..91603ed0d6c 100644 --- a/storm-client/src/jvm/org/apache/storm/testing/TestWordSpout.java +++ b/storm-client/src/jvm/org/apache/storm/testing/TestWordSpout.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -26,7 +32,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; - public class TestWordSpout extends BaseRichSpout { public static Logger LOG = LoggerFactory.getLogger(TestWordSpout.class); boolean isDistributed; @@ -41,7 +46,8 @@ public TestWordSpout(boolean isDistributed) { } @Override - public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { + public void open(Map conf, TopologyContext context, + SpoutOutputCollector collector) { this.collector = collector; } diff --git a/storm-client/src/jvm/org/apache/storm/testing/TmpPath.java b/storm-client/src/jvm/org/apache/storm/testing/TmpPath.java index 115a3f0706c..3c7e83cfabb 100644 --- a/storm-client/src/jvm/org/apache/storm/testing/TmpPath.java +++ b/storm-client/src/jvm/org/apache/storm/testing/TmpPath.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -53,7 +59,7 @@ public void close() { try { FileUtils.forceDelete(path); } catch (Exception e) { - //on windows, the host process still holds lock on the logfile + // on windows, the host process still holds lock on the logfile LOG.info(e.getMessage()); } } diff --git a/storm-client/src/jvm/org/apache/storm/testing/TupleCaptureBolt.java b/storm-client/src/jvm/org/apache/storm/testing/TupleCaptureBolt.java index 6d073720297..55018471983 100644 --- a/storm-client/src/jvm/org/apache/storm/testing/TupleCaptureBolt.java +++ b/storm-client/src/jvm/org/apache/storm/testing/TupleCaptureBolt.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -27,12 +33,16 @@ public class TupleCaptureBolt implements IRichBolt { /* - * Even though normally bolts do not need to care about thread safety, this particular bolt is different. - * It maintains a static field that is prepopulated before the topology starts, is written into by the topology, - * and is then read from after the topology is completed - all of this by potentially different threads. + * Even though normally bolts do not need to care about thread safety, this particular bolt is + * different. + * It maintains a static field that is prepopulated before the topology starts, is written into + * by the topology, + * and is then read from after the topology is completed - all of this by potentially different + * threads. */ - private static final transient Map>> emitted_tuples = new ConcurrentHashMap<>(); + private static final transient Map>> emitted_tuples = + new ConcurrentHashMap<>(); private final String name; private OutputCollector collector; @@ -43,7 +53,8 @@ public TupleCaptureBolt() { } @Override - public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) { + public void prepare(Map topoConf, TopologyContext context, + OutputCollector collector) { this.collector = collector; } diff --git a/storm-client/src/jvm/org/apache/storm/topology/BaseConfigurationDeclarer.java b/storm-client/src/jvm/org/apache/storm/topology/BaseConfigurationDeclarer.java index 96665058b57..d959b2c7175 100644 --- a/storm-client/src/jvm/org/apache/storm/topology/BaseConfigurationDeclarer.java +++ b/storm-client/src/jvm/org/apache/storm/topology/BaseConfigurationDeclarer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -105,7 +111,8 @@ public T addResource(String resourceName, Number resourceValue) { @Override public T addResources(Map resources) { if (resources != null) { - Map currentResources = (Map) getComponentConfiguration().computeIfAbsent( + Map currentResources = (Map) getComponentConfiguration() + .computeIfAbsent( Config.TOPOLOGY_COMPONENT_RESOURCES_MAP, (k) -> new HashMap<>()); currentResources.putAll(resources); } diff --git a/storm-client/src/jvm/org/apache/storm/topology/BaseStatefulBoltExecutor.java b/storm-client/src/jvm/org/apache/storm/topology/BaseStatefulBoltExecutor.java index 1060d187e5c..f5d401f9ba8 100644 --- a/storm-client/src/jvm/org/apache/storm/topology/BaseStatefulBoltExecutor.java +++ b/storm-client/src/jvm/org/apache/storm/topology/BaseStatefulBoltExecutor.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -51,7 +57,8 @@ protected void init(TopologyContext context, OutputCollector collector) { } /** - * returns the total number of input checkpoint streams across all input tasks to this component. + * Returns the total number of input checkpoint streams across all input tasks to this + * component. */ private int getCheckpointInputTaskCount(TopologyContext context) { int count = 0; @@ -73,10 +80,12 @@ public void execute(Tuple input) { } /** - * Invokes handleCheckpoint once checkpoint tuple is received on all input checkpoint streams to this component. + * Invokes handleCheckpoint once checkpoint tuple is received on all input checkpoint streams to + * this component. */ private void processCheckpoint(Tuple input) { - CheckPointState.Action action = (CheckPointState.Action) input.getValueByField(CHECKPOINT_FIELD_ACTION); + CheckPointState.Action action = (CheckPointState.Action) input + .getValueByField(CHECKPOINT_FIELD_ACTION); long txid = input.getLongByField(CHECKPOINT_FIELD_TXID); if (shouldProcessTransaction(action, txid)) { LOG.debug("Processing action {}, txid {}", action, txid); @@ -98,14 +107,16 @@ private void processCheckpoint(Tuple input) { collector.reportError(th); } } else { - LOG.debug("Waiting for action {}, txid {} from all input tasks. checkPointInputTaskCount {}, " + LOG.debug("Waiting for action {}, txid {} from all input tasks. " + + "checkPointInputTaskCount {}, " + "transactionRequestCount {}", action, txid, checkPointInputTaskCount, transactionRequestCount); collector.ack(input); } } /** - * Checks if check points have been received from all tasks across all input streams to this component. + * Checks if check points have been received from all tasks across all input streams to this + * component. */ private boolean shouldProcessTransaction(CheckPointState.Action action, long txid) { TransactionRequest request = new TransactionRequest(action, txid); @@ -124,7 +135,8 @@ private boolean shouldProcessTransaction(CheckPointState.Action action, long txi } protected void declareCheckpointStream(OutputFieldsDeclarer declarer) { - declarer.declareStream(CHECKPOINT_STREAM_ID, new Fields(CHECKPOINT_FIELD_TXID, CHECKPOINT_FIELD_ACTION)); + declarer.declareStream(CHECKPOINT_STREAM_ID, new Fields(CHECKPOINT_FIELD_TXID, + CHECKPOINT_FIELD_ACTION)); } /** @@ -141,7 +153,8 @@ protected void declareCheckpointStream(OutputFieldsDeclarer declarer) { * @param action the action (prepare, commit, rollback or initstate) * @param txid the transaction id. */ - protected abstract void handleCheckpoint(Tuple checkpointTuple, CheckPointState.Action action, long txid); + protected abstract void handleCheckpoint(Tuple checkpointTuple, CheckPointState.Action action, + long txid); protected static class AnchoringOutputCollector extends OutputCollector { AnchoringOutputCollector(IOutputCollector delegate) { @@ -150,12 +163,14 @@ protected static class AnchoringOutputCollector extends OutputCollector { @Override public List emit(String streamId, List tuple) { - throw new UnsupportedOperationException("Bolts in a stateful topology must emit anchored tuples."); + throw new UnsupportedOperationException("Bolts in a stateful topology must emit " + + "anchored tuples."); } @Override public void emitDirect(int taskId, String streamId, List tuple) { - throw new UnsupportedOperationException("Bolts in a stateful topology must emit anchored tuples."); + throw new UnsupportedOperationException("Bolts in a stateful topology must emit " + + "anchored tuples."); } } diff --git a/storm-client/src/jvm/org/apache/storm/topology/BasicBoltExecutor.java b/storm-client/src/jvm/org/apache/storm/topology/BasicBoltExecutor.java index 3a806d040c1..bb16868695f 100644 --- a/storm-client/src/jvm/org/apache/storm/topology/BasicBoltExecutor.java +++ b/storm-client/src/jvm/org/apache/storm/topology/BasicBoltExecutor.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -34,9 +40,9 @@ public void declareOutputFields(OutputFieldsDeclarer declarer) { bolt.declareOutputFields(declarer); } - @Override - public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) { + public void prepare(Map topoConf, TopologyContext context, + OutputCollector collector) { bolt.prepare(topoConf, context); this.collector = new BasicOutputCollector(collector); } diff --git a/storm-client/src/jvm/org/apache/storm/topology/BasicOutputCollector.java b/storm-client/src/jvm/org/apache/storm/topology/BasicOutputCollector.java index 8ff4999449e..7a00b3a1b19 100644 --- a/storm-client/src/jvm/org/apache/storm/topology/BasicOutputCollector.java +++ b/storm-client/src/jvm/org/apache/storm/topology/BasicOutputCollector.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -18,7 +24,6 @@ import org.apache.storm.tuple.Tuple; import org.apache.storm.utils.Utils; - public class BasicOutputCollector implements IBasicOutputCollector { private OutputCollector out; private Tuple inputTuple; @@ -50,8 +55,10 @@ public void emitDirect(int taskId, List tuple) { } /** - * Resets the message timeout for any tuple trees to which the given tuple belongs. The timeout is reset to - * Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS. Note that this is an expensive operation, and should be used sparingly. + * Resets the message timeout for any tuple trees to which the given tuple belongs. The timeout + * is reset to + * Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS. Note that this is an expensive operation, and should be + * used sparingly. * * @param tuple the tuple to reset timeout for */ diff --git a/storm-client/src/jvm/org/apache/storm/topology/BoltDeclarer.java b/storm-client/src/jvm/org/apache/storm/topology/BoltDeclarer.java index a198f97d776..5bec7c656c2 100644 --- a/storm-client/src/jvm/org/apache/storm/topology/BoltDeclarer.java +++ b/storm-client/src/jvm/org/apache/storm/topology/BoltDeclarer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/topology/CheckpointTupleForwarder.java b/storm-client/src/jvm/org/apache/storm/topology/CheckpointTupleForwarder.java index d4ed032e286..ad6fb15ac8a 100644 --- a/storm-client/src/jvm/org/apache/storm/topology/CheckpointTupleForwarder.java +++ b/storm-client/src/jvm/org/apache/storm/topology/CheckpointTupleForwarder.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -16,7 +22,6 @@ import static org.apache.storm.spout.CheckpointSpout.CHECKPOINT_STREAM_ID; import java.util.Map; - import org.apache.storm.task.OutputCollector; import org.apache.storm.task.TopologyContext; import org.apache.storm.tuple.Tuple; @@ -26,8 +31,9 @@ /** * Wraps {@link IRichBolt} and forwards checkpoint tuples in a stateful topology. - *

    - * When a storm topology contains one or more {@link IStatefulBolt} all non-stateful bolts are wrapped in {@link CheckpointTupleForwarder} + * + *

    When a storm topology contains one or more {@link IStatefulBolt} all non-stateful bolts are + * wrapped in {@link CheckpointTupleForwarder} * so that the checkpoint tuples can flow through the entire topology DAG. *

    */ @@ -40,7 +46,8 @@ public CheckpointTupleForwarder(IRichBolt bolt) { } @Override - public void prepare(Map topoConf, TopologyContext context, OutputCollector outputCollector) { + public void prepare(Map topoConf, TopologyContext context, + OutputCollector outputCollector) { init(context, new AnchoringOutputCollector(outputCollector)); bolt.prepare(topoConf, context, collector); } @@ -77,8 +84,9 @@ protected void handleCheckpoint(Tuple checkpointTuple, Action action, long txid) /** * Hands off tuple to the wrapped bolt to execute. * - *

    - * Right now tuples continue to get forwarded while waiting for checkpoints to arrive on other streams after checkpoint arrives on one + *

    Right now tuples continue to get forwarded while waiting for checkpoints to arrive on + * other + * streams after checkpoint arrives on one * of the streams. This can cause duplicates but still at least once. *

    * diff --git a/storm-client/src/jvm/org/apache/storm/topology/ComponentConfigurationDeclarer.java b/storm-client/src/jvm/org/apache/storm/topology/ComponentConfigurationDeclarer.java index 4b7c4b8c30a..86914feafc2 100644 --- a/storm-client/src/jvm/org/apache/storm/topology/ComponentConfigurationDeclarer.java +++ b/storm-client/src/jvm/org/apache/storm/topology/ComponentConfigurationDeclarer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -16,7 +22,7 @@ public interface ComponentConfigurationDeclarer extends ResourceDeclarer { /** - * add in several configs to the component. + * Add in several configs to the component. * * @param conf the configs to add * @return this for chaining. @@ -24,7 +30,7 @@ public interface ComponentConfigurationDeclarer conf); /** - * return the current component configuration. + * Return the current component configuration. * * @return the current configuration. */ diff --git a/storm-client/src/jvm/org/apache/storm/topology/ConfigurableTopology.java b/storm-client/src/jvm/org/apache/storm/topology/ConfigurableTopology.java index cf25a035f93..c00ec430840 100644 --- a/storm-client/src/jvm/org/apache/storm/topology/ConfigurableTopology.java +++ b/storm-client/src/jvm/org/apache/storm/topology/ConfigurableTopology.java @@ -39,7 +39,8 @@ import org.slf4j.LoggerFactory; /** - * Extensions of this class takes a reference to one or more configuration files. The main() method should call ConfigurableTopology.start() + * Extensions of this class takes a reference to one or more configuration files. The main() method + * should call ConfigurableTopology.start() * and it must instantiate a TopologyBuilder in the run() method. * *

    diff --git a/storm-client/src/jvm/org/apache/storm/topology/FailedException.java b/storm-client/src/jvm/org/apache/storm/topology/FailedException.java
    index fc0661877f3..0f44b86580d 100644
    --- a/storm-client/src/jvm/org/apache/storm/topology/FailedException.java
    +++ b/storm-client/src/jvm/org/apache/storm/topology/FailedException.java
    @@ -1,12 +1,18 @@
     /**
    - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements.  See the NOTICE file distributed with
    - * this work for additional information regarding copyright ownership.  The ASF licenses this file to you under the Apache License, Version
    - * 2.0 (the "License"); you may not use this file except in compliance with the License.  You may obtain a copy of the License at
    + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
    + * agreements. See the NOTICE file distributed with
    + * this work for additional information regarding copyright ownership. The ASF licenses this file to
    + * you under the Apache License, Version
    + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
    + * obtain a copy of the License at
      *
    - * http://www.apache.org/licenses/LICENSE-2.0
    + * 

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/topology/IBasicBolt.java b/storm-client/src/jvm/org/apache/storm/topology/IBasicBolt.java index 12c7a64b8ef..53daa1c6c81 100644 --- a/storm-client/src/jvm/org/apache/storm/topology/IBasicBolt.java +++ b/storm-client/src/jvm/org/apache/storm/topology/IBasicBolt.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/topology/IBasicOutputCollector.java b/storm-client/src/jvm/org/apache/storm/topology/IBasicOutputCollector.java index 592b5e5be95..f3f9cf10539 100644 --- a/storm-client/src/jvm/org/apache/storm/topology/IBasicOutputCollector.java +++ b/storm-client/src/jvm/org/apache/storm/topology/IBasicOutputCollector.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/topology/IComponent.java b/storm-client/src/jvm/org/apache/storm/topology/IComponent.java index 738d047559d..3bddc6ff6c5 100644 --- a/storm-client/src/jvm/org/apache/storm/topology/IComponent.java +++ b/storm-client/src/jvm/org/apache/storm/topology/IComponent.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -16,20 +22,24 @@ import java.util.Map; /** - * Common methods for all possible components in a topology. This interface is used when defining topologies using the Java API. + * Common methods for all possible components in a topology. This interface is used when defining + * topologies using the Java API. */ public interface IComponent extends Serializable { /** * Declare the output schema for all the streams of this topology. * - * @param declarer this is used to declare output stream ids, output fields, and whether or not each output stream is a direct stream + * @param declarer this is used to declare output stream ids, output fields, and whether or not + * each output stream is a direct stream */ void declareOutputFields(OutputFieldsDeclarer declarer); /** - * Declare configuration specific to this component. Only a subset of the "topology.*" configs can be overridden. The component - * configuration can be further overridden when constructing the topology using {@link TopologyBuilder} + * Declare configuration specific to this component. Only a subset of the "topology.*" configs + * can be overridden. The component + * configuration can be further overridden when constructing the topology using {@link + * TopologyBuilder} */ Map getComponentConfiguration(); diff --git a/storm-client/src/jvm/org/apache/storm/topology/IRichBolt.java b/storm-client/src/jvm/org/apache/storm/topology/IRichBolt.java index ea0806be6ea..b0db0161a85 100644 --- a/storm-client/src/jvm/org/apache/storm/topology/IRichBolt.java +++ b/storm-client/src/jvm/org/apache/storm/topology/IRichBolt.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -15,7 +21,8 @@ import org.apache.storm.task.IBolt; /** - * When writing topologies using Java, {@link IRichBolt} and {@link IRichSpout} are the main interfaces to use to implement components of + * When writing topologies using Java, {@link IRichBolt} and {@link IRichSpout} are the main + * interfaces to use to implement components of * the topology. */ public interface IRichBolt extends IBolt, IComponent { diff --git a/storm-client/src/jvm/org/apache/storm/topology/IRichSpout.java b/storm-client/src/jvm/org/apache/storm/topology/IRichSpout.java index e8e1f945142..246ecececc3 100644 --- a/storm-client/src/jvm/org/apache/storm/topology/IRichSpout.java +++ b/storm-client/src/jvm/org/apache/storm/topology/IRichSpout.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -15,7 +21,8 @@ import org.apache.storm.spout.ISpout; /** - * When writing topologies using Java, {@link IRichBolt} and {@link IRichSpout} are the main interfaces to use to implement components of + * When writing topologies using Java, {@link IRichBolt} and {@link IRichSpout} are the main + * interfaces to use to implement components of * the topology. */ public interface IRichSpout extends ISpout, IComponent { diff --git a/storm-client/src/jvm/org/apache/storm/topology/IRichStateSpout.java b/storm-client/src/jvm/org/apache/storm/topology/IRichStateSpout.java index 6c13e8e505d..a9c6c52316b 100644 --- a/storm-client/src/jvm/org/apache/storm/topology/IRichStateSpout.java +++ b/storm-client/src/jvm/org/apache/storm/topology/IRichStateSpout.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -14,7 +20,6 @@ import org.apache.storm.state.IStateSpout; - public interface IRichStateSpout extends IStateSpout, IComponent { } diff --git a/storm-client/src/jvm/org/apache/storm/topology/IStatefulBolt.java b/storm-client/src/jvm/org/apache/storm/topology/IStatefulBolt.java index 5238ad53894..9e17648d844 100644 --- a/storm-client/src/jvm/org/apache/storm/topology/IStatefulBolt.java +++ b/storm-client/src/jvm/org/apache/storm/topology/IStatefulBolt.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -19,26 +25,31 @@ import org.apache.storm.tuple.Tuple; /** - * A bolt abstraction for supporting stateful computation. The state of the bolt is periodically checkpointed. + * A bolt abstraction for supporting stateful computation. The state of the bolt is periodically + * checkpointed. * *

    The framework provides at-least once guarantee for the - * state updates. The stateful bolts are expected to anchor the tuples while emitting and ack the input tuples once its processed.

    + * state updates. The stateful bolts are expected to anchor the tuples while emitting and ack the + * input tuples once its processed.

    */ public interface IStatefulBolt extends IStatefulComponent { /** * Analogue to bolt function. + * * @see org.apache.storm.task.IBolt#prepare(Map, TopologyContext, OutputCollector) */ void prepare(Map topoConf, TopologyContext context, OutputCollector collector); /** * Analogue to bolt function. + * * @see org.apache.storm.task.IBolt#execute(Tuple) */ void execute(Tuple input); /** * Analogue to bolt function. + * * @see org.apache.storm.task.IBolt#cleanup() */ void cleanup(); diff --git a/storm-client/src/jvm/org/apache/storm/topology/IStatefulComponent.java b/storm-client/src/jvm/org/apache/storm/topology/IStatefulComponent.java index c4649390212..15a526c7d0a 100644 --- a/storm-client/src/jvm/org/apache/storm/topology/IStatefulComponent.java +++ b/storm-client/src/jvm/org/apache/storm/topology/IStatefulComponent.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -15,14 +21,15 @@ import org.apache.storm.state.State; /** - *

    - * Common methods for stateful components in the topology. + *

    Common methods for stateful components in the topology. *

    - * A stateful component is one that has state (e.g. the result of some computation in a bolt) and wants the framework to manage its state. + * A stateful component is one that has state (e.g. the result of some computation in a bolt) and + * wants the framework to manage its state. */ public interface IStatefulComponent extends IComponent { /** - * This method is invoked by the framework with the previously saved state of the component. This is invoked after prepare but before + * This method is invoked by the framework with the previously saved state of the component. + * This is invoked after prepare but before * the component starts processing tuples. * * @param state the previously saved state of the component. @@ -30,17 +37,20 @@ public interface IStatefulComponent extends IComponent { void initState(T state); /** - * This is a hook for the component to perform some actions just before the framework commits its state. + * This is a hook for the component to perform some actions just before the framework commits + * its state. */ void preCommit(long txid); /** - * This is a hook for the component to perform some actions just before the framework prepares its state. + * This is a hook for the component to perform some actions just before the framework prepares + * its state. */ void prePrepare(long txid); /** - * This is a hook for the component to perform some actions just before the framework rolls back the prepared state. + * This is a hook for the component to perform some actions just before the framework rolls back + * the prepared state. */ void preRollback(); } diff --git a/storm-client/src/jvm/org/apache/storm/topology/IStatefulWindowedBolt.java b/storm-client/src/jvm/org/apache/storm/topology/IStatefulWindowedBolt.java index cba44bf63ae..707b5459be0 100644 --- a/storm-client/src/jvm/org/apache/storm/topology/IStatefulWindowedBolt.java +++ b/storm-client/src/jvm/org/apache/storm/topology/IStatefulWindowedBolt.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -19,9 +25,10 @@ */ public interface IStatefulWindowedBolt extends IStatefulComponent, IWindowedBolt { /** - * If the stateful windowed bolt should have its windows persisted in state and maintain a subset of events in memory. - *

    - * The default is to keep all the window events in memory. + * If the stateful windowed bolt should have its windows persisted in state and maintain a + * subset of events in memory. + * + *

    The default is to keep all the window events in memory. *

    * * @return true if the windows should be persisted diff --git a/storm-client/src/jvm/org/apache/storm/topology/IWindowedBolt.java b/storm-client/src/jvm/org/apache/storm/topology/IWindowedBolt.java index 1f1fa3d6b58..9595ca4843f 100644 --- a/storm-client/src/jvm/org/apache/storm/topology/IWindowedBolt.java +++ b/storm-client/src/jvm/org/apache/storm/topology/IWindowedBolt.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -23,20 +29,23 @@ */ public interface IWindowedBolt extends IComponent { /** - * This is similar to the {@link org.apache.storm.task.IBolt#prepare(Map, TopologyContext, OutputCollector)} except that while emitting, + * This is similar to the {@link org.apache.storm.task.IBolt#prepare(Map, TopologyContext, + * OutputCollector)} except that while emitting, * the tuples are automatically anchored to the tuples in the inputWindow. */ void prepare(Map topoConf, TopologyContext context, OutputCollector collector); /** - * Process the tuple window and optionally emit new tuples based on the tuples in the input window. + * Process the tuple window and optionally emit new tuples based on the tuples in the input + * window. */ void execute(TupleWindow inputWindow); void cleanup(); /** - * Return a {@link TimestampExtractor} for extracting timestamps from a tuple for event time based processing, or null for processing + * Return a {@link TimestampExtractor} for extracting timestamps from a tuple for event time + * based processing, or null for processing * time. * * @return the timestamp extractor diff --git a/storm-client/src/jvm/org/apache/storm/topology/InputDeclarer.java b/storm-client/src/jvm/org/apache/storm/topology/InputDeclarer.java index 75030ea948d..5de2166202f 100644 --- a/storm-client/src/jvm/org/apache/storm/topology/InputDeclarer.java +++ b/storm-client/src/jvm/org/apache/storm/topology/InputDeclarer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -17,7 +23,6 @@ import org.apache.storm.grouping.CustomStreamGrouping; import org.apache.storm.tuple.Fields; - public interface InputDeclarer { /** * The stream is partitioned by the fields specified in the grouping. @@ -30,33 +35,39 @@ public interface InputDeclarer { T fieldsGrouping(String componentId, String streamId, Fields fields); /** - * The entire stream goes to a single one of the bolt's tasks. Specifically, it goes to the task with the lowest id. + * The entire stream goes to a single one of the bolt's tasks. Specifically, it goes to the task + * with the lowest id. */ T globalGrouping(String componentId); /** - * The entire stream goes to a single one of the bolt's tasks. Specifically, it goes to the task with the lowest id. + * The entire stream goes to a single one of the bolt's tasks. Specifically, it goes to the task + * with the lowest id. */ T globalGrouping(String componentId, String streamId); /** - * Tuples are randomly distributed across the bolt's tasks in a way such that each bolt is guaranteed to get an equal number of tuples. + * Tuples are randomly distributed across the bolt's tasks in a way such that each bolt is + * guaranteed to get an equal number of tuples. */ T shuffleGrouping(String componentId); /** - * Tuples are randomly distributed across the bolt's tasks in a way such that each bolt is guaranteed to get an equal number of tuples. + * Tuples are randomly distributed across the bolt's tasks in a way such that each bolt is + * guaranteed to get an equal number of tuples. */ T shuffleGrouping(String componentId, String streamId); /** - * If the target bolt has one or more tasks in the same worker process, tuples will be shuffled to just those in-process tasks. + * If the target bolt has one or more tasks in the same worker process, tuples will be shuffled + * to just those in-process tasks. * Otherwise, this acts like a normal shuffle grouping. */ T localOrShuffleGrouping(String componentId); /** - * If the target bolt has one or more tasks in the same worker process, tuples will be shuffled to just those in-process tasks. + * If the target bolt has one or more tasks in the same worker process, tuples will be shuffled + * to just those in-process tasks. * Otherwise, this acts like a normal shuffle grouping. */ T localOrShuffleGrouping(String componentId, String streamId); @@ -82,17 +93,20 @@ public interface InputDeclarer { T allGrouping(String componentId, String streamId); /** - * A stream grouped this way means that the producer of the tuple decides which task of the consumer will receive this tuple. + * A stream grouped this way means that the producer of the tuple decides which task of the + * consumer will receive this tuple. */ T directGrouping(String componentId); /** - * A stream grouped this way means that the producer of the tuple decides which task of the consumer will receive this tuple. + * A stream grouped this way means that the producer of the tuple decides which task of the + * consumer will receive this tuple. */ T directGrouping(String componentId, String streamId); /** - * Tuples are passed to two hashing functions and each target task is decided based on the comparison of the state of candidate nodes. + * Tuples are passed to two hashing functions and each target task is decided based on the + * comparison of the state of candidate nodes. * *

    See https://melmeric.files.wordpress.com/2014/11/the-power-of-both-choices-practical-load-balancing-for-distributed-stream * -processing-engines.pdf @@ -100,7 +114,8 @@ public interface InputDeclarer { T partialKeyGrouping(String componentId, Fields fields); /** - * Tuples are passed to two hashing functions and each target task is decided based on the comparison of the state of candidate nodes. + * Tuples are passed to two hashing functions and each target task is decided based on the + * comparison of the state of candidate nodes. * *

    See https://melmeric.files.wordpress.com/2014/11/the-power-of-both-choices-practical-load-balancing-for-distributed-stream * -processing-engines.pdf diff --git a/storm-client/src/jvm/org/apache/storm/topology/OutputFieldsDeclarer.java b/storm-client/src/jvm/org/apache/storm/topology/OutputFieldsDeclarer.java index 77e51c11da6..d2208f3ad9e 100644 --- a/storm-client/src/jvm/org/apache/storm/topology/OutputFieldsDeclarer.java +++ b/storm-client/src/jvm/org/apache/storm/topology/OutputFieldsDeclarer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -14,7 +20,6 @@ import org.apache.storm.tuple.Fields; - public interface OutputFieldsDeclarer { /** * Uses default stream id. diff --git a/storm-client/src/jvm/org/apache/storm/topology/OutputFieldsGetter.java b/storm-client/src/jvm/org/apache/storm/topology/OutputFieldsGetter.java index 3d4fddf57ea..04238708b73 100644 --- a/storm-client/src/jvm/org/apache/storm/topology/OutputFieldsGetter.java +++ b/storm-client/src/jvm/org/apache/storm/topology/OutputFieldsGetter.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -47,7 +53,6 @@ public void declareStream(String streamId, boolean direct, Fields fields) { this.fields.put(streamId, new StreamInfo(fields.toList(), direct)); } - public Map getFieldsDeclaration() { return fields; } diff --git a/storm-client/src/jvm/org/apache/storm/topology/PersistentWindowedBoltExecutor.java b/storm-client/src/jvm/org/apache/storm/topology/PersistentWindowedBoltExecutor.java index f055acc577e..2e17d58112c 100644 --- a/storm-client/src/jvm/org/apache/storm/topology/PersistentWindowedBoltExecutor.java +++ b/storm-client/src/jvm/org/apache/storm/topology/PersistentWindowedBoltExecutor.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -41,8 +47,10 @@ import org.slf4j.LoggerFactory; /** - * Wraps a {@link IStatefulWindowedBolt} and handles the execution. Uses state and the underlying checkpointing mechanisms to save the - * tuples in window to state. The tuples are also kept in-memory by transparently caching the window partitions and checkpointing them as + * Wraps a {@link IStatefulWindowedBolt} and handles the execution. Uses state and the underlying + * checkpointing mechanisms to save the + * tuples in window to state. The tuples are also kept in-memory by transparently caching the window + * partitions and checkpointing them as * needed. */ public class PersistentWindowedBoltExecutor extends WindowedBoltExecutor implements IStatefulBolt { @@ -60,8 +68,10 @@ public PersistentWindowedBoltExecutor(IStatefulWindowedBolt bolt) { } @Override - public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) { - List registrations = (List) topoConf.getOrDefault(Config.TOPOLOGY_STATE_KRYO_REGISTER, new ArrayList<>()); + public void prepare(Map topoConf, TopologyContext context, + OutputCollector collector) { + List registrations = (List) topoConf + .getOrDefault(Config.TOPOLOGY_STATE_KRYO_REGISTER, new ArrayList<>()); registrations.add(ConcurrentLinkedQueue.class.getName()); registrations.add(LinkedList.class.getName()); registrations.add(AtomicInteger.class.getName()); @@ -69,7 +79,8 @@ public void prepare(Map topoConf, TopologyContext context, Outpu registrations.add(WindowPartition.class.getName()); registrations.add(DefaultEvictionContext.class.getName()); topoConf.put(Config.TOPOLOGY_STATE_KRYO_REGISTER, registrations); - prepare(topoConf, context, collector, getWindowState(topoConf, context), getPartitionState(topoConf, context), + prepare(topoConf, context, collector, getWindowState(topoConf, context), + getPartitionState(topoConf, context), getWindowSystemState(topoConf, context)); } @@ -107,7 +118,8 @@ protected void validate(Map topoConf, int timeout = getTopologyTimeoutMillis(topoConf); if (interval > timeout) { throw new IllegalArgumentException(Config.TOPOLOGY_STATE_CHECKPOINT_INTERVAL + interval - + " is more than " + Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS + + " is more than " + + Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS + " value " + timeout); } } @@ -115,7 +127,8 @@ protected void validate(Map topoConf, private int getCheckpointIntervalMillis(Map topoConf) { int checkpointInterval = Integer.MAX_VALUE; if (topoConf.get(Config.TOPOLOGY_STATE_CHECKPOINT_INTERVAL) != null) { - checkpointInterval = ((Number) topoConf.get(Config.TOPOLOGY_STATE_CHECKPOINT_INTERVAL)).intValue(); + checkpointInterval = ((Number) topoConf.get(Config.TOPOLOGY_STATE_CHECKPOINT_INTERVAL)) + .intValue(); } return checkpointInterval; } @@ -132,7 +145,8 @@ protected void start() { @Override public void execute(Tuple input) { if (!stateInitialized) { - throw new IllegalStateException("execute invoked before initState with input tuple " + input); + throw new IllegalStateException("execute invoked before initState with input tuple " + + input); } super.execute(input); // StatefulBoltExecutor does the actual ack when the state is saved. @@ -206,8 +220,10 @@ public void onActivation(Supplier> eventsIt, Supplier> expiredIt, Long timestamp) { /* - * Here we don't set the tuples in windowedOutputCollector's context and emit un-anchored. - * The checkpoint tuple will trigger a checkpoint in the receiver with the emitted tuples. + * Here we don't set the tuples in windowedOutputCollector's context and emit + * un-anchored. + * The checkpoint tuple will trigger a checkpoint in the receiver with the emitted + * tuples. */ boltExecute(eventsIt, newEventsIt, expiredIt, timestamp); state.clearIteratorPins(); @@ -215,23 +231,32 @@ public void onActivation(Supplier> eventsIt, }; } - private KeyValueState> getWindowState(Map topoConf, TopologyContext context) { + private KeyValueState> getWindowState(Map topoConf, + TopologyContext context) { String namespace = context.getThisComponentId() + "-" + context.getThisTaskId() + "-window"; - return (KeyValueState>) StateFactory.getState(namespace, topoConf, context); + return (KeyValueState>) StateFactory.getState(namespace, + topoConf, context); } - private KeyValueState> getPartitionState(Map topoConf, TopologyContext context) { - String namespace = context.getThisComponentId() + "-" + context.getThisTaskId() + "-window-partitions"; - return (KeyValueState>) StateFactory.getState(namespace, topoConf, context); + private KeyValueState> getPartitionState(Map topoConf, + TopologyContext context) { + String namespace = context.getThisComponentId() + "-" + context.getThisTaskId() + + "-window-partitions"; + return (KeyValueState>) StateFactory.getState(namespace, topoConf, + context); } - private KeyValueState> getWindowSystemState(Map topoConf, TopologyContext context) { - String namespace = context.getThisComponentId() + "-" + context.getThisTaskId() + "-window-systemstate"; - return (KeyValueState>) StateFactory.getState(namespace, topoConf, context); + private KeyValueState> getWindowSystemState(Map topoConf, + TopologyContext context) { + String namespace = context.getThisComponentId() + "-" + context.getThisTaskId() + + "-window-systemstate"; + return (KeyValueState>) StateFactory.getState(namespace, topoConf, + context); } /** - * Creates an {@link OutputCollector} wrapper that ignores acks. The {@link PersistentWindowedBoltExecutor} acks the tuples in execute + * Creates an {@link OutputCollector} wrapper that ignores acks. The {@link + * PersistentWindowedBoltExecutor} acks the tuples in execute * and this is to prevent double ack-ing */ private static class NoAckOutputCollector extends OutputCollector { diff --git a/storm-client/src/jvm/org/apache/storm/topology/ReportedFailedException.java b/storm-client/src/jvm/org/apache/storm/topology/ReportedFailedException.java index 96192b17e6b..c2b94b4cd6f 100644 --- a/storm-client/src/jvm/org/apache/storm/topology/ReportedFailedException.java +++ b/storm-client/src/jvm/org/apache/storm/topology/ReportedFailedException.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/topology/ResourceDeclarer.java b/storm-client/src/jvm/org/apache/storm/topology/ResourceDeclarer.java index 8cba971a73b..bc99d43c6fe 100644 --- a/storm-client/src/jvm/org/apache/storm/topology/ResourceDeclarer.java +++ b/storm-client/src/jvm/org/apache/storm/topology/ResourceDeclarer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -15,7 +21,8 @@ import org.apache.storm.generated.SharedMemory; /** - * This is a new base interface that can be used by anything that wants to mirror RAS's basic API. Trident uses this to allow setting + * This is a new base interface that can be used by anything that wants to mirror RAS's basic API. + * Trident uses this to allow setting * resources in the Stream API. */ public interface ResourceDeclarer { @@ -46,7 +53,8 @@ public interface ResourceDeclarer { T setCPULoad(Number amount); /** - * Add in request for shared memory that this component will use. See {@link SharedOnHeap}, {@link SharedOffHeapWithinNode}, and {@link + * Add in request for shared memory that this component will use. See {@link SharedOnHeap}, + * {@link SharedOffHeapWithinNode}, and {@link * SharedOffHeapWithinWorker} for convenient ways to create shared memory requests. * * @param request the shared memory request for this component diff --git a/storm-client/src/jvm/org/apache/storm/topology/SharedOffHeapWithinNode.java b/storm-client/src/jvm/org/apache/storm/topology/SharedOffHeapWithinNode.java index 0803cb19b51..46d231855cc 100644 --- a/storm-client/src/jvm/org/apache/storm/topology/SharedOffHeapWithinNode.java +++ b/storm-client/src/jvm/org/apache/storm/topology/SharedOffHeapWithinNode.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/topology/SharedOffHeapWithinWorker.java b/storm-client/src/jvm/org/apache/storm/topology/SharedOffHeapWithinWorker.java index 7d2aacb3575..1d1c0b84b8e 100644 --- a/storm-client/src/jvm/org/apache/storm/topology/SharedOffHeapWithinWorker.java +++ b/storm-client/src/jvm/org/apache/storm/topology/SharedOffHeapWithinWorker.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/topology/SharedOnHeap.java b/storm-client/src/jvm/org/apache/storm/topology/SharedOnHeap.java index b678c31e094..1aa0b855f11 100644 --- a/storm-client/src/jvm/org/apache/storm/topology/SharedOnHeap.java +++ b/storm-client/src/jvm/org/apache/storm/topology/SharedOnHeap.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/topology/SpoutDeclarer.java b/storm-client/src/jvm/org/apache/storm/topology/SpoutDeclarer.java index 8d755886c95..d3a1d2e6cd8 100644 --- a/storm-client/src/jvm/org/apache/storm/topology/SpoutDeclarer.java +++ b/storm-client/src/jvm/org/apache/storm/topology/SpoutDeclarer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/topology/StatefulBoltExecutor.java b/storm-client/src/jvm/org/apache/storm/topology/StatefulBoltExecutor.java index d2202c4c727..ebe8773919c 100644 --- a/storm-client/src/jvm/org/apache/storm/topology/StatefulBoltExecutor.java +++ b/storm-client/src/jvm/org/apache/storm/topology/StatefulBoltExecutor.java @@ -1,22 +1,28 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.topology; -import static org.apache.storm.spout.CheckPointState.Action; import static org.apache.storm.spout.CheckPointState.Action.COMMIT; import static org.apache.storm.spout.CheckPointState.Action.INITSTATE; import static org.apache.storm.spout.CheckPointState.Action.PREPARE; import static org.apache.storm.spout.CheckPointState.Action.ROLLBACK; +import static org.apache.storm.spout.CheckPointState.Action; import java.util.ArrayList; import java.util.Iterator; @@ -51,14 +57,16 @@ public StatefulBoltExecutor(IStatefulBolt bolt) { } @Override - public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) { + public void prepare(Map topoConf, TopologyContext context, + OutputCollector collector) { // get the last successfully committed state from state store String namespace = context.getThisComponentId() + "-" + context.getThisTaskId(); prepare(topoConf, context, collector, StateFactory.getState(namespace, topoConf, context)); } // package access for unit tests - void prepare(Map topoConf, TopologyContext context, OutputCollector collector, State state) { + void prepare(Map topoConf, TopologyContext context, OutputCollector collector, + State state) { init(context, collector); this.collector = new AckTrackingOutputCollector(collector); bolt.prepare(topoConf, context, this.collector); @@ -81,10 +89,10 @@ public Map getComponentConfiguration() { return bolt.getComponentConfiguration(); } - @Override protected void handleCheckpoint(Tuple checkpointTuple, Action action, long txid) { - LOG.debug("handleCheckPoint with tuple {}, action {}, txid {}", checkpointTuple, action, txid); + LOG.debug("handleCheckPoint with tuple {}, action {}, txid {}", checkpointTuple, action, + txid); if (action == PREPARE) { if (boltInitialized) { bolt.prePrepare(txid); @@ -95,7 +103,8 @@ protected void handleCheckpoint(Tuple checkpointTuple, Action action, long txid) * May be the task restarted in the middle and the state needs be initialized. * Fail fast and trigger recovery. */ - LOG.debug("Failing checkpointTuple, PREPARE received when bolt state is not initialized."); + LOG.debug("Failing checkpointTuple, PREPARE received when bolt state is not " + + "initialized."); collector.fail(checkpointTuple); return; } @@ -119,15 +128,18 @@ protected void handleCheckpoint(Tuple checkpointTuple, Action action, long txid) pendingTuples.clear(); } else { /* - * If a worker crashes, the states of all workers are rolled back and an initState message is sent across + * If a worker crashes, the states of all workers are rolled back and an initState + * message is sent across * the topology so that crashed workers can initialize their state. * The bolts that have their state already initialized need not be re-initialized. */ - LOG.debug("Bolt state is already initialized, ignoring tuple {}, action {}, txid {}", + LOG.debug("Bolt state is already initialized, ignoring tuple {}, action {}, txid " + + "{}", checkpointTuple, action, txid); } } - collector.emit(CheckpointSpout.CHECKPOINT_STREAM_ID, checkpointTuple, new Values(txid, action)); + collector.emit(CheckpointSpout.CHECKPOINT_STREAM_ID, checkpointTuple, new Values(txid, + action)); collector.delegate.ack(checkpointTuple); } diff --git a/storm-client/src/jvm/org/apache/storm/topology/StatefulWindowedBoltExecutor.java b/storm-client/src/jvm/org/apache/storm/topology/StatefulWindowedBoltExecutor.java index 0bde3e7577e..f8f267a784e 100644 --- a/storm-client/src/jvm/org/apache/storm/topology/StatefulWindowedBoltExecutor.java +++ b/storm-client/src/jvm/org/apache/storm/topology/StatefulWindowedBoltExecutor.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -29,7 +35,8 @@ import org.slf4j.LoggerFactory; /** - * Wraps a {@link IStatefulWindowedBolt} and handles the execution. Saves the last expired and evaluated states of the window during + * Wraps a {@link IStatefulWindowedBolt} and handles the execution. Saves the last expired and + * evaluated states of the window during * checkpoint and restores the state during recovery. */ public class StatefulWindowedBoltExecutor extends WindowedBoltExecutor implements IStatefulBolt { @@ -52,7 +59,8 @@ public StatefulWindowedBoltExecutor(IStatefulWindowedBolt bolt) { } @Override - public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) { + public void prepare(Map topoConf, TopologyContext context, + OutputCollector collector) { prepare(topoConf, context, collector, getWindowState(topoConf, context)); } @@ -63,12 +71,14 @@ void prepare(Map topoConf, TopologyContext context, OutputCollec super.prepare(topoConf, context, collector); } - private void init(Map topoConf, TopologyContext context, OutputCollector collector, + private void init(Map topoConf, TopologyContext context, + OutputCollector collector, KeyValueState windowState) { if (topoConf.containsKey(Config.TOPOLOGY_BOLTS_MESSAGE_ID_FIELD_NAME)) { msgIdFieldName = (String) topoConf.get(Config.TOPOLOGY_BOLTS_MESSAGE_ID_FIELD_NAME); } else { - throw new IllegalArgumentException(Config.TOPOLOGY_BOLTS_MESSAGE_ID_FIELD_NAME + " is not set"); + throw new IllegalArgumentException(Config.TOPOLOGY_BOLTS_MESSAGE_ID_FIELD_NAME + + " is not set"); } topologyContext = context; outputCollector = collector; @@ -82,7 +92,8 @@ private void init(Map topoConf, TopologyContext context, OutputC @Override public void execute(Tuple input) { if (!isStateInitialized()) { - throw new IllegalStateException("execute invoked before initState with input tuple " + input); + throw new IllegalStateException("execute invoked before initState with input tuple " + + input); } else if (isRecovering()) { handleRecovery(input); } else { @@ -107,12 +118,14 @@ private void handleRecovery(Tuple input) { if (state != null) { LOG.debug("Tuple msgid {}, saved state {}", msgId, state); if (msgId <= state.lastExpired) { - LOG.debug("Ignoring tuple since msg id {} <= lastExpired id {}", msgId, state.lastExpired); + LOG.debug("Ignoring tuple since msg id {} <= lastExpired id {}", msgId, + state.lastExpired); outputCollector.ack(input); } else if (msgId <= state.lastEvaluated) { super.execute(input); } else { - LOG.debug("Tuple msg id {} > lastEvaluated id {}, adding to pendingTuples and clearing recovery state " + LOG.debug("Tuple msg id {} > lastEvaluated id {}, adding to pendingTuples and " + + "clearing recovery state " + "for taskStream {}", msgId, state.lastEvaluated, @@ -204,9 +217,12 @@ public void onExpiry(List events) { } @Override - public void onActivation(List events, List newEvents, List expired, Long timestamp) { + public void onActivation(List events, List newEvents, List expired, + Long timestamp) { if (isRecovering()) { - String msg = String.format("Unexpected activation with events %s, newEvents %s, expired %s in " + String msg = String + .format("Unexpected activation with events %s, newEvents %s, expired " + + "%s in " + "recovering state. recoveryStates %s ", events, newEvents, @@ -223,14 +239,16 @@ public void onActivation(List events, List newEvents, List } private void updateWindowState(List expired, List newEvents) { - LOG.debug("Update window state, {} expired, {} new events", expired.size(), newEvents.size()); + LOG.debug("Update window state, {} expired, {} new events", expired.size(), newEvents + .size()); Map state = new HashMap<>(); updateState(state, expired, false); updateState(state, newEvents, true); updateStreamState(state); } - private void updateState(Map state, List tuples, boolean newEvents) { + private void updateState(Map state, List tuples, + boolean newEvents) { for (Tuple tuple : tuples) { TaskStream taskStream = TaskStream.fromTuple(tuple); WindowState curState = state.get(taskStream); @@ -250,9 +268,12 @@ private void updateStreamState(Map state) { if (curState == null) { streamState.put(taskStream, newState); } else { - WindowState updatedState = new WindowState(Math.max(newState.lastExpired, curState.lastExpired), - Math.max(newState.lastEvaluated, curState.lastEvaluated)); - LOG.debug("Update window state, taskStream {}, curState {}, newState {}", taskStream, curState, updatedState); + WindowState updatedState = new WindowState(Math.max(newState.lastExpired, + curState.lastExpired), + Math.max(newState.lastEvaluated, + curState.lastEvaluated)); + LOG.debug("Update window state, taskStream {}, curState {}, newState {}", + taskStream, curState, updatedState); streamState.put(taskStream, updatedState); } } @@ -280,9 +301,11 @@ private long getMsgId(Tuple input) { return input.getLongByField(msgIdFieldName); } - private KeyValueState getWindowState(Map topoConf, TopologyContext context) { + private KeyValueState getWindowState(Map topoConf, + TopologyContext context) { String namespace = context.getThisComponentId() + "-" + context.getThisTaskId() + "-window"; - return (KeyValueState) StateFactory.getState(namespace, topoConf, context); + return (KeyValueState) StateFactory.getState(namespace, topoConf, + context); } static class WindowState { diff --git a/storm-client/src/jvm/org/apache/storm/topology/TopologyBuilder.java b/storm-client/src/jvm/org/apache/storm/topology/TopologyBuilder.java index 6f3ca964967..83fa9321ee2 100644 --- a/storm-client/src/jvm/org/apache/storm/topology/TopologyBuilder.java +++ b/storm-client/src/jvm/org/apache/storm/topology/TopologyBuilder.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -55,35 +61,46 @@ import org.apache.storm.windowing.TupleWindow; /** - * TopologyBuilder exposes the Java API for specifying a topology for Storm to execute. Topologies are Thrift structures in the end, but - * since the Thrift API is so verbose, TopologyBuilder greatly eases the process of creating topologies. The template for creating and + * TopologyBuilder exposes the Java API for specifying a topology for Storm to execute. Topologies + * are Thrift structures in the end, but + * since the Thrift API is so verbose, TopologyBuilder greatly eases the process of creating + * topologies. The template for creating and * submitting a topology looks something like: * *

    ```java TopologyBuilder builder = new TopologyBuilder(); * - *

    builder.setSpout("1", new TestWordSpout(true), 5); builder.setSpout("2", new TestWordSpout(true), 3); builder.setBolt("3", new - * TestWordCounter(), 3) .fieldsGrouping("1", new Fields("word")) .fieldsGrouping("2", new Fields("word")); builder.setBolt("4", new + *

    builder.setSpout("1", new TestWordSpout(true), 5); builder.setSpout("2", new + * TestWordSpout(true), 3); builder.setBolt("3", new + * TestWordCounter(), 3) .fieldsGrouping("1", new Fields("word")) .fieldsGrouping("2", new + * Fields("word")); builder.setBolt("4", new * TestGlobalCount()) .globalGrouping("1"); * *

    Map<String, Object> conf = new HashMap(); conf.put(Config.TOPOLOGY_WORKERS, 4); * *

    StormSubmitter.submitTopology("mytopology", conf, builder.createTopology()); ``` * - *

    Running the exact same topology in local mode (in process), and configuring it to log all tuples emitted, looks - * like the following. Note that it lets the topology run for 10 seconds before shutting down the local cluster. + *

    Running the exact same topology in local mode (in process), and configuring it to log all + * tuples emitted, looks + * like the following. Note that it lets the topology run for 10 seconds before shutting down the + * local cluster. * *

    ```java TopologyBuilder builder = new TopologyBuilder(); * - *

    builder.setSpout("1", new TestWordSpout(true), 5); builder.setSpout("2", new TestWordSpout(true), 3); builder.setBolt("3", new - * TestWordCounter(), 3) .fieldsGrouping("1", new Fields("word")) .fieldsGrouping("2", new Fields("word")); builder.setBolt("4", new + *

    builder.setSpout("1", new TestWordSpout(true), 5); builder.setSpout("2", new + * TestWordSpout(true), 3); builder.setBolt("3", new + * TestWordCounter(), 3) .fieldsGrouping("1", new Fields("word")) .fieldsGrouping("2", new + * Fields("word")); builder.setBolt("4", new * TestGlobalCount()) .globalGrouping("1"); * - *

    Map<String, Object> conf = new HashMap(); conf.put(Config.TOPOLOGY_WORKERS, 4); conf.put(Config.TOPOLOGY_DEBUG, true); + *

    Map<String, Object> conf = new HashMap(); conf.put(Config.TOPOLOGY_WORKERS, 4); + * conf.put(Config.TOPOLOGY_DEBUG, true); * - *

    try (LocalCluster cluster = new LocalCluster(); LocalTopology topo = cluster.submitTopology("mytopology", conf, + *

    try (LocalCluster cluster = new LocalCluster(); LocalTopology topo = + * cluster.submitTopology("mytopology", conf, * builder.createTopology());){ Utils.sleep(10000); } ``` * - *

    The pattern for `TopologyBuilder` is to map component ids to components using the setSpout and setBolt methods. Those methods return + *

    The pattern for `TopologyBuilder` is to map component ids to components using the setSpout and + * setBolt methods. Those methods return * objects that are then used to declare the inputs for that component. */ public class TopologyBuilder { @@ -113,10 +130,13 @@ public StormTopology createTopology() { ComponentCommon common = getComponentCommon(boltId, bolt); try { maybeAddCheckpointInputs(common); - boltSpecs.put(boltId, new Bolt(ComponentObject.serialized_java(Utils.javaSerialize(bolt)), common)); + boltSpecs.put(boltId, new Bolt(ComponentObject.serialized_java(Utils + .javaSerialize(bolt)), common)); } catch (RuntimeException wrapperCause) { - if (wrapperCause.getCause() != null && NotSerializableException.class.equals(wrapperCause.getCause().getClass())) { - throw new IllegalStateException("Bolt '" + boltId + "' contains a non-serializable field of type " + if (wrapperCause.getCause() != null && NotSerializableException.class + .equals(wrapperCause.getCause().getClass())) { + throw new IllegalStateException("Bolt '" + boltId + + "' contains a non-serializable field of type " + wrapperCause.getCause().getMessage() + ", " + "which was instantiated prior to topology creation. " + wrapperCause.getCause().getMessage() @@ -133,9 +153,11 @@ public StormTopology createTopology() { IRichSpout spout = spouts.get(spoutId); ComponentCommon common = getComponentCommon(spoutId, spout); try { - spoutSpecs.put(spoutId, new SpoutSpec(ComponentObject.serialized_java(Utils.javaSerialize(spout)), common)); + spoutSpecs.put(spoutId, new SpoutSpec(ComponentObject.serialized_java(Utils + .javaSerialize(spout)), common)); } catch (RuntimeException wrapperCause) { - if (wrapperCause.getCause() != null && NotSerializableException.class.equals(wrapperCause.getCause().getClass())) { + if (wrapperCause.getCause() != null && NotSerializableException.class + .equals(wrapperCause.getCause().getClass())) { throw new IllegalStateException( "Spout '" + spoutId + "' contains a non-serializable field of type " + wrapperCause.getCause().getMessage() @@ -167,7 +189,8 @@ public StormTopology createTopology() { /** * Define a new bolt in this topology with parallelism of just one thread. * - * @param id the id of this component. This id is referenced by other components that want to consume this bolt's outputs. + * @param id the id of this component. This id is referenced by other components that want to + * consume this bolt's outputs. * @param bolt the bolt * @return use the returned object to declare the inputs to this component * @@ -180,16 +203,19 @@ public BoltDeclarer setBolt(String id, IRichBolt bolt) throws IllegalArgumentExc /** * Define a new bolt in this topology with the specified amount of parallelism. * - * @param id the id of this component. This id is referenced by other components that want to consume this bolt's + * @param id the id of this component. This id is referenced by other components that want to + * consume this bolt's * outputs. * @param bolt the bolt - * @param parallelismHint the number of tasks that should be assigned to execute this bolt. Each task will run on a thread in a process + * @param parallelismHint the number of tasks that should be assigned to execute this bolt. Each + * task will run on a thread in a process * somewhere around the cluster. * @return use the returned object to declare the inputs to this component * * @throws IllegalArgumentException if {@code parallelism_hint} is not positive */ - public BoltDeclarer setBolt(String id, IRichBolt bolt, Number parallelismHint) throws IllegalArgumentException { + public BoltDeclarer setBolt(String id, IRichBolt bolt, + Number parallelismHint) throws IllegalArgumentException { validateUnusedId(id); initCommon(id, bolt, parallelismHint); bolts.put(id, bolt); @@ -197,11 +223,14 @@ public BoltDeclarer setBolt(String id, IRichBolt bolt, Number parallelismHint) t } /** - * Define a new bolt in this topology. This defines a basic bolt, which is a simpler to use but more restricted kind of bolt. Basic - * bolts are intended for non-aggregation processing and automate the anchoring/acking process to achieve proper reliability in the + * Define a new bolt in this topology. This defines a basic bolt, which is a simpler to use but + * more restricted kind of bolt. Basic + * bolts are intended for non-aggregation processing and automate the anchoring/acking process + * to achieve proper reliability in the * topology. * - * @param id the id of this component. This id is referenced by other components that want to consume this bolt's outputs. + * @param id the id of this component. This id is referenced by other components that want to + * consume this bolt's outputs. * @param bolt the basic bolt * @return use the returned object to declare the inputs to this component * @@ -212,28 +241,36 @@ public BoltDeclarer setBolt(String id, IBasicBolt bolt) throws IllegalArgumentEx } /** - * Define a new bolt in this topology. This defines a basic bolt, which is a simpler to use but more restricted kind of bolt. Basic - * bolts are intended for non-aggregation processing and automate the anchoring/acking process to achieve proper reliability in the + * Define a new bolt in this topology. This defines a basic bolt, which is a simpler to use but + * more restricted kind of bolt. Basic + * bolts are intended for non-aggregation processing and automate the anchoring/acking process + * to achieve proper reliability in the * topology. * - * @param id the id of this component. This id is referenced by other components that want to consume this bolt's + * @param id the id of this component. This id is referenced by other components that want to + * consume this bolt's * outputs. * @param bolt the basic bolt - * @param parallelismHint the number of tasks that should be assigned to execute this bolt. Each task will run on a thread in a process + * @param parallelismHint the number of tasks that should be assigned to execute this bolt. Each + * task will run on a thread in a process * somewhere around the cluster. * @return use the returned object to declare the inputs to this component * * @throws IllegalArgumentException if {@code parallelism_hint} is not positive */ - public BoltDeclarer setBolt(String id, IBasicBolt bolt, Number parallelismHint) throws IllegalArgumentException { + public BoltDeclarer setBolt(String id, IBasicBolt bolt, + Number parallelismHint) throws IllegalArgumentException { return setBolt(id, new BasicBoltExecutor(bolt), parallelismHint); } /** - * Define a new bolt in this topology. This defines a windowed bolt, intended for windowing operations. The {@link - * IWindowedBolt#execute(TupleWindow)} method is triggered for each window interval with the list of current events in the window. + * Define a new bolt in this topology. This defines a windowed bolt, intended for windowing + * operations. The {@link + * IWindowedBolt#execute(TupleWindow)} method is triggered for each window interval with the + * list of current events in the window. * - * @param id the id of this component. This id is referenced by other components that want to consume this bolt's outputs. + * @param id the id of this component. This id is referenced by other components that want to + * consume this bolt's outputs. * @param bolt the windowed bolt * @return use the returned object to declare the inputs to this component * @@ -244,98 +281,127 @@ public BoltDeclarer setBolt(String id, IWindowedBolt bolt) throws IllegalArgumen } /** - * Define a new bolt in this topology. This defines a windowed bolt, intended for windowing operations. The {@link - * IWindowedBolt#execute(TupleWindow)} method is triggered for each window interval with the list of current events in the window. + * Define a new bolt in this topology. This defines a windowed bolt, intended for windowing + * operations. The {@link + * IWindowedBolt#execute(TupleWindow)} method is triggered for each window interval with the + * list of current events in the window. * - * @param id the id of this component. This id is referenced by other components that want to consume this bolt's + * @param id the id of this component. This id is referenced by other components that want to + * consume this bolt's * outputs. * @param bolt the windowed bolt - * @param parallelismHint the number of tasks that should be assigned to execute this bolt. Each task will run on a thread in a process + * @param parallelismHint the number of tasks that should be assigned to execute this bolt. Each + * task will run on a thread in a process * somwehere around the cluster. * @return use the returned object to declare the inputs to this component * * @throws IllegalArgumentException if {@code parallelism_hint} is not positive */ - public BoltDeclarer setBolt(String id, IWindowedBolt bolt, Number parallelismHint) throws IllegalArgumentException { + public BoltDeclarer setBolt(String id, IWindowedBolt bolt, + Number parallelismHint) throws IllegalArgumentException { return setBolt(id, new WindowedBoltExecutor(bolt), parallelismHint); } /** - * Define a new bolt in this topology. This defines a stateful bolt, that requires its state (of computation) to be saved. When this - * bolt is initialized, the {@link IStatefulBolt#initState(State)} method is invoked after {@link IStatefulBolt#prepare(Map, - * TopologyContext, OutputCollector)} but before {@link IStatefulBolt#execute(Tuple)} with its previously saved state. - *

    - * The framework provides at-least once guarantee for the state updates. Bolts (both stateful and non-stateful) in a stateful topology + * Define a new bolt in this topology. This defines a stateful bolt, that requires its state (of + * computation) to be saved. When this + * bolt is initialized, the {@link IStatefulBolt#initState(State)} method is invoked after + * {@link IStatefulBolt#prepare(Map, + * TopologyContext, OutputCollector)} but before {@link IStatefulBolt#execute(Tuple)} with its + * previously saved state. + * + *

    The framework provides at-least once guarantee for the state updates. Bolts (both stateful + * and non-stateful) in a stateful topology * are expected to anchor the tuples while emitting and ack the input tuples once its processed. *

    * - * @param id the id of this component. This id is referenced by other components that want to consume this bolt's outputs. + * @param id the id of this component. This id is referenced by other components that want to + * consume this bolt's outputs. * @param bolt the stateful bolt * @return use the returned object to declare the inputs to this component * * @throws IllegalArgumentException if {@code parallelism_hint} is not positive */ - public BoltDeclarer setBolt(String id, IStatefulBolt bolt) throws IllegalArgumentException { + public BoltDeclarer setBolt(String id, + IStatefulBolt bolt) throws IllegalArgumentException { return setBolt(id, bolt, null); } /** - * Define a new bolt in this topology. This defines a stateful bolt, that requires its state (of computation) to be saved. When this - * bolt is initialized, the {@link IStatefulBolt#initState(State)} method is invoked after {@link IStatefulBolt#prepare(Map, - * TopologyContext, OutputCollector)} but before {@link IStatefulBolt#execute(Tuple)} with its previously saved state. - *

    - * The framework provides at-least once guarantee for the state updates. Bolts (both stateful and non-stateful) in a stateful topology + * Define a new bolt in this topology. This defines a stateful bolt, that requires its state (of + * computation) to be saved. When this + * bolt is initialized, the {@link IStatefulBolt#initState(State)} method is invoked after + * {@link IStatefulBolt#prepare(Map, + * TopologyContext, OutputCollector)} but before {@link IStatefulBolt#execute(Tuple)} with its + * previously saved state. + * + *

    The framework provides at-least once guarantee for the state updates. Bolts (both stateful + * and non-stateful) in a stateful topology * are expected to anchor the tuples while emitting and ack the input tuples once its processed. *

    * - * @param id the id of this component. This id is referenced by other components that want to consume this bolt's + * @param id the id of this component. This id is referenced by other components that want to + * consume this bolt's * outputs. * @param bolt the stateful bolt - * @param parallelismHint the number of tasks that should be assigned to execute this bolt. Each task will run on a thread in a process + * @param parallelismHint the number of tasks that should be assigned to execute this bolt. Each + * task will run on a thread in a process * somwehere around the cluster. * @return use the returned object to declare the inputs to this component * * @throws IllegalArgumentException if {@code parallelism_hint} is not positive */ - public BoltDeclarer setBolt(String id, IStatefulBolt bolt, Number parallelismHint) throws + public BoltDeclarer setBolt(String id, IStatefulBolt bolt, + Number parallelismHint) throws IllegalArgumentException { hasStatefulBolt = true; return setBolt(id, new StatefulBoltExecutor(bolt), parallelismHint); } /** - * Define a new bolt in this topology. This defines a stateful windowed bolt, intended for stateful windowing operations. The {@link - * IStatefulWindowedBolt#execute(TupleWindow)} method is triggered for each window interval with the list of current events in the - * window. During initialization of this bolt {@link IStatefulWindowedBolt#initState(State)} is invoked with its previously saved + * Define a new bolt in this topology. This defines a stateful windowed bolt, intended for + * stateful windowing operations. The {@link + * IStatefulWindowedBolt#execute(TupleWindow)} method is triggered for each window interval with + * the list of current events in the + * window. During initialization of this bolt {@link IStatefulWindowedBolt#initState(State)} is + * invoked with its previously saved * state. * - * @param id the id of this component. This id is referenced by other components that want to consume this bolt's outputs. + * @param id the id of this component. This id is referenced by other components that want to + * consume this bolt's outputs. * @param bolt the stateful windowed bolt * @param the type of the state (e.g. {@link org.apache.storm.state.KeyValueState}) * @return use the returned object to declare the inputs to this component * * @throws IllegalArgumentException if {@code parallelism_hint} is not positive */ - public BoltDeclarer setBolt(String id, IStatefulWindowedBolt bolt) throws IllegalArgumentException { + public BoltDeclarer setBolt(String id, + IStatefulWindowedBolt bolt) throws IllegalArgumentException { return setBolt(id, bolt, null); } /** - * Define a new bolt in this topology. This defines a stateful windowed bolt, intended for stateful windowing operations. The {@link - * IStatefulWindowedBolt#execute(TupleWindow)} method is triggered for each window interval with the list of current events in the - * window. During initialization of this bolt {@link IStatefulWindowedBolt#initState(State)} is invoked with its previously saved + * Define a new bolt in this topology. This defines a stateful windowed bolt, intended for + * stateful windowing operations. The {@link + * IStatefulWindowedBolt#execute(TupleWindow)} method is triggered for each window interval with + * the list of current events in the + * window. During initialization of this bolt {@link IStatefulWindowedBolt#initState(State)} is + * invoked with its previously saved * state. * - * @param id the id of this component. This id is referenced by other components that want to consume this bolt's + * @param id the id of this component. This id is referenced by other components that want to + * consume this bolt's * outputs. * @param bolt the stateful windowed bolt - * @param parallelismHint the number of tasks that should be assigned to execute this bolt. Each task will run on a thread in a process + * @param parallelismHint the number of tasks that should be assigned to execute this bolt. Each + * task will run on a thread in a process * somewhere around the cluster. * @return use the returned object to declare the inputs to this component * * @throws IllegalArgumentException if {@code parallelism_hint} is not positive */ - public BoltDeclarer setBolt(String id, IStatefulWindowedBolt bolt, Number parallelismHint) throws + public BoltDeclarer setBolt(String id, IStatefulWindowedBolt bolt, + Number parallelismHint) throws IllegalArgumentException { hasStatefulBolt = true; IStatefulBolt executor; @@ -348,79 +414,98 @@ public BoltDeclarer setBolt(String id, IStatefulWindowedBolt biConsumer, String... fields) throws + public BoltDeclarer setBolt(String id, SerializableBiConsumer biConsumer, String... fields) throws IllegalArgumentException { return setBolt(id, biConsumer, null, fields); } /** - * Define a new bolt in this topology. This defines a lambda basic bolt, which is a simpler to use but more restricted kind of bolt. - * Basic bolts are intended for non-aggregation processing and automate the anchoring/acking process to achieve proper reliability in + * Define a new bolt in this topology. This defines a lambda basic bolt, which is a simpler to + * use but more restricted kind of bolt. + * Basic bolts are intended for non-aggregation processing and automate the anchoring/acking + * process to achieve proper reliability in * the topology. * - * @param id the id of this component. This id is referenced by other components that want to consume this bolt's + * @param id the id of this component. This id is referenced by other components that want to + * consume this bolt's * outputs. * @param biConsumer lambda expression that implements tuple processing for this bolt * @param fields fields for tuple that should be emitted to downstream bolts - * @param parallelismHint the number of tasks that should be assigned to execute this bolt. Each task will run on a thread in a process + * @param parallelismHint the number of tasks that should be assigned to execute this bolt. Each + * task will run on a thread in a process * somewhere around the cluster. * @return use the returned object to declare the inputs to this component * * @throws IllegalArgumentException if {@code parallelism_hint} is not positive */ - public BoltDeclarer setBolt(String id, SerializableBiConsumer biConsumer, Number parallelismHint, + public BoltDeclarer setBolt(String id, SerializableBiConsumer biConsumer, Number parallelismHint, String... fields) throws IllegalArgumentException { return setBolt(id, new LambdaBiConsumerBolt(biConsumer, fields), parallelismHint); } /** - * Define a new bolt in this topology. This defines a lambda basic bolt, which is a simpler to use but more restricted kind of bolt. - * Basic bolts are intended for non-aggregation processing and automate the anchoring/acking process to achieve proper reliability in + * Define a new bolt in this topology. This defines a lambda basic bolt, which is a simpler to + * use but more restricted kind of bolt. + * Basic bolts are intended for non-aggregation processing and automate the anchoring/acking + * process to achieve proper reliability in * the topology. * - * @param id the id of this component. This id is referenced by other components that want to consume this bolt's outputs. + * @param id the id of this component. This id is referenced by other components that want to + * consume this bolt's outputs. * @param consumer lambda expression that implements tuple processing for this bolt * @return use the returned object to declare the inputs to this component * * @throws IllegalArgumentException if {@code parallelism_hint} is not positive */ - public BoltDeclarer setBolt(String id, SerializableConsumer consumer) throws IllegalArgumentException { + public BoltDeclarer setBolt(String id, + SerializableConsumer consumer) throws IllegalArgumentException { return setBolt(id, consumer, null); } /** - * Define a new bolt in this topology. This defines a lambda basic bolt, which is a simpler to use but more restricted kind of bolt. - * Basic bolts are intended for non-aggregation processing and automate the anchoring/acking process to achieve proper reliability in + * Define a new bolt in this topology. This defines a lambda basic bolt, which is a simpler to + * use but more restricted kind of bolt. + * Basic bolts are intended for non-aggregation processing and automate the anchoring/acking + * process to achieve proper reliability in * the topology. * - * @param id the id of this component. This id is referenced by other components that want to consume this bolt's + * @param id the id of this component. This id is referenced by other components that want to + * consume this bolt's * outputs. * @param consumer lambda expression that implements tuple processing for this bolt - * @param parallelismHint the number of tasks that should be assigned to execute this bolt. Each task will run on a thread in a process + * @param parallelismHint the number of tasks that should be assigned to execute this bolt. Each + * task will run on a thread in a process * somewhere around the cluster. * @return use the returned object to declare the inputs to this component * * @throws IllegalArgumentException if {@code parallelism_hint} is not positive */ - public BoltDeclarer setBolt(String id, SerializableConsumer consumer, Number parallelismHint) throws IllegalArgumentException { + public BoltDeclarer setBolt(String id, SerializableConsumer consumer, + Number parallelismHint) throws IllegalArgumentException { return setBolt(id, new LambdaConsumerBolt(consumer), parallelismHint); } /** * Define a new spout in this topology. * - * @param id the id of this component. This id is referenced by other components that want to consume this spout's outputs. + * @param id the id of this component. This id is referenced by other components that want to + * consume this spout's outputs. * @param spout the spout * @throws IllegalArgumentException if {@code parallelism_hint} is not positive */ @@ -429,17 +514,21 @@ public SpoutDeclarer setSpout(String id, IRichSpout spout) throws IllegalArgumen } /** - * Define a new spout in this topology with the specified parallelism. If the spout declares itself as non-distributed, the + * Define a new spout in this topology with the specified parallelism. If the spout declares + * itself as non-distributed, the * parallelism_hint will be ignored and only one task will be allocated to this component. * - * @param id the id of this component. This id is referenced by other components that want to consume this spout's + * @param id the id of this component. This id is referenced by other components that want to + * consume this spout's * outputs. - * @param parallelismHint the number of tasks that should be assigned to execute this spout. Each task will run on a thread in a + * @param parallelismHint the number of tasks that should be assigned to execute this spout. + * Each task will run on a thread in a * process somewhere around the cluster. * @param spout the spout * @throws IllegalArgumentException if {@code parallelism_hint} is not positive */ - public SpoutDeclarer setSpout(String id, IRichSpout spout, Number parallelismHint) throws IllegalArgumentException { + public SpoutDeclarer setSpout(String id, IRichSpout spout, + Number parallelismHint) throws IllegalArgumentException { validateUnusedId(id); initCommon(id, spout, parallelismHint); spouts.put(id, spout); @@ -449,26 +538,32 @@ public SpoutDeclarer setSpout(String id, IRichSpout spout, Number parallelismHin /** * Define a new spout in this topology. * - * @param id the id of this component. This id is referenced by other components that want to consume this spout's outputs. + * @param id the id of this component. This id is referenced by other components that want to + * consume this spout's outputs. * @param supplier lambda expression that implements tuple generating for this spout * @throws IllegalArgumentException if {@code parallelism_hint} is not positive */ - public SpoutDeclarer setSpout(String id, SerializableSupplier supplier) throws IllegalArgumentException { + public SpoutDeclarer setSpout(String id, + SerializableSupplier supplier) throws IllegalArgumentException { return setSpout(id, supplier, null); } /** - * Define a new spout in this topology with the specified parallelism. If the spout declares itself as non-distributed, the + * Define a new spout in this topology with the specified parallelism. If the spout declares + * itself as non-distributed, the * parallelism_hint will be ignored and only one task will be allocated to this component. * - * @param id the id of this component. This id is referenced by other components that want to consume this spout's + * @param id the id of this component. This id is referenced by other components that want to + * consume this spout's * outputs. - * @param parallelismHint the number of tasks that should be assigned to execute this spout. Each task will run on a thread in a + * @param parallelismHint the number of tasks that should be assigned to execute this spout. + * Each task will run on a thread in a * process somewhere around the cluster. * @param supplier lambda expression that implements tuple generating for this spout * @throws IllegalArgumentException if {@code parallelism_hint} is not positive */ - public SpoutDeclarer setSpout(String id, SerializableSupplier supplier, Number parallelismHint) throws IllegalArgumentException { + public SpoutDeclarer setSpout(String id, SerializableSupplier supplier, + Number parallelismHint) throws IllegalArgumentException { return setSpout(id, new LambdaSpout(supplier), parallelismHint); } @@ -493,12 +588,14 @@ private void validateUnusedId(String id) { throw new IllegalArgumentException("Spout has already been declared for id " + id); } if (stateSpouts.containsKey(id)) { - throw new IllegalArgumentException("State spout has already been declared for id " + id); + throw new IllegalArgumentException("State spout has already been declared for id " + + id); } } /** - * If the topology has at least one stateful bolt add a {@link CheckpointSpout} component to the topology. + * If the topology has at least one stateful bolt add a {@link CheckpointSpout} component to the + * topology. */ private void maybeAddCheckpointSpout() { if (hasStatefulBolt) { @@ -513,7 +610,8 @@ private void maybeAddCheckpointInputs(ComponentCommon common) { } /** - * If the topology has at least one stateful bolt all the non-stateful bolts are wrapped in {@link CheckpointTupleForwarder} so that the + * If the topology has at least one stateful bolt all the non-stateful bolts are wrapped in + * {@link CheckpointTupleForwarder} so that the * checkpoint tuples can flow through the topology. */ private IRichBolt maybeAddCheckpointTupleForwarder(IRichBolt bolt) { @@ -524,7 +622,8 @@ private IRichBolt maybeAddCheckpointTupleForwarder(IRichBolt bolt) { } /** - * For bolts that has incoming streams from spouts (the root bolts), add checkpoint stream from checkpoint spout to its input. For other + * For bolts that has incoming streams from spouts (the root bolts), add checkpoint stream from + * checkpoint spout to its input. For other * bolts, add checkpoint stream from the previous bolt to its input. */ private void addCheckPointInputs(ComponentCommon component) { @@ -532,7 +631,8 @@ private void addCheckPointInputs(ComponentCommon component) { for (GlobalStreamId inputStream : component.get_inputs().keySet()) { String sourceId = inputStream.get_componentId(); if (spouts.containsKey(sourceId)) { - checkPointInputs.add(new GlobalStreamId(CHECKPOINT_COMPONENT_ID, CHECKPOINT_STREAM_ID)); + checkPointInputs.add(new GlobalStreamId(CHECKPOINT_COMPONENT_ID, + CHECKPOINT_STREAM_ID)); } else { checkPointInputs.add(new GlobalStreamId(sourceId, CHECKPOINT_STREAM_ID)); } @@ -550,7 +650,8 @@ private ComponentCommon getComponentCommon(String id, IComponent component) { return ret; } - private void initCommon(String id, IComponent component, Number parallelism) throws IllegalArgumentException { + private void initCommon(String id, IComponent component, + Number parallelism) throws IllegalArgumentException { ComponentCommon common = new ComponentCommon(); common.set_inputs(new HashMap()); if (parallelism != null) { @@ -579,7 +680,8 @@ public ConfigGetter(String id) { public T addConfigurations(Map conf) { if (conf != null) { if (conf.containsKey(Config.TOPOLOGY_KRYO_REGISTER)) { - throw new IllegalArgumentException("Cannot set serializations for a component using fluent API"); + throw new IllegalArgumentException("Cannot set serializations for a component " + + "using fluent API"); } if (!conf.isEmpty()) { String currConf = commons.get(id).get_json_conf(); @@ -590,7 +692,7 @@ public T addConfigurations(Map conf) { } /** - * return the current component configuration. + * Return the current component configuration. * * @return the current configuration. */ @@ -605,7 +707,9 @@ public T addResources(Map resources) { String currConf = commons.get(id).get_json_conf(); Map conf = parseJson(currConf); Map currentResources = - (Map) conf.computeIfAbsent(Config.TOPOLOGY_COMPONENT_RESOURCES_MAP, (k) -> new HashMap<>()); + (Map) conf + .computeIfAbsent(Config.TOPOLOGY_COMPONENT_RESOURCES_MAP, + (k) -> new HashMap<>()); currentResources.putAll(resources); commons.get(id).set_json_conf(JSONValue.toJSONString(conf)); } @@ -629,7 +733,8 @@ public T addResource(String resourceName, Number resourceValue) { public T addSharedMemory(SharedMemory request) { SharedMemory found = sharedMemory.get(request.get_name()); if (found != null && !found.equals(request)) { - throw new IllegalArgumentException("Cannot have multiple different shared memory regions with the same name"); + throw new IllegalArgumentException("Cannot have multiple different shared memory " + + "regions with the same name"); } sharedMemory.put(request.get_name(), request); Set mems = componentToSharedMemory.computeIfAbsent(id, (k) -> new HashSet<>()); @@ -748,8 +853,10 @@ public BoltDeclarer customGrouping(String componentId, CustomStreamGrouping grou } @Override - public BoltDeclarer customGrouping(String componentId, String streamId, CustomStreamGrouping grouping) { - return grouping(componentId, streamId, Grouping.custom_serialized(Utils.javaSerialize(grouping))); + public BoltDeclarer customGrouping(String componentId, String streamId, + CustomStreamGrouping grouping) { + return grouping(componentId, streamId, Grouping.custom_serialized(Utils + .javaSerialize(grouping))); } } } diff --git a/storm-client/src/jvm/org/apache/storm/topology/TupleFieldTimestampExtractor.java b/storm-client/src/jvm/org/apache/storm/topology/TupleFieldTimestampExtractor.java index ac5cb7f1a33..7b0a95d28bf 100644 --- a/storm-client/src/jvm/org/apache/storm/topology/TupleFieldTimestampExtractor.java +++ b/storm-client/src/jvm/org/apache/storm/topology/TupleFieldTimestampExtractor.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/topology/WindowedBoltExecutor.java b/storm-client/src/jvm/org/apache/storm/topology/WindowedBoltExecutor.java index 7f7f2321cd5..5d7ec1d392a 100644 --- a/storm-client/src/jvm/org/apache/storm/topology/WindowedBoltExecutor.java +++ b/storm-client/src/jvm/org/apache/storm/topology/WindowedBoltExecutor.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -63,7 +69,8 @@ */ public class WindowedBoltExecutor implements IRichBolt { /** - * Name of the field carrying a late tuple on the late tuple stream. The value is a {@link DetachedTuple}, + * Name of the field carrying a late tuple on the late tuple stream. The value is a {@link + * DetachedTuple}, * a serializable copy of the original tuple detached from the topology context. */ public static final String LATE_TUPLE_FIELD = "late_tuple"; @@ -90,7 +97,8 @@ public WindowedBoltExecutor(IWindowedBolt bolt) { protected int getTopologyTimeoutMillis(Map topoConf) { if (topoConf.get(Config.TOPOLOGY_ENABLE_MESSAGE_TIMEOUTS) != null) { - boolean timeOutsEnabled = (boolean) topoConf.get(Config.TOPOLOGY_ENABLE_MESSAGE_TIMEOUTS); + boolean timeOutsEnabled = (boolean) topoConf + .get(Config.TOPOLOGY_ENABLE_MESSAGE_TIMEOUTS); if (!timeOutsEnabled) { return Integer.MAX_VALUE; } @@ -112,20 +120,24 @@ private int getMaxSpoutPending(Map topoConf) { private void ensureDurationLessThanTimeout(int duration, int timeout) { if (duration > timeout) { - throw new IllegalArgumentException("Window duration (length + sliding interval) value " + duration - + " is more than " + Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS + " value " + timeout); + throw new IllegalArgumentException("Window duration (length + sliding interval) value " + + duration + + " is more than " + Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS + " value " + + timeout); } } private void ensureCountLessThanMaxPending(int count, int maxPending) { if (count > maxPending) { - throw new IllegalArgumentException("Window count (length + sliding interval) value " + count + throw new IllegalArgumentException("Window count (length + sliding interval) value " + + count + " is more than " + Config.TOPOLOGY_MAX_SPOUT_PENDING + " value " + maxPending); } } - protected void validate(Map topoConf, Count windowLengthCount, Duration windowLengthDuration, + protected void validate(Map topoConf, Count windowLengthCount, + Duration windowLengthDuration, Count slidingIntervalCount, Duration slidingIntervalDuration) { int topologyTimeout = getTopologyTimeoutMillis(topoConf); @@ -135,7 +147,8 @@ protected void validate(Map topoConf, Count windowLengthCount, D } if (windowLengthDuration != null && slidingIntervalDuration != null) { - ensureDurationLessThanTimeout(windowLengthDuration.value + slidingIntervalDuration.value, topologyTimeout); + ensureDurationLessThanTimeout(windowLengthDuration.value + + slidingIntervalDuration.value, topologyTimeout); } else if (windowLengthDuration != null) { ensureDurationLessThanTimeout(windowLengthDuration.value, topologyTimeout); } else if (slidingIntervalDuration != null) { @@ -143,7 +156,8 @@ protected void validate(Map topoConf, Count windowLengthCount, D } if (windowLengthCount != null && slidingIntervalCount != null) { - ensureCountLessThanMaxPending(windowLengthCount.value + slidingIntervalCount.value, maxSpoutPending); + ensureCountLessThanMaxPending(windowLengthCount.value + slidingIntervalCount.value, + maxSpoutPending); } else if (windowLengthCount != null) { ensureCountLessThanMaxPending(windowLengthCount.value, maxSpoutPending); } else if (slidingIntervalCount != null) { @@ -151,7 +165,8 @@ protected void validate(Map topoConf, Count windowLengthCount, D } } - private WindowManager initWindowManager(WindowLifecycleListener lifecycleListener, Map topoConf, + private WindowManager initWindowManager(WindowLifecycleListener lifecycleListener, + Map topoConf, TopologyContext context, Collection> queue, boolean stateful) { WindowManager manager = stateful @@ -163,7 +178,8 @@ private WindowManager initWindowManager(WindowLifecycleListener li Count slidingIntervalCount = null; // window length if (topoConf.containsKey(Config.TOPOLOGY_BOLTS_WINDOW_LENGTH_COUNT)) { - windowLengthCount = new Count(((Number) topoConf.get(Config.TOPOLOGY_BOLTS_WINDOW_LENGTH_COUNT)).intValue()); + windowLengthCount = new Count(((Number) topoConf + .get(Config.TOPOLOGY_BOLTS_WINDOW_LENGTH_COUNT)).intValue()); } else if (topoConf.containsKey(Config.TOPOLOGY_BOLTS_WINDOW_LENGTH_DURATION_MS)) { windowLengthDuration = new Duration( ((Number) topoConf.get(Config.TOPOLOGY_BOLTS_WINDOW_LENGTH_DURATION_MS)).intValue(), @@ -171,10 +187,13 @@ private WindowManager initWindowManager(WindowLifecycleListener li } // sliding interval if (topoConf.containsKey(Config.TOPOLOGY_BOLTS_SLIDING_INTERVAL_COUNT)) { - slidingIntervalCount = new Count(((Number) topoConf.get(Config.TOPOLOGY_BOLTS_SLIDING_INTERVAL_COUNT)).intValue()); + slidingIntervalCount = new Count(((Number) topoConf + .get(Config.TOPOLOGY_BOLTS_SLIDING_INTERVAL_COUNT)).intValue()); } else if (topoConf.containsKey(Config.TOPOLOGY_BOLTS_SLIDING_INTERVAL_DURATION_MS)) { slidingIntervalDuration = - new Duration(((Number) topoConf.get(Config.TOPOLOGY_BOLTS_SLIDING_INTERVAL_DURATION_MS)).intValue(), TimeUnit.MILLISECONDS); + new Duration(((Number) topoConf + .get(Config.TOPOLOGY_BOLTS_SLIDING_INTERVAL_DURATION_MS)) + .intValue(), TimeUnit.MILLISECONDS); } else { // default is a sliding window of count 1 slidingIntervalCount = new Count(1); @@ -186,19 +205,22 @@ private WindowManager initWindowManager(WindowLifecycleListener li if (lateTupleStream != null) { if (!context.getThisStreams().contains(lateTupleStream)) { throw new IllegalArgumentException( - "Stream for late tuples must be defined with the builder method withLateTupleStream"); + "Stream for late tuples must be defined with the builder method " + + "withLateTupleStream"); } } // max lag if (topoConf.containsKey(Config.TOPOLOGY_BOLTS_TUPLE_TIMESTAMP_MAX_LAG_MS)) { - maxLagMs = ((Number) topoConf.get(Config.TOPOLOGY_BOLTS_TUPLE_TIMESTAMP_MAX_LAG_MS)).intValue(); + maxLagMs = ((Number) topoConf.get(Config.TOPOLOGY_BOLTS_TUPLE_TIMESTAMP_MAX_LAG_MS)) + .intValue(); } else { maxLagMs = DEFAULT_MAX_LAG_MS; } // watermark interval int watermarkInterval; if (topoConf.containsKey(Config.TOPOLOGY_BOLTS_WATERMARK_EVENT_INTERVAL_MS)) { - watermarkInterval = ((Number) topoConf.get(Config.TOPOLOGY_BOLTS_WATERMARK_EVENT_INTERVAL_MS)).intValue(); + watermarkInterval = ((Number) topoConf + .get(Config.TOPOLOGY_BOLTS_WATERMARK_EVENT_INTERVAL_MS)).intValue(); } else { watermarkInterval = DEFAULT_WATERMARK_EVENT_INTERVAL_MS; } @@ -206,7 +228,8 @@ private WindowManager initWindowManager(WindowLifecycleListener li maxLagMs, getComponentStreams(context)); } else { if (topoConf.containsKey(Config.TOPOLOGY_BOLTS_LATE_TUPLE_STREAM)) { - throw new IllegalArgumentException("Late tuple stream can be defined only when specifying a timestamp field"); + throw new IllegalArgumentException("Late tuple stream can be defined only when " + + "specifying a timestamp field"); } } // validate @@ -254,24 +277,30 @@ private boolean isTupleTs() { return timestampExtractor != null; } - private TriggerPolicy getTriggerPolicy(Count slidingIntervalCount, Duration slidingIntervalDuration, + private TriggerPolicy getTriggerPolicy(Count slidingIntervalCount, + Duration slidingIntervalDuration, WindowManager manager, EvictionPolicy evictionPolicy) { if (slidingIntervalCount != null) { if (isTupleTs()) { - return new WatermarkCountTriggerPolicy<>(slidingIntervalCount.value, manager, evictionPolicy, manager); + return new WatermarkCountTriggerPolicy<>(slidingIntervalCount.value, manager, + evictionPolicy, manager); } else { - return new CountTriggerPolicy<>(slidingIntervalCount.value, manager, evictionPolicy); + return new CountTriggerPolicy<>(slidingIntervalCount.value, manager, + evictionPolicy); } } else { if (isTupleTs()) { - return new WatermarkTimeTriggerPolicy<>(slidingIntervalDuration.value, manager, evictionPolicy, manager); + return new WatermarkTimeTriggerPolicy<>(slidingIntervalDuration.value, manager, + evictionPolicy, manager); } else { - return new TimeTriggerPolicy<>(slidingIntervalDuration.value, manager, evictionPolicy); + return new TimeTriggerPolicy<>(slidingIntervalDuration.value, manager, + evictionPolicy); } } } - private EvictionPolicy getEvictionPolicy(Count windowLengthCount, Duration windowLengthDuration) { + private EvictionPolicy getEvictionPolicy(Count windowLengthCount, + Duration windowLengthDuration) { if (windowLengthCount != null) { if (isTupleTs()) { return new WatermarkCountEvictionPolicy<>(windowLengthCount.value); @@ -288,12 +317,14 @@ private boolean isTupleTs() { } @Override - public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) { + public void prepare(Map topoConf, TopologyContext context, + OutputCollector collector) { doPrepare(topoConf, context, collector, new ConcurrentLinkedQueue<>(), false); } // NOTE: the queue has to be thread safe. - protected void doPrepare(Map topoConf, TopologyContext context, OutputCollector collector, + protected void doPrepare(Map topoConf, TopologyContext context, + OutputCollector collector, Collection> queue, boolean stateful) { Objects.requireNonNull(topoConf); Objects.requireNonNull(context); @@ -315,10 +346,13 @@ public void execute(Tuple input) { windowManager.add(input, ts); } else { if (lateTupleStream != null) { - // emit a detached copy: the original tuple references the topology context and cannot be serialized - windowedOutputCollector.emit(lateTupleStream, input, new Values(new DetachedTuple(input))); + // emit a detached copy: the original tuple references the topology context and + // cannot be serialized + windowedOutputCollector.emit(lateTupleStream, input, + new Values(new DetachedTuple(input))); } else { - LOG.info("Received a late tuple {} with ts {}. This will not be processed.", input, ts); + LOG.info("Received a late tuple {} with ts {}. This will not be processed.", + input, ts); } windowedOutputCollector.ack(input); } @@ -343,7 +377,8 @@ WindowManager getWindowManager() { @Override public void declareOutputFields(OutputFieldsDeclarer declarer) { - String lateTupleStream = (String) getComponentConfiguration().get(Config.TOPOLOGY_BOLTS_LATE_TUPLE_STREAM); + String lateTupleStream = (String) getComponentConfiguration() + .get(Config.TOPOLOGY_BOLTS_LATE_TUPLE_STREAM); if (lateTupleStream != null) { declarer.declareStream(lateTupleStream, new Fields(LATE_TUPLE_FIELD)); } @@ -352,7 +387,8 @@ public void declareOutputFields(OutputFieldsDeclarer declarer) { @Override public Map getComponentConfiguration() { - return bolt.getComponentConfiguration() != null ? bolt.getComponentConfiguration() : Collections.emptyMap(); + return bolt.getComponentConfiguration() != null ? bolt + .getComponentConfiguration() : Collections.emptyMap(); } protected WindowLifecycleListener newWindowLifecycleListener() { @@ -365,7 +401,8 @@ public void onExpiry(List tuples) { } @Override - public void onActivation(List tuples, List newTuples, List expiredTuples, Long timestamp) { + public void onActivation(List tuples, List newTuples, + List expiredTuples, Long timestamp) { windowedOutputCollector.setContext(tuples); boltExecute(tuples, newTuples, expiredTuples, timestamp); } @@ -373,14 +410,17 @@ public void onActivation(List tuples, List newTuples, List }; } - protected void boltExecute(List tuples, List newTuples, List expiredTuples, Long timestamp) { - bolt.execute(new TupleWindowImpl(tuples, newTuples, expiredTuples, getWindowStartTs(timestamp), timestamp)); + protected void boltExecute(List tuples, List newTuples, List expiredTuples, + Long timestamp) { + bolt.execute(new TupleWindowImpl(tuples, newTuples, expiredTuples, + getWindowStartTs(timestamp), timestamp)); } protected void boltExecute(Supplier> tuples, Supplier> newTuples, Supplier> expiredTuples, Long timestamp) { - bolt.execute(new TupleWindowIterImpl(tuples, newTuples, expiredTuples, getWindowStartTs(timestamp), timestamp)); + bolt.execute(new TupleWindowIterImpl(tuples, newTuples, expiredTuples, + getWindowStartTs(timestamp), timestamp)); } private Long getWindowStartTs(Long endTs) { @@ -392,7 +432,8 @@ private Long getWindowStartTs(Long endTs) { } /** - * Creates an {@link OutputCollector} wrapper that automatically anchors the tuples to inputTuples while emitting. + * Creates an {@link OutputCollector} wrapper that automatically anchors the tuples to + * inputTuples while emitting. */ private static class WindowedOutputCollector extends OutputCollector { private List inputTuples; diff --git a/storm-client/src/jvm/org/apache/storm/topology/base/BaseBasicBolt.java b/storm-client/src/jvm/org/apache/storm/topology/base/BaseBasicBolt.java index 19b3053aeca..530e5d6eb32 100644 --- a/storm-client/src/jvm/org/apache/storm/topology/base/BaseBasicBolt.java +++ b/storm-client/src/jvm/org/apache/storm/topology/base/BaseBasicBolt.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/topology/base/BaseBatchBolt.java b/storm-client/src/jvm/org/apache/storm/topology/base/BaseBatchBolt.java index 7939e3d2aba..b74058ee5ea 100644 --- a/storm-client/src/jvm/org/apache/storm/topology/base/BaseBatchBolt.java +++ b/storm-client/src/jvm/org/apache/storm/topology/base/BaseBatchBolt.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/topology/base/BaseComponent.java b/storm-client/src/jvm/org/apache/storm/topology/base/BaseComponent.java index ac2e078cc04..16f60f51118 100644 --- a/storm-client/src/jvm/org/apache/storm/topology/base/BaseComponent.java +++ b/storm-client/src/jvm/org/apache/storm/topology/base/BaseComponent.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/topology/base/BaseRichBolt.java b/storm-client/src/jvm/org/apache/storm/topology/base/BaseRichBolt.java index 6f25998e2cd..acfab24a8b3 100644 --- a/storm-client/src/jvm/org/apache/storm/topology/base/BaseRichBolt.java +++ b/storm-client/src/jvm/org/apache/storm/topology/base/BaseRichBolt.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/topology/base/BaseRichSpout.java b/storm-client/src/jvm/org/apache/storm/topology/base/BaseRichSpout.java index 80646869758..70d614cfc6f 100644 --- a/storm-client/src/jvm/org/apache/storm/topology/base/BaseRichSpout.java +++ b/storm-client/src/jvm/org/apache/storm/topology/base/BaseRichSpout.java @@ -1,12 +1,17 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ /* diff --git a/storm-client/src/jvm/org/apache/storm/topology/base/BaseStatefulBolt.java b/storm-client/src/jvm/org/apache/storm/topology/base/BaseStatefulBolt.java index 721cc99adbf..862d4c22dd4 100644 --- a/storm-client/src/jvm/org/apache/storm/topology/base/BaseStatefulBolt.java +++ b/storm-client/src/jvm/org/apache/storm/topology/base/BaseStatefulBolt.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -22,7 +28,8 @@ public abstract class BaseStatefulBolt implements IStatefulBolt { @Override - public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) { + public void prepare(Map topoConf, TopologyContext context, + OutputCollector collector) { // NOOP } diff --git a/storm-client/src/jvm/org/apache/storm/topology/base/BaseStatefulWindowedBolt.java b/storm-client/src/jvm/org/apache/storm/topology/base/BaseStatefulWindowedBolt.java index 063555bddb7..aba5246bae0 100644 --- a/storm-client/src/jvm/org/apache/storm/topology/base/BaseStatefulWindowedBolt.java +++ b/storm-client/src/jvm/org/apache/storm/topology/base/BaseStatefulWindowedBolt.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -142,7 +148,8 @@ public BaseStatefulWindowedBolt withWatermarkInterval(Duration interval) { } /** - * Specify the name of the field in the tuple that holds the message id. This is used to track the windowing boundaries and + * Specify the name of the field in the tuple that holds the message id. This is used to track + * the windowing boundaries and * re-evaluating the windowing operation during recovery of IStatefulWindowedBolt * * @param fieldName the name of the field that contains the message id @@ -153,7 +160,8 @@ public BaseStatefulWindowedBolt withMessageIdField(String fieldName) { } /** - * If set, the stateful windowed bolt would use the backend state for window persistence and only keep a sub-set of events in memory as + * If set, the stateful windowed bolt would use the backend state for window persistence and + * only keep a sub-set of events in memory as * specified by {@link #withMaxEventsInMemory(long)}. */ public BaseStatefulWindowedBolt withPersistence() { @@ -162,8 +170,10 @@ public BaseStatefulWindowedBolt withPersistence() { } /** - * The maximum number of window events to keep in memory. This is meaningful only if {@link #withPersistence()} is also set. As the - * number of events in memory grows close to the maximum, the events that are less likely to be used again are evicted and persisted. + * The maximum number of window events to keep in memory. This is meaningful only if {@link + * #withPersistence()} is also set. As the + * number of events in memory grows close to the maximum, the events that are less likely to be + * used again are evicted and persisted. * The default value for this is {@code 1,000,000}. * * @param maxEventsInMemory the maximum number of window events to keep in memory @@ -180,7 +190,8 @@ public boolean isPersistent() { @Override public long maxEventsInMemory() { - return maxEventsInMemory > 0 ? maxEventsInMemory : IStatefulWindowedBolt.super.maxEventsInMemory(); + return maxEventsInMemory > 0 ? maxEventsInMemory : IStatefulWindowedBolt.super + .maxEventsInMemory(); } @Override diff --git a/storm-client/src/jvm/org/apache/storm/topology/base/BaseTickTupleAwareRichBolt.java b/storm-client/src/jvm/org/apache/storm/topology/base/BaseTickTupleAwareRichBolt.java index 5e406c39913..2852febb6b9 100644 --- a/storm-client/src/jvm/org/apache/storm/topology/base/BaseTickTupleAwareRichBolt.java +++ b/storm-client/src/jvm/org/apache/storm/topology/base/BaseTickTupleAwareRichBolt.java @@ -40,7 +40,8 @@ public void execute(final Tuple tuple) { } /** - * Process a single tick tuple of input. Tick tuple doesn't need to be acked. It provides default "DO NOTHING" implementation for + * Process a single tick tuple of input. Tick tuple doesn't need to be acked. It provides + * default "DO NOTHING" implementation for * convenient. Override this method if needed. * *

    More details on {@link org.apache.storm.task.IBolt#execute(Tuple)}. @@ -51,7 +52,8 @@ protected void onTickTuple(final Tuple tuple) { } /** - * Process a single non-tick tuple of input. Implementation needs to handle ack manually. More details on {@link + * Process a single non-tick tuple of input. Implementation needs to handle ack manually. More + * details on {@link * org.apache.storm.task.IBolt#execute(Tuple)}. * * @param tuple The input tuple to be processed. diff --git a/storm-client/src/jvm/org/apache/storm/topology/base/BaseWindowedBolt.java b/storm-client/src/jvm/org/apache/storm/topology/base/BaseWindowedBolt.java index 318700d5d58..e9998079dff 100644 --- a/storm-client/src/jvm/org/apache/storm/topology/base/BaseWindowedBolt.java +++ b/storm-client/src/jvm/org/apache/storm/topology/base/BaseWindowedBolt.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -75,7 +81,8 @@ private BaseWindowedBolt withSlidingInterval(Duration duration) { throw new IllegalArgumentException("Sliding interval duration cannot be set null"); } if (duration.value <= 0) { - throw new IllegalArgumentException("Sliding interval must be positive [" + duration + "]"); + throw new IllegalArgumentException("Sliding interval must be positive [" + duration + + "]"); } windowConfiguration.put(Config.TOPOLOGY_BOLTS_SLIDING_INTERVAL_DURATION_MS, duration.value); return this; @@ -158,8 +165,10 @@ public BaseWindowedBolt withTumblingWindow(Duration duration) { } /** - * Specify a field in the tuple that represents the timestamp as a long value. If this field is not present in the incoming tuple, an - * {@link IllegalArgumentException} will be thrown. The field MUST contain a timestamp in milliseconds + * Specify a field in the tuple that represents the timestamp as a long value. If this field is + * not present in the incoming tuple, an + * {@link IllegalArgumentException} will be thrown. The field MUST contain a timestamp in + * milliseconds * * @param fieldName the name of the field that contains the timestamp */ @@ -177,7 +186,8 @@ public BaseWindowedBolt withTimestampExtractor(TimestampExtractor timestampExtra throw new IllegalArgumentException("Timestamp extractor cannot be set to null"); } if (this.timestampExtractor != null) { - throw new IllegalArgumentException("Window is already configured with a timestamp extractor: " + timestampExtractor); + throw new IllegalArgumentException("Window is already configured with a timestamp " + + "extractor: " + timestampExtractor); } this.timestampExtractor = timestampExtractor; return this; @@ -189,12 +199,17 @@ public TimestampExtractor getTimestampExtractor() { } /** - * Specify a stream id on which late tuples are going to be emitted. They are going to be accessible via the {@link - * org.apache.storm.topology.WindowedBoltExecutor#LATE_TUPLE_FIELD} field. It must be defined on a per-component basis, and in - * conjunction with the {@link BaseWindowedBolt#withTimestampField}, otherwise {@link IllegalArgumentException} will be thrown. + * Specify a stream id on which late tuples are going to be emitted. They are going to be + * accessible via the {@link + * org.apache.storm.topology.WindowedBoltExecutor#LATE_TUPLE_FIELD} field. It must be defined on + * a per-component basis, and in + * conjunction with the {@link BaseWindowedBolt#withTimestampField}, otherwise {@link + * IllegalArgumentException} will be thrown. * - *

    The late tuple is emitted as a {@link org.apache.storm.tuple.DetachedTuple}, a serializable copy of the original - * tuple detached from the topology context, so it can be consumed by bolts running in other workers. + *

    The late tuple is emitted as a {@link org.apache.storm.tuple.DetachedTuple}, a + * serializable copy of the original + * tuple detached from the topology context, so it can be consumed by bolts running in other + * workers. * * @param streamId the name of the stream used to emit late tuples on */ @@ -207,7 +222,8 @@ public BaseWindowedBolt withLateTupleStream(String streamId) { } /** - * Specify the maximum time lag of the tuple timestamp in milliseconds. It means that the tuple timestamps cannot be out of order by + * Specify the maximum time lag of the tuple timestamp in milliseconds. It means that the tuple + * timestamps cannot be out of order by * more than this amount. * * @param duration the max lag duration @@ -218,7 +234,8 @@ public BaseWindowedBolt withLag(Duration duration) { } /** - * Specify the watermark event generation interval. For tuple based timestamps, watermark events are used to track the progress of time + * Specify the watermark event generation interval. For tuple based timestamps, watermark events + * are used to track the progress of time * * @param interval the interval at which watermark events are generated */ @@ -231,7 +248,8 @@ public BaseWindowedBolt withWatermarkInterval(Duration interval) { } @Override - public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) { + public void prepare(Map topoConf, TopologyContext context, + OutputCollector collector) { // NOOP } diff --git a/storm-client/src/jvm/org/apache/storm/transactional/TransactionAttempt.java b/storm-client/src/jvm/org/apache/storm/transactional/TransactionAttempt.java index 150c9300d9e..d86f626988d 100644 --- a/storm-client/src/jvm/org/apache/storm/transactional/TransactionAttempt.java +++ b/storm-client/src/jvm/org/apache/storm/transactional/TransactionAttempt.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -22,7 +28,6 @@ public class TransactionAttempt { BigInteger txid; long attemptId; - // for kryo compatibility public TransactionAttempt() { diff --git a/storm-client/src/jvm/org/apache/storm/trident/JoinOutFieldsMode.java b/storm-client/src/jvm/org/apache/storm/trident/JoinOutFieldsMode.java index d095b6e4dba..85847486952 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/JoinOutFieldsMode.java +++ b/storm-client/src/jvm/org/apache/storm/trident/JoinOutFieldsMode.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -15,11 +21,14 @@ /** * This enum defines how the output fields of JOIN is constructed. * - *

    If user specifies COMPACT while calling JOIN, the tuples emitted from the join will contain: First, the list of - * join fields. Please note that joining fields are exposed only once from emitted tuples. Next, a list of all non-join + *

    If user specifies COMPACT while calling JOIN, the tuples emitted from the join will contain: + * First, the list of + * join fields. Please note that joining fields are exposed only once from emitted tuples. Next, a + * list of all non-join * fields from all streams, in order of how the streams were passed to the join method. * - *

    If user specifies PRESERVE while calling JOIN, the tuples emitted from the join will contain: a list of all fields + *

    If user specifies PRESERVE while calling JOIN, the tuples emitted from the join will contain: + * a list of all fields * from all streams, in order of how the streams were passed to the join method. */ public enum JoinOutFieldsMode { diff --git a/storm-client/src/jvm/org/apache/storm/trident/JoinType.java b/storm-client/src/jvm/org/apache/storm/trident/JoinType.java index 6e90faba7af..3bb6d3cf5b3 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/JoinType.java +++ b/storm-client/src/jvm/org/apache/storm/trident/JoinType.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/Stream.java b/storm-client/src/jvm/org/apache/storm/trident/Stream.java index 8e42c03a936..fd9b5a24c74 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/Stream.java +++ b/storm-client/src/jvm/org/apache/storm/trident/Stream.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -82,19 +88,27 @@ import org.apache.storm.utils.Utils; /** - * A Stream represents the core data model in Trident, and can be thought of as a "stream" of tuples that are processed as a series of small - * batches. A stream is partitioned across the nodes in the cluster, and operations are applied to a stream in parallel across each + * A Stream represents the core data model in Trident, and can be thought of as a "stream" of tuples + * that are processed as a series of small + * batches. A stream is partitioned across the nodes in the cluster, and operations are applied to a + * stream in parallel across each * partition. * *

    There are five types of operations that can be performed on streams in Trident * - *

    1. **Partiton-Local Operations** - Operations that are applied locally to each partition and do not involve network transfer 2. - * **Repartitioning Operations** - Operations that change how tuples are partitioned across tasks(thus causing network transfer), but do not - * change the content of the stream. 3. **Aggregation Operations** - Operations that *may* repartition a stream (thus causing network - * transfer) 4. **Grouping Operations** - Operations that may repartition a stream on specific fields and group together tuples whose fields - * values are equal. 5. **Merge and Join Operations** - Operations that combine different streams together. + *

    1. **Partiton-Local Operations** - Operations that are applied locally to each partition and + * do not involve network transfer 2. + * **Repartitioning Operations** - Operations that change how tuples are partitioned across + * tasks(thus causing network transfer), but do not + * change the content of the stream. 3. **Aggregation Operations** - Operations that *may* + * repartition a stream (thus causing network + * transfer) 4. **Grouping Operations** - Operations that may repartition a stream on specific + * fields and group together tuples whose fields + * values are equal. 5. **Merge and Join Operations** - Operations that combine different streams + * together. */ -// TODO: need to be able to replace existing fields with the function fields (like Cascading Fields.REPLACE) +// TODO: need to be able to replace existing fields with the function fields (like Cascading +// Fields.REPLACE) public class Stream implements IAggregatableStream, ResourceDeclarer { final Node node; final String name; @@ -107,7 +121,8 @@ protected Stream(TridentTopology topology, String name, Node node) { } /** - * Applies a label to the stream. Naming a stream will append the label to the name of the bolt(s) created by Trident and will be + * Applies a label to the stream. Naming a stream will append the label to the name of the + * bolt(s) created by Trident and will be * visible in the Storm UI. * * @param name - The label to apply to the stream @@ -158,9 +173,11 @@ public Stream addSharedMemory(SharedMemory request) { } /** - * Filters out fields from a stream, resulting in a Stream containing only the fields specified by `keepFields`. + * Filters out fields from a stream, resulting in a Stream containing only the fields specified + * by `keepFields`. * - *

    For example, if you had a Stream `mystream` containing the fields `["a", "b", "c","d"]`, calling" + *

    For example, if you had a Stream `mystream` containing the fields `["a", "b", "c","d"]`, + * calling" * *

    ```java mystream.project(new Fields("b", "d")) ``` * @@ -204,20 +221,23 @@ public Stream partition(CustomStreamGrouping partitioner) { /** * ## Repartitioning Operation. * - *

    This method takes in a custom partitioning function that implements {@link org.apache.storm.grouping.CustomStreamGrouping} + *

    This method takes in a custom partitioning function that implements {@link + * org.apache.storm.grouping.CustomStreamGrouping} */ public Stream partition(Grouping grouping) { if (node instanceof PartitionNode) { return each(new Fields(), new TrueFilter()).partition(grouping); } else { - return topology.addSourcedNode(this, new PartitionNode(node.streamId, name, getOutputFields(), grouping)); + return topology.addSourcedNode(this, new PartitionNode(node.streamId, name, + getOutputFields(), grouping)); } } /** * ## Repartitioning Operation. * - *

    Use random round robin algorithm to evenly redistribute tuples across all target partitions. + *

    Use random round robin algorithm to evenly redistribute tuples across all target + * partitions. */ public Stream shuffle() { return partition(Grouping.shuffle(new NullStruct())); @@ -226,20 +246,22 @@ public Stream shuffle() { /** * ## Repartitioning Operation. * - *

    Use random round robin algorithm to evenly redistribute tuples across all target partitions, with a preference for local tasks. + *

    Use random round robin algorithm to evenly redistribute tuples across all target + * partitions, with a preference for local tasks. */ public Stream localOrShuffle() { return partition(Grouping.local_or_shuffle(new NullStruct())); } - /** * ## Repartitioning Operation. * - *

    All tuples are sent to the same partition. The same partition is chosen for all batches in the stream. + *

    All tuples are sent to the same partition. The same partition is chosen for all batches in + * the stream. */ public Stream global() { - // use this instead of storm's built in one so that we can specify a singleemitbatchtopartition + // use this instead of storm's built in one so that we can specify a + // singleemitbatchtopartition // without knowledge of storm's internals return partition(new GlobalGrouping()); } @@ -247,7 +269,8 @@ public Stream global() { /** * ## Repartitioning Operation. * - *

    All tuples in the batch are sent to the same partition. Different batches in the stream may go to different partitions. + *

    All tuples in the batch are sent to the same partition. Different batches in the stream + * may go to different partitions. */ public Stream batchGlobal() { // the first field is the batch id @@ -257,7 +280,8 @@ public Stream batchGlobal() { /** * ## Repartitioning Operation. * - *

    Every tuple is replicated to all target partitions. This can useful during DRPC – for example, if you need to do a stateQuery on + *

    Every tuple is replicated to all target partitions. This can useful during DRPC – for + * example, if you need to do a stateQuery on * every partition of data. */ public Stream broadcast() { @@ -299,7 +323,7 @@ public Stream each(Fields inputFields, Filter filter) { return each(inputFields, new FilterExecutor(filter), new Fields()); } - //creates brand new tuples with brand new fields + // creates brand new tuples with brand new fields @Override public Stream partitionAggregate(Fields inputFields, Aggregator agg, Fields functionFields) { projectionValidation(inputFields); @@ -319,7 +343,8 @@ public Stream partitionAggregate(CombinerAggregator agg, Fields functionFields) return partitionAggregate(null, agg, functionFields); } - public Stream partitionAggregate(Fields inputFields, CombinerAggregator agg, Fields functionFields) { + public Stream partitionAggregate(Fields inputFields, CombinerAggregator agg, + Fields functionFields) { projectionValidation(inputFields); return chainedAgg() .partitionAggregate(inputFields, agg, functionFields) @@ -330,14 +355,16 @@ public Stream partitionAggregate(ReducerAggregator agg, Fields functionFields) { return partitionAggregate(null, agg, functionFields); } - public Stream partitionAggregate(Fields inputFields, ReducerAggregator agg, Fields functionFields) { + public Stream partitionAggregate(Fields inputFields, ReducerAggregator agg, + Fields functionFields) { projectionValidation(inputFields); return chainedAgg() .partitionAggregate(inputFields, agg, functionFields) .chainEnd(); } - public Stream stateQuery(TridentState state, Fields inputFields, QueryFunction function, Fields functionFields) { + public Stream stateQuery(TridentState state, Fields inputFields, QueryFunction function, + Fields functionFields) { projectionValidation(inputFields); String stateId = state.node.stateInfo.id; Node n = new ProcessorNode(topology.getUniqueStreamId(), @@ -353,11 +380,13 @@ public Stream stateQuery(TridentState state, QueryFunction function, Fields func return stateQuery(state, null, function, functionFields); } - public TridentState partitionPersist(StateFactory stateFactory, Fields inputFields, StateUpdater updater, Fields functionFields) { + public TridentState partitionPersist(StateFactory stateFactory, Fields inputFields, + StateUpdater updater, Fields functionFields) { return partitionPersist(new StateSpec(stateFactory), inputFields, updater, functionFields); } - public TridentState partitionPersist(StateSpec stateSpec, Fields inputFields, StateUpdater updater, Fields functionFields) { + public TridentState partitionPersist(StateSpec stateSpec, Fields inputFields, + StateUpdater updater, Fields functionFields) { projectionValidation(inputFields); String id = topology.getUniqueStateId(); ProcessorNode n = new ProcessorNode(topology.getUniqueStreamId(), @@ -370,19 +399,23 @@ public TridentState partitionPersist(StateSpec stateSpec, Fields inputFields, St return topology.addSourcedStateNode(this, n); } - public TridentState partitionPersist(StateFactory stateFactory, Fields inputFields, StateUpdater updater) { + public TridentState partitionPersist(StateFactory stateFactory, Fields inputFields, + StateUpdater updater) { return partitionPersist(stateFactory, inputFields, updater, new Fields()); } - public TridentState partitionPersist(StateSpec stateSpec, Fields inputFields, StateUpdater updater) { + public TridentState partitionPersist(StateSpec stateSpec, Fields inputFields, + StateUpdater updater) { return partitionPersist(stateSpec, inputFields, updater, new Fields()); } - public TridentState partitionPersist(StateFactory stateFactory, StateUpdater updater, Fields functionFields) { + public TridentState partitionPersist(StateFactory stateFactory, StateUpdater updater, + Fields functionFields) { return partitionPersist(new StateSpec(stateFactory), updater, functionFields); } - public TridentState partitionPersist(StateSpec stateSpec, StateUpdater updater, Fields functionFields) { + public TridentState partitionPersist(StateSpec stateSpec, StateUpdater updater, + Fields functionFields) { return partitionPersist(stateSpec, null, updater, functionFields); } @@ -397,7 +430,8 @@ public TridentState partitionPersist(StateSpec stateSpec, StateUpdater updater) /** * Returns a stream consisting of the elements of this stream that match the given filter. * - * @param filter the filter to apply to each trident tuple to determine if it should be included. + * @param filter the filter to apply to each trident tuple to determine if it should be + * included. * @return the new stream */ public Stream filter(Filter filter) { @@ -408,7 +442,8 @@ public Stream filter(Filter filter) { * Returns a stream consisting of the elements of this stream that match the given filter. * * @param inputFields the fields of the input trident tuple to be selected. - * @param filter the filter to apply to each trident tuple to determine if it should be included. + * @param filter the filter to apply to each trident tuple to determine if it should be + * included. * @return the new stream */ public Stream filter(Fields inputFields, Filter filter) { @@ -416,7 +451,8 @@ public Stream filter(Fields inputFields, Filter filter) { } /** - * Returns a stream consisting of the result of applying the given mapping function to the values of this stream. + * Returns a stream consisting of the result of applying the given mapping function to the + * values of this stream. * * @param function a mapping function to be applied to each value in this stream. * @return the new stream @@ -432,7 +468,8 @@ public Stream map(MapFunction function) { } /** - * Returns a stream consisting of the result of applying the given mapping function to the values of this stream. This method replaces + * Returns a stream consisting of the result of applying the given mapping function to the + * values of this stream. This method replaces * old output fields with new output fields, achieving T -> V conversion. * * @param function a mapping function to be applied to each value in this stream. @@ -450,11 +487,14 @@ public Stream map(MapFunction function, Fields outputFields) { } /** - * Returns a stream consisting of the results of replacing each value of this stream with the contents produced by applying the provided - * mapping function to each value. This has the effect of applying a one-to-many transformation to the values of the stream, and then + * Returns a stream consisting of the results of replacing each value of this stream with the + * contents produced by applying the provided + * mapping function to each value. This has the effect of applying a one-to-many transformation + * to the values of the stream, and then * flattening the resulting elements into a new stream. * - * @param function a mapping function to be applied to each value in this stream which produces new values. + * @param function a mapping function to be applied to each value in this stream which produces + * new values. * @return the new stream */ public Stream flatMap(FlatMapFunction function) { @@ -464,16 +504,21 @@ public Stream flatMap(FlatMapFunction function) { name, getOutputFields(), getOutputFields(), - new MapProcessor(getOutputFields(), new FlatMapFunctionExecutor(function)))); + new MapProcessor(getOutputFields(), + new FlatMapFunctionExecutor(function)))); } /** - * Returns a stream consisting of the results of replacing each value of this stream with the contents produced by applying the provided - * mapping function to each value. This has the effect of applying a one-to-many transformation to the values of the stream, and then - * flattening the resulting elements into a new stream. This method replaces old output fields with new output fields, achieving T -> V + * Returns a stream consisting of the results of replacing each value of this stream with the + * contents produced by applying the provided + * mapping function to each value. This has the effect of applying a one-to-many transformation + * to the values of the stream, and then + * flattening the resulting elements into a new stream. This method replaces old output fields + * with new output fields, achieving T -> V * conversion. * - * @param function a mapping function to be applied to each value in this stream which produces new values. + * @param function a mapping function to be applied to each value in this stream which produces + * new values. * @param outputFields new output fields * @return the new stream */ @@ -484,12 +529,15 @@ public Stream flatMap(FlatMapFunction function, Fields outputFields) { name, outputFields, outputFields, - new MapProcessor(getOutputFields(), new FlatMapFunctionExecutor(function)))); + new MapProcessor(getOutputFields(), + new FlatMapFunctionExecutor(function)))); } /** - * Returns a stream consisting of the trident tuples of this stream, additionally performing the provided action on each trident tuple - * as they are consumed from the resulting stream. This is mostly useful for debugging to see the tuples as they flow past a certain + * Returns a stream consisting of the trident tuples of this stream, additionally performing the + * provided action on each trident tuple + * as they are consumed from the resulting stream. This is mostly useful for debugging to see + * the tuples as they flow past a certain * point in a pipeline. * * @param action the action to perform on the trident tuple as they are consumed from the stream @@ -532,7 +580,8 @@ public Stream minBy(String inputFieldName) { * @return the new stream with this operation. */ public Stream minBy(String inputFieldName, Comparator comparator) { - Aggregator min = new MinWithComparator<>(inputFieldName, comparator); + Aggregator min = new MinWithComparator<>(inputFieldName, + comparator); return comparableAggregateStream(inputFieldName, min); } @@ -571,7 +620,8 @@ public Stream maxBy(String inputFieldName) { * @return the new stream with this operation. */ public Stream maxBy(String inputFieldName, Comparator comparator) { - Aggregator max = new MaxWithComparator<>(inputFieldName, comparator); + Aggregator max = new MaxWithComparator<>(inputFieldName, + comparator); return comparableAggregateStream(inputFieldName, max); } @@ -633,13 +683,15 @@ public Stream aggregate(Fields inputFields, ReducerAggregator agg, Fields functi * @param windowCount represents number of tuples in the window * @param windowStoreFactory intermediary tuple store for storing windowing tuples * @param inputFields projected fields for aggregator - * @param aggregator aggregator to run on the window of tuples to compute the result and emit to the stream. + * @param aggregator aggregator to run on the window of tuples to compute the result and emit to + * the stream. * @param functionFields fields of values to emit with aggregation. * @return the new stream with this operation. */ public Stream tumblingWindow(int windowCount, WindowsStoreFactory windowStoreFactory, Fields inputFields, Aggregator aggregator, Fields functionFields) { - return window(TumblingCountWindow.of(windowCount), windowStoreFactory, inputFields, aggregator, functionFields); + return window(TumblingCountWindow.of(windowCount), windowStoreFactory, inputFields, + aggregator, functionFields); } /** @@ -648,13 +700,16 @@ public Stream tumblingWindow(int windowCount, WindowsStoreFactory windowStoreFac * @param windowDuration represents tumbling window duration configuration * @param windowStoreFactory intermediary tuple store for storing windowing tuples * @param inputFields projected fields for aggregator - * @param aggregator aggregator to run on the window of tuples to compute the result and emit to the stream. + * @param aggregator aggregator to run on the window of tuples to compute the result and emit to + * the stream. * @param functionFields fields of values to emit with aggregation. * @return the new stream with this operation. */ - public Stream tumblingWindow(BaseWindowedBolt.Duration windowDuration, WindowsStoreFactory windowStoreFactory, + public Stream tumblingWindow(BaseWindowedBolt.Duration windowDuration, + WindowsStoreFactory windowStoreFactory, Fields inputFields, Aggregator aggregator, Fields functionFields) { - return window(TumblingDurationWindow.of(windowDuration), windowStoreFactory, inputFields, aggregator, functionFields); + return window(TumblingDurationWindow.of(windowDuration), windowStoreFactory, inputFields, + aggregator, functionFields); } /** @@ -665,13 +720,16 @@ public Stream tumblingWindow(BaseWindowedBolt.Duration windowDuration, WindowsSt * @param slideCount the number of tuples after which the window slides * @param windowStoreFactory intermediary tuple store for storing windowing tuples * @param inputFields projected fields for aggregator - * @param aggregator aggregator to run on the window of tuples to compute the result and emit to the stream. + * @param aggregator aggregator to run on the window of tuples to compute the result and emit to + * the stream. * @param functionFields fields of values to emit with aggregation. * @return the new stream with this operation. */ - public Stream slidingWindow(int windowCount, int slideCount, WindowsStoreFactory windowStoreFactory, + public Stream slidingWindow(int windowCount, int slideCount, + WindowsStoreFactory windowStoreFactory, Fields inputFields, Aggregator aggregator, Fields functionFields) { - return window(SlidingCountWindow.of(windowCount, slideCount), windowStoreFactory, inputFields, aggregator, functionFields); + return window(SlidingCountWindow.of(windowCount, slideCount), windowStoreFactory, + inputFields, aggregator, functionFields); } /** @@ -682,30 +740,38 @@ public Stream slidingWindow(int windowCount, int slideCount, WindowsStoreFactory * @param slidingInterval the time duration after which the window slides * @param windowStoreFactory intermediary tuple store for storing windowing tuples * @param inputFields projected fields for aggregator - * @param aggregator aggregator to run on the window of tuples to compute the result and emit to the stream. + * @param aggregator aggregator to run on the window of tuples to compute the result and emit to + * the stream. * @param functionFields fields of values to emit with aggregation. * @return the new stream with this operation. */ - public Stream slidingWindow(BaseWindowedBolt.Duration windowDuration, BaseWindowedBolt.Duration slidingInterval, + public Stream slidingWindow(BaseWindowedBolt.Duration windowDuration, + BaseWindowedBolt.Duration slidingInterval, WindowsStoreFactory windowStoreFactory, Fields inputFields, Aggregator aggregator, Fields functionFields) { - return window(SlidingDurationWindow.of(windowDuration, slidingInterval), windowStoreFactory, inputFields, aggregator, + return window(SlidingDurationWindow.of(windowDuration, slidingInterval), windowStoreFactory, + inputFields, aggregator, functionFields); } /** - * Returns a stream of aggregated results based on the given window configuration which uses inmemory windowing tuple store. + * Returns a stream of aggregated results based on the given window configuration which uses + * inmemory windowing tuple store. * * @param windowConfig window configuration like window length and slide length. * @param inputFields input fields - * @param aggregator aggregator to run on the window of tuples to compute the result and emit to the stream. + * @param aggregator aggregator to run on the window of tuples to compute the result and emit to + * the stream. * @param functionFields fields of values to emit with aggregation. * @return the new stream with this operation. */ - public Stream window(WindowConfig windowConfig, Fields inputFields, Aggregator aggregator, Fields functionFields) { - // this store is used only for storing triggered aggregated results but not tuples as storeTuplesInStore is set + public Stream window(WindowConfig windowConfig, Fields inputFields, Aggregator aggregator, + Fields functionFields) { + // this store is used only for storing triggered aggregated results but not tuples as + // storeTuplesInStore is set // as false int he below call. InMemoryWindowsStoreFactory inMemoryWindowsStoreFactory = new InMemoryWindowsStoreFactory(); - return window(windowConfig, inMemoryWindowsStoreFactory, inputFields, aggregator, functionFields, false); + return window(windowConfig, inMemoryWindowsStoreFactory, inputFields, aggregator, + functionFields, false); } /** @@ -714,23 +780,28 @@ public Stream window(WindowConfig windowConfig, Fields inputFields, Aggregator a * @param windowConfig window configuration like window length and slide length. * @param windowStoreFactory intermediary tuple store for storing tuples for windowing * @param inputFields input fields - * @param aggregator aggregator to run on the window of tuples to compute the result and emit to the stream. + * @param aggregator aggregator to run on the window of tuples to compute the result and emit to + * the stream. * @param functionFields fields of values to emit with aggregation. * @return the new stream with this operation. */ - public Stream window(WindowConfig windowConfig, WindowsStoreFactory windowStoreFactory, Fields inputFields, + public Stream window(WindowConfig windowConfig, WindowsStoreFactory windowStoreFactory, + Fields inputFields, Aggregator aggregator, Fields functionFields) { - return window(windowConfig, windowStoreFactory, inputFields, aggregator, functionFields, true); + return window(windowConfig, windowStoreFactory, inputFields, aggregator, functionFields, + true); } - private Stream window(WindowConfig windowConfig, WindowsStoreFactory windowStoreFactory, Fields inputFields, Aggregator aggregator, + private Stream window(WindowConfig windowConfig, WindowsStoreFactory windowStoreFactory, + Fields inputFields, Aggregator aggregator, Fields functionFields, boolean storeTuplesInStore) { projectionValidation(inputFields); windowConfig.validate(); Fields fields = addTriggerField(functionFields); - // when storeTuplesInStore is false then the given windowStoreFactory is only used to store triggers and + // when storeTuplesInStore is false then the given windowStoreFactory is only used to store + // triggers and // that store is passed to WindowStateUpdater to remove them after committing the batch. Stream stream = topology.addSourcedNode(this, new ProcessorNode(topology.getUniqueStreamId(), @@ -745,11 +816,13 @@ private Stream window(WindowConfig windowConfig, WindowsStoreFactory windowStore Stream effectiveStream = stream.project(functionFields); - // create StateUpdater with the given windowStoreFactory to remove triggered aggregation results form store + // create StateUpdater with the given windowStoreFactory to remove triggered aggregation + // results form store // when they are successfully processed. StateFactory stateFactory = new WindowsStateFactory(); StateUpdater stateUpdater = new WindowsStateUpdater(windowStoreFactory); - stream.partitionPersist(stateFactory, new Fields(WindowTridentProcessor.TRIGGER_FIELD_NAME), stateUpdater, new Fields()); + stream.partitionPersist(stateFactory, new Fields(WindowTridentProcessor.TRIGGER_FIELD_NAME), + stateUpdater, new Fields()); return effectiveStream; } @@ -763,42 +836,53 @@ private Fields addTriggerField(Fields functionFields) { return new Fields(fieldsList); } - public TridentState persistentAggregate(StateFactory stateFactory, CombinerAggregator agg, Fields functionFields) { + public TridentState persistentAggregate(StateFactory stateFactory, CombinerAggregator agg, + Fields functionFields) { return persistentAggregate(new StateSpec(stateFactory), agg, functionFields); } - public TridentState persistentAggregate(StateSpec spec, CombinerAggregator agg, Fields functionFields) { + public TridentState persistentAggregate(StateSpec spec, CombinerAggregator agg, + Fields functionFields) { return persistentAggregate(spec, null, agg, functionFields); } - public TridentState persistentAggregate(StateFactory stateFactory, Fields inputFields, CombinerAggregator agg, Fields functionFields) { + public TridentState persistentAggregate(StateFactory stateFactory, Fields inputFields, + CombinerAggregator agg, Fields functionFields) { return persistentAggregate(new StateSpec(stateFactory), inputFields, agg, functionFields); } - public TridentState persistentAggregate(StateSpec spec, Fields inputFields, CombinerAggregator agg, Fields functionFields) { + public TridentState persistentAggregate(StateSpec spec, Fields inputFields, + CombinerAggregator agg, Fields functionFields) { projectionValidation(inputFields); - // replaces normal aggregation here with a global grouping because it needs to be consistent across batches + // replaces normal aggregation here with a global grouping because it needs to be consistent + // across batches return new ChainedAggregatorDeclarer(this, new GlobalAggScheme()) .aggregate(inputFields, agg, functionFields) .chainEnd() - .partitionPersist(spec, functionFields, new CombinerAggStateUpdater(agg), functionFields); + .partitionPersist(spec, functionFields, new CombinerAggStateUpdater(agg), + functionFields); } - public TridentState persistentAggregate(StateFactory stateFactory, ReducerAggregator agg, Fields functionFields) { + public TridentState persistentAggregate(StateFactory stateFactory, ReducerAggregator agg, + Fields functionFields) { return persistentAggregate(new StateSpec(stateFactory), agg, functionFields); } - public TridentState persistentAggregate(StateSpec spec, ReducerAggregator agg, Fields functionFields) { + public TridentState persistentAggregate(StateSpec spec, ReducerAggregator agg, + Fields functionFields) { return persistentAggregate(spec, null, agg, functionFields); } - public TridentState persistentAggregate(StateFactory stateFactory, Fields inputFields, ReducerAggregator agg, Fields functionFields) { + public TridentState persistentAggregate(StateFactory stateFactory, Fields inputFields, + ReducerAggregator agg, Fields functionFields) { return persistentAggregate(new StateSpec(stateFactory), inputFields, agg, functionFields); } - public TridentState persistentAggregate(StateSpec spec, Fields inputFields, ReducerAggregator agg, Fields functionFields) { + public TridentState persistentAggregate(StateSpec spec, Fields inputFields, + ReducerAggregator agg, Fields functionFields) { projectionValidation(inputFields); - return global().partitionPersist(spec, inputFields, new ReducerAggStateUpdater(agg), functionFields); + return global().partitionPersist(spec, inputFields, new ReducerAggStateUpdater(agg), + functionFields); } @Override @@ -829,7 +913,8 @@ private void projectionValidation(Fields projFields) { for (String field : projFields) { if (!allFields.contains(field)) { throw new IllegalArgumentException( - "Trying to select non-existent field: '" + field + "' from stream containing fields fields: <" + allFields + ">"); + "Trying to select non-existent field: '" + field + + "' from stream containing fields fields: <" + allFields + ">"); } } } diff --git a/storm-client/src/jvm/org/apache/storm/trident/TridentState.java b/storm-client/src/jvm/org/apache/storm/trident/TridentState.java index d02ae414948..bca625c526b 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/TridentState.java +++ b/storm-client/src/jvm/org/apache/storm/trident/TridentState.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -16,7 +22,6 @@ import org.apache.storm.topology.ResourceDeclarer; import org.apache.storm.trident.planner.Node; - public class TridentState implements ResourceDeclarer { TridentTopology topology; Node node; diff --git a/storm-client/src/jvm/org/apache/storm/trident/TridentTopology.java b/storm-client/src/jvm/org/apache/storm/trident/TridentTopology.java index 98924b174ad..e080324f547 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/TridentTopology.java +++ b/storm-client/src/jvm/org/apache/storm/trident/TridentTopology.java @@ -92,7 +92,7 @@ // all operations have finishBatch and can optionally be committers public class TridentTopology { - //TODO: add a method for drpc stream, needs to know how to automatically do return results, etc + // TODO: add a method for drpc stream, needs to know how to automatically do return results, etc // is it too expensive to do a batch per drpc request? final DefaultDirectedGraph graph; @@ -107,25 +107,29 @@ public TridentTopology() { new UniqueIdGen()); } - private TridentTopology(DefaultDirectedGraph graph, Map> colocate, UniqueIdGen gen) { + private TridentTopology(DefaultDirectedGraph graph, Map> colocate, UniqueIdGen gen) { this.graph = graph; this.colocate = colocate; this.gen = gen; } - // automatically turn it into a batch spout, should take parameters as to how much to batch // public Stream newStream(IRichSpout spout) { - // Node n = new SpoutNode(getUniqueStreamId(), TridentUtils.getSingleOutputStreamFields(spout), null, spout, SpoutNode + // Node n = new SpoutNode(getUniqueStreamId(), TridentUtils.getSingleOutputStreamFields(spout), + // null, spout, SpoutNode // .SpoutType.BATCH); // return addNode(n); // } - private static Map mergeDefaultResources(Map res, Map defaultConfig) { + private static Map mergeDefaultResources(Map res, Map defaultConfig) { Map ret = new HashMap<>(); - Number onHeapDefault = defaultConfig.get(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB); - Number offHeapDefault = defaultConfig.get(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB); + Number onHeapDefault = defaultConfig + .get(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB); + Number offHeapDefault = defaultConfig + .get(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB); Number cpuLoadDefault = defaultConfig.get(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT); if (res == null) { @@ -176,7 +180,8 @@ private static Map mergeDefaultResources(Map res return ret; } - private static void completeDrpc(DefaultDirectedGraph graph, Map> colocate, UniqueIdGen gen) { + private static void completeDrpc(DefaultDirectedGraph graph, Map> colocate, UniqueIdGen gen) { List> connectedComponents = new ConnectivityInspector<>(graph).connectedSets(); for (Set g : connectedComponents) { @@ -209,7 +214,7 @@ private static Node getLastAddedNode(Collection g) { return ret; } - //returns null if it's not a drpc group + // returns null if it's not a drpc group private static SpoutNode getDrpcSpoutNode(Collection g) { for (Node n : g) { if (n instanceof SpoutNode) { @@ -236,7 +241,8 @@ private static void checkValidJoins(Collection g) { } } if (hasBatchSpout && hasDrpcSpout) { - throw new RuntimeException("Cannot join DRPC stream with streams originating from other spouts"); + throw new RuntimeException("Cannot join DRPC stream with streams originating from " + + "other spouts"); } } @@ -249,7 +255,8 @@ private static Collection uniquedSubscriptions(Set for (PartitionNode n : subscriptions) { PartitionNode curr = ret.get(n.streamId); if (curr != null && !curr.thriftGrouping.equals(n.thriftGrouping)) { - throw new RuntimeException("Multiple subscriptions to the same stream with different groupings. Should " + throw new RuntimeException("Multiple subscriptions to the same stream with " + + "different groupings. Should " + "be impossible since that is explicitly guarded against."); } ret.put(n.streamId, n); @@ -263,7 +270,7 @@ private static Map genSpoutIds(Collection spoutNodes) { for (SpoutNode n : spoutNodes) { if (n.type == SpoutNode.SpoutType.BATCH) { // if Batch spout then id contains txId ret.put(n, "spout-" + n.txId); - } else if (n.type == SpoutNode.SpoutType.DRPC) { //if DRPC spout then id contains function + } else if (n.type == SpoutNode.SpoutType.DRPC) { // if DRPC spout then id contains function ret.put(n, "spout-" + ((DRPCSpout) n.spout).get_function() + ctr); ctr++; } else { @@ -311,7 +318,8 @@ private static String getGroupName(Group g) { return Utils.join(names, "-"); } - private static Map getOutputStreamBatchGroups(Group g, Map batchGroupMap) { + private static Map getOutputStreamBatchGroups(Group g, Map batchGroupMap) { Map ret = new HashMap<>(); Set externalGroupOutputs = externalGroupOutputs(g); for (PartitionNode n : externalGroupOutputs) { @@ -332,7 +340,8 @@ private static Set committerBatches(Group g, Map batchGrou return ret; } - private static Map getGroupParallelisms(Graph graph, GraphGrouper grouper, + private static Map getGroupParallelisms(Graph graph, + GraphGrouper grouper, Collection groups) { Graph equivs = new Pseudograph<>(Object.class); for (Group g : groups) { @@ -356,7 +365,8 @@ private static Map getGroupParallelisms(Graph Integer fixedP = getFixedParallelism(equivGroup); Integer maxP = getMaxParallelism(equivGroup); if (fixedP != null && maxP != null && maxP < fixedP) { - throw new RuntimeException("Parallelism is fixed to " + fixedP + " but max parallelism is less than that: " + maxP); + throw new RuntimeException("Parallelism is fixed to " + fixedP + + " but max parallelism is less than that: " + maxP); } @@ -421,7 +431,8 @@ private static Integer getFixedParallelism(Set groups) { if (n.stateInfo != null && n.stateInfo.spec.requiredNumPartitions != null) { int reqPartitions = n.stateInfo.spec.requiredNumPartitions; if (ret != null && ret != reqPartitions) { - throw new RuntimeException("Cannot have one group have fixed parallelism of two different values"); + throw new RuntimeException("Cannot have one group have fixed parallelism " + + "of two different values"); } ret = reqPartitions; } @@ -433,7 +444,8 @@ private static Integer getFixedParallelism(Set groups) { private static boolean isIdentityPartition(PartitionNode n) { Grouping g = n.thriftGrouping; if (g.is_set_custom_serialized()) { - CustomStreamGrouping csg = (CustomStreamGrouping) Utils.javaDeserialize(g.get_custom_serialized(), Serializable.class); + CustomStreamGrouping csg = (CustomStreamGrouping) Utils.javaDeserialize(g + .get_custom_serialized(), Serializable.class); return csg instanceof IdentityGrouping; } return false; @@ -487,7 +499,8 @@ private static Set externalGroupOutputs(Group g) { private static PartitionNode makeIdentityPartition(Node basis) { return new PartitionNode(basis.streamId, basis.name, basis.allOutputFields, - Grouping.custom_serialized(Utils.javaSerialize(new IdentityGrouping()))); + Grouping.custom_serialized(Utils + .javaSerialize(new IdentityGrouping()))); } private static List getAllOutputFields(List streams) { @@ -498,7 +511,8 @@ private static List getAllOutputFields(List streams) { return ret; } - private static List groupedStreams(List streams, List joinFields) { + private static List groupedStreams(List streams, + List joinFields) { List ret = new ArrayList<>(); for (int i = 0; i < streams.size(); i++) { ret.add(streams.get(i).groupBy(joinFields.get(i))); @@ -509,7 +523,8 @@ private static List groupedStreams(List streams, List strippedInputFields(List streams, List joinFields) { List ret = new ArrayList<>(); for (int i = 0; i < streams.size(); i++) { - ret.add(TridentUtils.fieldsSubtract(streams.get(i).getOutputFields(), joinFields.get(i))); + ret.add(TridentUtils.fieldsSubtract(streams.get(i).getOutputFields(), joinFields + .get(i))); } return ret; } @@ -527,12 +542,14 @@ public Stream newStream(String txId, IRichSpout spout) { } public Stream newStream(String txId, IBatchSpout spout) { - Node n = new SpoutNode(getUniqueStreamId(), spout.getOutputFields(), txId, spout, SpoutNode.SpoutType.BATCH); + Node n = new SpoutNode(getUniqueStreamId(), spout.getOutputFields(), txId, spout, + SpoutNode.SpoutType.BATCH); return addNode(n); } public Stream newStream(String txId, ITridentSpout spout) { - Node n = new SpoutNode(getUniqueStreamId(), spout.getOutputFields(), txId, spout, SpoutNode.SpoutType.BATCH); + Node n = new SpoutNode(getUniqueStreamId(), spout.getOutputFields(), txId, spout, + SpoutNode.SpoutType.BATCH); return addNode(n); } @@ -576,10 +593,12 @@ public Stream newDRPCStream(String function, ILocalDRPC server) { @SuppressWarnings("checkstyle:AbbreviationAsWordInName") private Stream newDRPCStream(DRPCSpout spout) { - // TODO: consider adding a shuffle grouping after the spout to avoid so much routing of the args/return-info all over the place + // TODO: consider adding a shuffle grouping after the spout to avoid so much routing of the + // args/return-info all over the place // (at least until its possible to just pack bolt logic into the spout itself) - Node n = new SpoutNode(getUniqueStreamId(), TridentUtils.getSingleOutputStreamFields(spout), null, spout, SpoutNode.SpoutType.DRPC); + Node n = new SpoutNode(getUniqueStreamId(), TridentUtils.getSingleOutputStreamFields(spout), + null, spout, SpoutNode.SpoutType.DRPC); Stream nextStream = addNode(n); // later on, this will be joined back with return-info and all the results return nextStream.project(new Fields("args")); @@ -601,40 +620,49 @@ public Stream multiReduce(Stream s1, Stream s2, MultiReducer function, Fields ou return multiReduce(Arrays.asList(s1, s2), function, outputFields); } - public Stream multiReduce(Fields inputFields1, Stream s1, Fields inputFields2, Stream s2, MultiReducer function, Fields outputFields) { - return multiReduce(Arrays.asList(inputFields1, inputFields2), Arrays.asList(s1, s2), function, outputFields); + public Stream multiReduce(Fields inputFields1, Stream s1, Fields inputFields2, Stream s2, + MultiReducer function, Fields outputFields) { + return multiReduce(Arrays.asList(inputFields1, inputFields2), Arrays.asList(s1, s2), + function, outputFields); } - public Stream multiReduce(GroupedStream s1, GroupedStream s2, GroupedMultiReducer function, Fields outputFields) { + public Stream multiReduce(GroupedStream s1, GroupedStream s2, GroupedMultiReducer function, + Fields outputFields) { return multiReduce(Arrays.asList(s1, s2), function, outputFields); } - public Stream multiReduce(Fields inputFields1, GroupedStream s1, Fields inputFields2, GroupedStream s2, GroupedMultiReducer function, + public Stream multiReduce(Fields inputFields1, GroupedStream s1, Fields inputFields2, + GroupedStream s2, GroupedMultiReducer function, Fields outputFields) { - return multiReduce(Arrays.asList(inputFields1, inputFields2), Arrays.asList(s1, s2), function, outputFields); + return multiReduce(Arrays.asList(inputFields1, inputFields2), Arrays.asList(s1, s2), + function, outputFields); } public Stream multiReduce(List streams, MultiReducer function, Fields outputFields) { return multiReduce(getAllOutputFields(streams), streams, function, outputFields); } - public Stream multiReduce(List streams, GroupedMultiReducer function, Fields outputFields) { + public Stream multiReduce(List streams, GroupedMultiReducer function, + Fields outputFields) { return multiReduce(getAllOutputFields(streams), streams, function, outputFields); } - public Stream multiReduce(List inputFields, List streams, MultiReducer function, Fields outputFields) { + public Stream multiReduce(List inputFields, List streams, MultiReducer function, + Fields outputFields) { List names = new ArrayList<>(); for (Stream s : streams) { if (s.name != null) { names.add(s.name); } } - Node n = new ProcessorNode(getUniqueStreamId(), Utils.join(names, "-"), outputFields, outputFields, + Node n = new ProcessorNode(getUniqueStreamId(), Utils.join(names, "-"), outputFields, + outputFields, new MultiReducerProcessor(inputFields, function)); return addSourcedNode(streams, n); } - public Stream multiReduce(List inputFields, List groupedStreams, GroupedMultiReducer function, + public Stream multiReduce(List inputFields, List groupedStreams, + GroupedMultiReducer function, Fields outputFields) { List fullInputFields = new ArrayList<>(); List streams = new ArrayList<>(); @@ -647,7 +675,8 @@ public Stream multiReduce(List inputFields, List groupedS fullInputFields.add(TridentUtils.fieldsUnion(groupFields, inputFields.get(i))); } - return multiReduce(fullInputFields, streams, new GroupedMultiReducerExecutor(function, fullGroupFields, inputFields), outputFields); + return multiReduce(fullInputFields, streams, new GroupedMultiReducerExecutor(function, + fullGroupFields, inputFields), outputFields); } public Stream merge(Fields outputFields, Stream... streams) { @@ -666,7 +695,8 @@ public Stream merge(List streams) { return merge(streams.get(0).getOutputFields(), streams); } - public Stream join(Stream s1, Fields joinFields1, Stream s2, Fields joinFields2, Fields outFields) { + public Stream join(Stream s1, Fields joinFields1, Stream s2, Fields joinFields2, + Fields outFields) { return join(Arrays.asList(s1, s2), Arrays.asList(joinFields1, joinFields2), outFields); } @@ -674,58 +704,75 @@ public Stream join(List streams, List joinFields, Fields outFiel return join(streams, joinFields, outFields, JoinType.INNER); } - public Stream join(Stream s1, Fields joinFields1, Stream s2, Fields joinFields2, Fields outFields, JoinType type) { - return join(Arrays.asList(s1, s2), Arrays.asList(joinFields1, joinFields2), outFields, type); + public Stream join(Stream s1, Fields joinFields1, Stream s2, Fields joinFields2, + Fields outFields, JoinType type) { + return join(Arrays.asList(s1, s2), Arrays.asList(joinFields1, joinFields2), outFields, + type); } - public Stream join(List streams, List joinFields, Fields outFields, JoinType type) { + public Stream join(List streams, List joinFields, Fields outFields, + JoinType type) { return join(streams, joinFields, outFields, repeat(streams.size(), type)); } - public Stream join(Stream s1, Fields joinFields1, Stream s2, Fields joinFields2, Fields outFields, List mixed) { - return join(Arrays.asList(s1, s2), Arrays.asList(joinFields1, joinFields2), outFields, mixed); + public Stream join(Stream s1, Fields joinFields1, Stream s2, Fields joinFields2, + Fields outFields, List mixed) { + return join(Arrays.asList(s1, s2), Arrays.asList(joinFields1, joinFields2), outFields, + mixed); } - public Stream join(List streams, List joinFields, Fields outFields, List mixed) { + public Stream join(List streams, List joinFields, Fields outFields, + List mixed) { return join(streams, joinFields, outFields, mixed, JoinOutFieldsMode.COMPACT); } - public Stream join(Stream s1, Fields joinFields1, Stream s2, Fields joinFields2, Fields outFields, JoinOutFieldsMode mode) { - return join(Arrays.asList(s1, s2), Arrays.asList(joinFields1, joinFields2), outFields, mode); + public Stream join(Stream s1, Fields joinFields1, Stream s2, Fields joinFields2, + Fields outFields, JoinOutFieldsMode mode) { + return join(Arrays.asList(s1, s2), Arrays.asList(joinFields1, joinFields2), outFields, + mode); } - public Stream join(List streams, List joinFields, Fields outFields, JoinOutFieldsMode mode) { + public Stream join(List streams, List joinFields, Fields outFields, + JoinOutFieldsMode mode) { return join(streams, joinFields, outFields, JoinType.INNER, mode); } - public Stream join(Stream s1, Fields joinFields1, Stream s2, Fields joinFields2, Fields outFields, JoinType type, + public Stream join(Stream s1, Fields joinFields1, Stream s2, Fields joinFields2, + Fields outFields, JoinType type, JoinOutFieldsMode mode) { - return join(Arrays.asList(s1, s2), Arrays.asList(joinFields1, joinFields2), outFields, type, mode); + return join(Arrays.asList(s1, s2), Arrays.asList(joinFields1, joinFields2), outFields, type, + mode); } - public Stream join(List streams, List joinFields, Fields outFields, JoinType type, JoinOutFieldsMode mode) { + public Stream join(List streams, List joinFields, Fields outFields, + JoinType type, JoinOutFieldsMode mode) { return join(streams, joinFields, outFields, repeat(streams.size(), type), mode); } - public Stream join(Stream s1, Fields joinFields1, Stream s2, Fields joinFields2, Fields outFields, List mixed, + public Stream join(Stream s1, Fields joinFields1, Stream s2, Fields joinFields2, + Fields outFields, List mixed, JoinOutFieldsMode mode) { - return join(Arrays.asList(s1, s2), Arrays.asList(joinFields1, joinFields2), outFields, mixed, mode); + return join(Arrays.asList(s1, s2), Arrays.asList(joinFields1, joinFields2), outFields, + mixed, mode); } - public Stream join(List streams, List joinFields, Fields outFields, List mixed, JoinOutFieldsMode mode) { + public Stream join(List streams, List joinFields, Fields outFields, + List mixed, JoinOutFieldsMode mode) { switch (mode) { case COMPACT: return multiReduce(strippedInputFields(streams, joinFields), groupedStreams(streams, joinFields), - new JoinerMultiReducer(mixed, joinFields.get(0).size(), strippedInputFields(streams, joinFields)), + new JoinerMultiReducer(mixed, joinFields.get(0).size(), + strippedInputFields(streams, joinFields)), outFields); case PRESERVE: return multiReduce(strippedInputFields(streams, joinFields), groupedStreams(streams, joinFields), - new PreservingFieldsOrderJoinerMultiReducer(mixed, joinFields.get(0).size(), + new PreservingFieldsOrderJoinerMultiReducer(mixed, joinFields + .get(0).size(), getAllOutputFields(streams), joinFields, strippedInputFields(streams, joinFields)), outFields); @@ -787,30 +834,36 @@ public StormTopology build() { Group g2 = grouper.nodeGroup(e.target); // g1 being null means the source is a spout node if (g1 == null && !(e.source instanceof SpoutNode)) { - throw new RuntimeException("Planner exception: Null source group must indicate a spout node at this phase of planning"); + throw new RuntimeException("Planner exception: Null source group must " + + "indicate a spout node at this phase of planning"); } if (g1 == null || !g1.equals(g2)) { graph.removeEdge(e); PartitionNode partitionNode = makeIdentityPartition(e.source); graph.addVertex(partitionNode); - graph.addEdge(e.source, partitionNode, new IndexedEdge(e.source, partitionNode, 0)); - graph.addEdge(partitionNode, e.target, new IndexedEdge(partitionNode, e.target, e.index)); + graph.addEdge(e.source, partitionNode, new IndexedEdge(e.source, partitionNode, + 0)); + graph.addEdge(partitionNode, e.target, new IndexedEdge(partitionNode, e.target, + e.index)); } } } // if one group subscribes to the same stream with same partitioning multiple times, - // merge those together (otherwise can end up with many output streams created for that partitioning + // merge those together (otherwise can end up with many output streams created for that + // partitioning // if need to split into multiple output streams because of same input having different // partitioning to the group) // this is because can't currently merge splitting logic into a spout - // not the most kosher algorithm here, since the grouper indexes are being trounced via the adding of nodes to random groups, but it + // not the most kosher algorithm here, since the grouper indexes are being trounced via the + // adding of nodes to random groups, but it // works out List forNewGroups = new ArrayList<>(); for (Group g : mergedGroups) { for (PartitionNode n : extraPartitionInputs(g)) { Node idNode = makeIdentityNode(n.allOutputFields); - Node newPartitionNode = new PartitionNode(idNode.streamId, n.name, idNode.allOutputFields, n.thriftGrouping); + Node newPartitionNode = new PartitionNode(idNode.streamId, n.name, + idNode.allOutputFields, n.thriftGrouping); graph.removeVertex(n); graph.addVertex(idNode); @@ -889,10 +942,13 @@ public StormTopology build() { } else if (sn.spout instanceof ITridentSpout) { s = (ITridentSpout) sn.spout; } else { - throw new RuntimeException("Regular rich spouts not supported yet... try wrapping in a RichSpoutBatchExecutor"); - // TODO: handle regular rich spout without batches (need lots of updates to support this throughout) + throw new RuntimeException("Regular rich spouts not supported yet... try " + + "wrapping in a RichSpoutBatchExecutor"); + // TODO: handle regular rich spout without batches (need lots of updates to + // support this throughout) } - spoutDeclarer = builder.setSpout(spoutIds.get(sn), sn.streamId, sn.txId, s, parallelism, batchGroupMap.get(sn)); + spoutDeclarer = builder.setSpout(spoutIds.get(sn), sn.streamId, sn.txId, s, + parallelism, batchGroupMap.get(sn)); } if (onHeap != null) { @@ -915,10 +971,12 @@ public StormTopology build() { Map groupRes = g.getResources(resourceDefaults); Number onHeap = groupRes.get(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB); - Number offHeap = groupRes.get(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB); + Number offHeap = groupRes + .get(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB); Number cpuLoad = groupRes.get(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT); - BoltDeclarer d = builder.setBolt(boltIds.get(g), new SubtopologyBolt(graph, g.nodes, batchGroupMap), p, + BoltDeclarer d = builder.setBolt(boltIds.get(g), new SubtopologyBolt(graph, g.nodes, + batchGroupMap), p, committerBatches(g, batchGroupMap), streamToGroup); if (onHeap != null) { @@ -954,7 +1012,8 @@ public StormTopology build() { private Node makeIdentityNode(Fields allOutputFields) { return new ProcessorNode(getUniqueStreamId(), null, allOutputFields, new Fields(), - new EachProcessor(new Fields(), new FilterExecutor(new TrueFilter()))); + new EachProcessor(new Fields(), + new FilterExecutor(new TrueFilter()))); } protected String getUniqueStreamId() { diff --git a/storm-client/src/jvm/org/apache/storm/trident/drpc/ReturnResultsReducer.java b/storm-client/src/jvm/org/apache/storm/trident/drpc/ReturnResultsReducer.java index 520cb487ed1..de5213f4aae 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/drpc/ReturnResultsReducer.java +++ b/storm-client/src/jvm/org/apache/storm/trident/drpc/ReturnResultsReducer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -50,7 +56,8 @@ public ReturnResultsState init(TridentCollector collector) { } @Override - public void execute(ReturnResultsState state, int streamIndex, TridentTuple input, TridentCollector collector) { + public void execute(ReturnResultsState state, int streamIndex, TridentTuple input, + TridentCollector collector) { if (streamIndex == 0) { state.returnInfo = input.getString(0); } else { diff --git a/storm-client/src/jvm/org/apache/storm/trident/fluent/ChainedAggregatorDeclarer.java b/storm-client/src/jvm/org/apache/storm/trident/fluent/ChainedAggregatorDeclarer.java index 20fe5b33d43..b87ad85258c 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/fluent/ChainedAggregatorDeclarer.java +++ b/storm-client/src/jvm/org/apache/storm/trident/fluent/ChainedAggregatorDeclarer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -24,19 +30,19 @@ import org.apache.storm.trident.operation.impl.CombinerAggregatorCombineImpl; import org.apache.storm.trident.operation.impl.CombinerAggregatorInitImpl; import org.apache.storm.trident.operation.impl.ReducerAggregatorImpl; -import org.apache.storm.trident.operation.impl.SingleEmitAggregator; import org.apache.storm.trident.operation.impl.SingleEmitAggregator.BatchToPartition; +import org.apache.storm.trident.operation.impl.SingleEmitAggregator; import org.apache.storm.trident.tuple.ComboList; import org.apache.storm.tuple.Fields; - public class ChainedAggregatorDeclarer implements ChainedFullAggregatorDeclarer, ChainedPartitionAggregatorDeclarer { List aggs = new ArrayList<>(); IAggregatableStream stream; AggType type = null; GlobalAggregationScheme globalScheme; - public ChainedAggregatorDeclarer(IAggregatableStream stream, GlobalAggregationScheme globalScheme) { + public ChainedAggregatorDeclarer(IAggregatableStream stream, + GlobalAggregationScheme globalScheme) { this.stream = stream; this.globalScheme = globalScheme; } @@ -66,12 +72,14 @@ public Stream chainEnd() { allInFields.addAll(infields.toList()); } if (new HashSet(allOutFields).size() != allOutFields.size()) { - throw new IllegalArgumentException("Output fields for chained aggregators must be distinct: " + allOutFields.toString()); + throw new IllegalArgumentException("Output fields for chained aggregators must be " + + "distinct: " + allOutFields.toString()); } Fields inFields = new Fields(new ArrayList<>(allInFields)); Fields outFields = new Fields(allOutFields); - Aggregator combined = new ChainedAggregatorImpl(aggs, inputFields, new ComboList.Factory(outSizes)); + Aggregator combined = new ChainedAggregatorImpl(aggs, inputFields, new ComboList + .Factory(outSizes)); if (type != AggType.FULL) { stream = stream.partitionAggregate(inFields, combined, outFields); @@ -91,35 +99,42 @@ public Stream chainEnd() { } @Override - public ChainedPartitionAggregatorDeclarer partitionAggregate(Aggregator agg, Fields functionFields) { + public ChainedPartitionAggregatorDeclarer partitionAggregate(Aggregator agg, + Fields functionFields) { return partitionAggregate(null, agg, functionFields); } @Override - public ChainedPartitionAggregatorDeclarer partitionAggregate(Fields inputFields, Aggregator agg, Fields functionFields) { + public ChainedPartitionAggregatorDeclarer partitionAggregate(Fields inputFields, Aggregator agg, + Fields functionFields) { type = AggType.PARTITION; aggs.add(new AggSpec(inputFields, agg, functionFields)); return this; } @Override - public ChainedPartitionAggregatorDeclarer partitionAggregate(CombinerAggregator agg, Fields functionFields) { + public ChainedPartitionAggregatorDeclarer partitionAggregate(CombinerAggregator agg, + Fields functionFields) { return partitionAggregate(null, agg, functionFields); } @Override - public ChainedPartitionAggregatorDeclarer partitionAggregate(Fields inputFields, CombinerAggregator agg, Fields functionFields) { + public ChainedPartitionAggregatorDeclarer partitionAggregate(Fields inputFields, + CombinerAggregator agg, Fields functionFields) { initCombiner(inputFields, agg, functionFields); - return partitionAggregate(functionFields, new CombinerAggregatorCombineImpl(agg), functionFields); + return partitionAggregate(functionFields, new CombinerAggregatorCombineImpl(agg), + functionFields); } @Override - public ChainedPartitionAggregatorDeclarer partitionAggregate(ReducerAggregator agg, Fields functionFields) { + public ChainedPartitionAggregatorDeclarer partitionAggregate(ReducerAggregator agg, + Fields functionFields) { return partitionAggregate(null, agg, functionFields); } @Override - public ChainedPartitionAggregatorDeclarer partitionAggregate(Fields inputFields, ReducerAggregator agg, Fields functionFields) { + public ChainedPartitionAggregatorDeclarer partitionAggregate(Fields inputFields, + ReducerAggregator agg, Fields functionFields) { return partitionAggregate(inputFields, new ReducerAggregatorImpl(agg), functionFields); } @@ -129,11 +144,13 @@ public ChainedFullAggregatorDeclarer aggregate(Aggregator agg, Fields functionFi } @Override - public ChainedFullAggregatorDeclarer aggregate(Fields inputFields, Aggregator agg, Fields functionFields) { + public ChainedFullAggregatorDeclarer aggregate(Fields inputFields, Aggregator agg, + Fields functionFields) { return aggregate(inputFields, agg, functionFields, false); } - private ChainedFullAggregatorDeclarer aggregate(Fields inputFields, Aggregator agg, Fields functionFields, boolean isCombiner) { + private ChainedFullAggregatorDeclarer aggregate(Fields inputFields, Aggregator agg, + Fields functionFields, boolean isCombiner) { if (isCombiner) { if (type == null) { type = AggType.FULL_COMBINE; @@ -151,9 +168,11 @@ public ChainedFullAggregatorDeclarer aggregate(CombinerAggregator agg, Fields fu } @Override - public ChainedFullAggregatorDeclarer aggregate(Fields inputFields, CombinerAggregator agg, Fields functionFields) { + public ChainedFullAggregatorDeclarer aggregate(Fields inputFields, CombinerAggregator agg, + Fields functionFields) { initCombiner(inputFields, agg, functionFields); - return aggregate(functionFields, new CombinerAggregatorCombineImpl(agg), functionFields, true); + return aggregate(functionFields, new CombinerAggregatorCombineImpl(agg), functionFields, + true); } @Override @@ -162,7 +181,8 @@ public ChainedFullAggregatorDeclarer aggregate(ReducerAggregator agg, Fields fun } @Override - public ChainedFullAggregatorDeclarer aggregate(Fields inputFields, ReducerAggregator agg, Fields functionFields) { + public ChainedFullAggregatorDeclarer aggregate(Fields inputFields, ReducerAggregator agg, + Fields functionFields) { return aggregate(inputFields, new ReducerAggregatorImpl(agg), functionFields); } @@ -180,7 +200,8 @@ public interface AggregationPartition { Stream partition(Stream input); } - // inputFields can be equal to outFields, but multiple aggregators cannot have intersection outFields + // inputFields can be equal to outFields, but multiple aggregators cannot have intersection + // outFields private static class AggSpec { Fields inFields; Aggregator agg; diff --git a/storm-client/src/jvm/org/apache/storm/trident/fluent/ChainedFullAggregatorDeclarer.java b/storm-client/src/jvm/org/apache/storm/trident/fluent/ChainedFullAggregatorDeclarer.java index 4c36e598769..bab2f9d40bb 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/fluent/ChainedFullAggregatorDeclarer.java +++ b/storm-client/src/jvm/org/apache/storm/trident/fluent/ChainedFullAggregatorDeclarer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -20,13 +26,16 @@ public interface ChainedFullAggregatorDeclarer extends IChainedAggregatorDeclarer { ChainedFullAggregatorDeclarer aggregate(Aggregator agg, Fields functionFields); - ChainedFullAggregatorDeclarer aggregate(Fields inputFields, Aggregator agg, Fields functionFields); + ChainedFullAggregatorDeclarer aggregate(Fields inputFields, Aggregator agg, + Fields functionFields); ChainedFullAggregatorDeclarer aggregate(CombinerAggregator agg, Fields functionFields); - ChainedFullAggregatorDeclarer aggregate(Fields inputFields, CombinerAggregator agg, Fields functionFields); + ChainedFullAggregatorDeclarer aggregate(Fields inputFields, CombinerAggregator agg, + Fields functionFields); ChainedFullAggregatorDeclarer aggregate(ReducerAggregator agg, Fields functionFields); - ChainedFullAggregatorDeclarer aggregate(Fields inputFields, ReducerAggregator agg, Fields functionFields); + ChainedFullAggregatorDeclarer aggregate(Fields inputFields, ReducerAggregator agg, + Fields functionFields); } diff --git a/storm-client/src/jvm/org/apache/storm/trident/fluent/ChainedPartitionAggregatorDeclarer.java b/storm-client/src/jvm/org/apache/storm/trident/fluent/ChainedPartitionAggregatorDeclarer.java index dbc6a7b1c95..9fd3f248381 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/fluent/ChainedPartitionAggregatorDeclarer.java +++ b/storm-client/src/jvm/org/apache/storm/trident/fluent/ChainedPartitionAggregatorDeclarer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -20,13 +26,18 @@ public interface ChainedPartitionAggregatorDeclarer extends IChainedAggregatorDeclarer { ChainedPartitionAggregatorDeclarer partitionAggregate(Aggregator agg, Fields functionFields); - ChainedPartitionAggregatorDeclarer partitionAggregate(Fields inputFields, Aggregator agg, Fields functionFields); + ChainedPartitionAggregatorDeclarer partitionAggregate(Fields inputFields, Aggregator agg, + Fields functionFields); - ChainedPartitionAggregatorDeclarer partitionAggregate(CombinerAggregator agg, Fields functionFields); + ChainedPartitionAggregatorDeclarer partitionAggregate(CombinerAggregator agg, + Fields functionFields); - ChainedPartitionAggregatorDeclarer partitionAggregate(Fields inputFields, CombinerAggregator agg, Fields functionFields); + ChainedPartitionAggregatorDeclarer partitionAggregate(Fields inputFields, + CombinerAggregator agg, Fields functionFields); - ChainedPartitionAggregatorDeclarer partitionAggregate(ReducerAggregator agg, Fields functionFields); + ChainedPartitionAggregatorDeclarer partitionAggregate(ReducerAggregator agg, + Fields functionFields); - ChainedPartitionAggregatorDeclarer partitionAggregate(Fields inputFields, ReducerAggregator agg, Fields functionFields); + ChainedPartitionAggregatorDeclarer partitionAggregate(Fields inputFields, ReducerAggregator agg, + Fields functionFields); } diff --git a/storm-client/src/jvm/org/apache/storm/trident/fluent/GlobalAggregationScheme.java b/storm-client/src/jvm/org/apache/storm/trident/fluent/GlobalAggregationScheme.java index dc9bb8b8fce..4c7d3a6b2e7 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/fluent/GlobalAggregationScheme.java +++ b/storm-client/src/jvm/org/apache/storm/trident/fluent/GlobalAggregationScheme.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -14,7 +20,6 @@ import org.apache.storm.trident.operation.impl.SingleEmitAggregator.BatchToPartition; - public interface GlobalAggregationScheme { IAggregatableStream aggPartition(S stream); // how to partition for second stage of aggregation diff --git a/storm-client/src/jvm/org/apache/storm/trident/fluent/GroupedStream.java b/storm-client/src/jvm/org/apache/storm/trident/fluent/GroupedStream.java index 2c8dcf31570..cb84d638c3d 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/fluent/GroupedStream.java +++ b/storm-client/src/jvm/org/apache/storm/trident/fluent/GroupedStream.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -28,7 +34,6 @@ import org.apache.storm.trident.util.TridentUtils; import org.apache.storm.tuple.Fields; - public class GroupedStream implements IAggregatableStream, GlobalAggregationScheme { Fields groupFields; Stream stream; @@ -76,19 +81,23 @@ public Stream aggregate(Fields inputFields, ReducerAggregator agg, Fields functi .chainEnd(); } - public TridentState persistentAggregate(StateFactory stateFactory, CombinerAggregator agg, Fields functionFields) { + public TridentState persistentAggregate(StateFactory stateFactory, CombinerAggregator agg, + Fields functionFields) { return persistentAggregate(new StateSpec(stateFactory), agg, functionFields); } - public TridentState persistentAggregate(StateSpec spec, CombinerAggregator agg, Fields functionFields) { + public TridentState persistentAggregate(StateSpec spec, CombinerAggregator agg, + Fields functionFields) { return persistentAggregate(spec, null, agg, functionFields); } - public TridentState persistentAggregate(StateFactory stateFactory, Fields inputFields, CombinerAggregator agg, Fields functionFields) { + public TridentState persistentAggregate(StateFactory stateFactory, Fields inputFields, + CombinerAggregator agg, Fields functionFields) { return persistentAggregate(new StateSpec(stateFactory), inputFields, agg, functionFields); } - public TridentState persistentAggregate(StateSpec spec, Fields inputFields, CombinerAggregator agg, Fields functionFields) { + public TridentState persistentAggregate(StateSpec spec, Fields inputFields, + CombinerAggregator agg, Fields functionFields) { return aggregate(inputFields, agg, functionFields) .partitionPersist(spec, TridentUtils.fieldsUnion(groupFields, functionFields), @@ -96,27 +105,33 @@ public TridentState persistentAggregate(StateSpec spec, Fields inputFields, Comb TridentUtils.fieldsConcat(groupFields, functionFields)); } - public TridentState persistentAggregate(StateFactory stateFactory, Fields inputFields, ReducerAggregator agg, Fields functionFields) { + public TridentState persistentAggregate(StateFactory stateFactory, Fields inputFields, + ReducerAggregator agg, Fields functionFields) { return persistentAggregate(new StateSpec(stateFactory), inputFields, agg, functionFields); } - public TridentState persistentAggregate(StateSpec spec, Fields inputFields, ReducerAggregator agg, Fields functionFields) { + public TridentState persistentAggregate(StateSpec spec, Fields inputFields, + ReducerAggregator agg, Fields functionFields) { return stream.partitionBy(groupFields) .partitionPersist(spec, TridentUtils.fieldsUnion(groupFields, inputFields), - new MapReducerAggStateUpdater(agg, groupFields, inputFields), + new MapReducerAggStateUpdater(agg, groupFields, + inputFields), TridentUtils.fieldsConcat(groupFields, functionFields)); } - public TridentState persistentAggregate(StateFactory stateFactory, ReducerAggregator agg, Fields functionFields) { + public TridentState persistentAggregate(StateFactory stateFactory, ReducerAggregator agg, + Fields functionFields) { return persistentAggregate(new StateSpec(stateFactory), agg, functionFields); } - public TridentState persistentAggregate(StateSpec spec, ReducerAggregator agg, Fields functionFields) { + public TridentState persistentAggregate(StateSpec spec, ReducerAggregator agg, + Fields functionFields) { return persistentAggregate(spec, null, agg, functionFields); } - public Stream stateQuery(TridentState state, Fields inputFields, QueryFunction function, Fields functionFields) { + public Stream stateQuery(TridentState state, Fields inputFields, QueryFunction function, + Fields functionFields) { return stream.partitionBy(groupFields) .stateQuery(state, inputFields, @@ -135,8 +150,10 @@ public IAggregatableStream each(Fields inputFields, Function function, Fields fu } @Override - public IAggregatableStream partitionAggregate(Fields inputFields, Aggregator agg, Fields functionFields) { - Aggregator groupedAgg = new GroupedAggregator(agg, groupFields, inputFields, functionFields.size()); + public IAggregatableStream partitionAggregate(Fields inputFields, Aggregator agg, + Fields functionFields) { + Aggregator groupedAgg = new GroupedAggregator(agg, groupFields, inputFields, functionFields + .size()); Fields allInFields = TridentUtils.fieldsUnion(groupFields, inputFields); Fields allOutFields = TridentUtils.fieldsConcat(groupFields, functionFields); Stream s = stream.partitionAggregate(allInFields, groupedAgg, allOutFields); diff --git a/storm-client/src/jvm/org/apache/storm/trident/fluent/IAggregatableStream.java b/storm-client/src/jvm/org/apache/storm/trident/fluent/IAggregatableStream.java index 6ff814be536..c581a398520 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/fluent/IAggregatableStream.java +++ b/storm-client/src/jvm/org/apache/storm/trident/fluent/IAggregatableStream.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -20,7 +26,8 @@ public interface IAggregatableStream { IAggregatableStream each(Fields inputFields, Function function, Fields functionFields); - IAggregatableStream partitionAggregate(Fields inputFields, Aggregator agg, Fields functionFields); + IAggregatableStream partitionAggregate(Fields inputFields, Aggregator agg, + Fields functionFields); Stream toStream(); diff --git a/storm-client/src/jvm/org/apache/storm/trident/fluent/IChainedAggregatorDeclarer.java b/storm-client/src/jvm/org/apache/storm/trident/fluent/IChainedAggregatorDeclarer.java index 8808c94f4a1..42aa5583832 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/fluent/IChainedAggregatorDeclarer.java +++ b/storm-client/src/jvm/org/apache/storm/trident/fluent/IChainedAggregatorDeclarer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/fluent/UniqueIdGen.java b/storm-client/src/jvm/org/apache/storm/trident/fluent/UniqueIdGen.java index f04e76ae40b..4018453b714 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/fluent/UniqueIdGen.java +++ b/storm-client/src/jvm/org/apache/storm/trident/fluent/UniqueIdGen.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/graph/GraphGrouper.java b/storm-client/src/jvm/org/apache/storm/trident/graph/GraphGrouper.java index 3a890b70c1f..f3ebb223050 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/graph/GraphGrouper.java +++ b/storm-client/src/jvm/org/apache/storm/trident/graph/GraphGrouper.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/graph/Group.java b/storm-client/src/jvm/org/apache/storm/trident/graph/Group.java index 8dc359f3910..5188d4e9ac7 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/graph/Group.java +++ b/storm-client/src/jvm/org/apache/storm/trident/graph/Group.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -63,6 +69,7 @@ public Set incomingNodes() { /** * Get shared memory. + * * @return the shared memory requests for the entire group */ public Set getSharedMemory() { @@ -74,7 +81,8 @@ public Set getSharedMemory() { } /** - * In case no resources are specified, returns empty map. In case differing types of resources are specified, throw. Otherwise, add all + * In case no resources are specified, returns empty map. In case differing types of resources + * are specified, throw. Otherwise, add all * the resources for a group. */ public Map getResources(Map defaults) { @@ -100,16 +108,19 @@ public Map getResources(Map defaults) { for (Node nod : nodes) { Set resourceKeys = new HashSet<>(defaults.keySet()); resourceKeys.addAll(nod.getResources().keySet()); - ops.append("\t[ " + nod.shortString() + ", Resources Set: " + resourceKeys + " ]\n"); + ops.append("\t[ " + nod.shortString() + ", Resources Set: " + resourceKeys + + " ]\n"); } if (nodeRes.keySet().containsAll(resources.keySet())) { Set diffset = new HashSet<>(nodeRes.keySet()); diffset.removeAll(resources.keySet()); throw new RuntimeException( - "Found an operation with resources set which are not set in other operations in the group:\n" + "Found an operation with resources set which are not set in other " + + "operations in the group:\n" + "\t[ " + n.shortString() + " ]: " + diffset + "\n" - + "Either set these resources in all other operations in the group, add a default " + + "Either set these resources in all other operations in the " + + "group, add a default " + "setting, or remove the setting from this operation.\n" + "The group at fault:\n" + ops); @@ -117,9 +128,11 @@ public Map getResources(Map defaults) { Set diffset = new HashSet<>(resources.keySet()); diffset.removeAll(nodeRes.keySet()); throw new RuntimeException( - "Found an operation with resources unset which are set in other operations in the group:\n" + "Found an operation with resources unset which are set in other " + + "operations in the group:\n" + "\t[ " + n.shortString() + " ]: " + diffset + "\n" - + "Either set these resources in all other operations in the group, add a default " + + "Either set these resources in all other operations in the " + + "group, add a default " + "setting, or remove the setting from all other operations.\n" + "The group at fault:\n" + ops); @@ -130,7 +143,8 @@ public Map getResources(Map defaults) { String key = kv.getKey(); Number val = kv.getValue(); - Number newval = new Double(val.doubleValue() + resources.get(key).doubleValue()); + Number newval = new Double(val.doubleValue() + resources.get(key) + .doubleValue()); resources.put(key, newval); } } diff --git a/storm-client/src/jvm/org/apache/storm/trident/operation/Aggregator.java b/storm-client/src/jvm/org/apache/storm/trident/operation/Aggregator.java index ecfacbf4062..debbe21dcb7 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/operation/Aggregator.java +++ b/storm-client/src/jvm/org/apache/storm/trident/operation/Aggregator.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/operation/Assembly.java b/storm-client/src/jvm/org/apache/storm/trident/operation/Assembly.java index d4d71725ce1..ca29791a0bd 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/operation/Assembly.java +++ b/storm-client/src/jvm/org/apache/storm/trident/operation/Assembly.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -14,9 +20,9 @@ import org.apache.storm.trident.Stream; - /** - * The `Assembly` interface provides a means to encapsulate logic applied to a {@link org.apache.storm.trident.Stream}. + * The `Assembly` interface provides a means to encapsulate logic applied to a {@link + * org.apache.storm.trident.Stream}. * *

    Usage: * diff --git a/storm-client/src/jvm/org/apache/storm/trident/operation/BaseAggregator.java b/storm-client/src/jvm/org/apache/storm/trident/operation/BaseAggregator.java index 9d82b2b637f..5c9cc1aeac4 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/operation/BaseAggregator.java +++ b/storm-client/src/jvm/org/apache/storm/trident/operation/BaseAggregator.java @@ -1,18 +1,23 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.trident.operation; - public abstract class BaseAggregator extends BaseOperation implements Aggregator { } diff --git a/storm-client/src/jvm/org/apache/storm/trident/operation/BaseFilter.java b/storm-client/src/jvm/org/apache/storm/trident/operation/BaseFilter.java index 8125a122a02..5a6f43b8f9b 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/operation/BaseFilter.java +++ b/storm-client/src/jvm/org/apache/storm/trident/operation/BaseFilter.java @@ -1,18 +1,23 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.trident.operation; - public abstract class BaseFilter extends BaseOperation implements Filter { } diff --git a/storm-client/src/jvm/org/apache/storm/trident/operation/BaseFunction.java b/storm-client/src/jvm/org/apache/storm/trident/operation/BaseFunction.java index 8cb6608cf4f..8ab8d873d62 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/operation/BaseFunction.java +++ b/storm-client/src/jvm/org/apache/storm/trident/operation/BaseFunction.java @@ -1,18 +1,23 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.trident.operation; - public abstract class BaseFunction extends BaseOperation implements Function { } diff --git a/storm-client/src/jvm/org/apache/storm/trident/operation/BaseMultiReducer.java b/storm-client/src/jvm/org/apache/storm/trident/operation/BaseMultiReducer.java index 3dfc8cef8c8..aca4e4f677b 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/operation/BaseMultiReducer.java +++ b/storm-client/src/jvm/org/apache/storm/trident/operation/BaseMultiReducer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -20,7 +26,6 @@ public abstract class BaseMultiReducer implements MultiReducer { public void prepare(Map conf, TridentMultiReducerContext context) { } - @Override public void cleanup() { } diff --git a/storm-client/src/jvm/org/apache/storm/trident/operation/BaseOperation.java b/storm-client/src/jvm/org/apache/storm/trident/operation/BaseOperation.java index 413ede42422..7843913de88 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/operation/BaseOperation.java +++ b/storm-client/src/jvm/org/apache/storm/trident/operation/BaseOperation.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -25,7 +31,8 @@ public class BaseOperation implements Operation { * No-op implementation. * * @param conf the Storm configuration map - * @param context the operation context which provides information such as the number of partitions in the stream, and the current + * @param context the operation context which provides information such as the number of + * partitions in the stream, and the current * partition index. It also provides methods for registering operation-specific metrics. */ @Override diff --git a/storm-client/src/jvm/org/apache/storm/trident/operation/CombinerAggregator.java b/storm-client/src/jvm/org/apache/storm/trident/operation/CombinerAggregator.java index add6657e4ee..38085f434ec 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/operation/CombinerAggregator.java +++ b/storm-client/src/jvm/org/apache/storm/trident/operation/CombinerAggregator.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/operation/Consumer.java b/storm-client/src/jvm/org/apache/storm/trident/operation/Consumer.java index 55a102235e0..781aa33a340 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/operation/Consumer.java +++ b/storm-client/src/jvm/org/apache/storm/trident/operation/Consumer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -16,7 +22,8 @@ import org.apache.storm.trident.tuple.TridentTuple; /** - * Represents an operation that accepts a single input argument and returns no result. This is similar to the Consumer interface in Java 8. + * Represents an operation that accepts a single input argument and returns no result. This is + * similar to the Consumer interface in Java 8. */ public interface Consumer extends Serializable { /** diff --git a/storm-client/src/jvm/org/apache/storm/trident/operation/DefaultResourceDeclarer.java b/storm-client/src/jvm/org/apache/storm/trident/operation/DefaultResourceDeclarer.java index f21d59c738f..8a484d83547 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/operation/DefaultResourceDeclarer.java +++ b/storm-client/src/jvm/org/apache/storm/trident/operation/DefaultResourceDeclarer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -22,14 +28,19 @@ /** * Default implementation of resources declarer. - * @param Must always be the type of the extending class. i.e. public class SubResourceDeclarer extends + * + * @param Must always be the type of the extending class. i.e. public class + * SubResourceDeclarer extends * DefaultResourceDeclarer<SubResourceDeclarer> {...} */ public class DefaultResourceDeclarer implements ResourceDeclarer, ITridentResource { - //@{link org.apache.storm.trident.planner.Node} and several other trident classes inherit from DefaultResourceDeclarer - // These classes are serialized out as part of the bolts and spouts of a topology, often for each bolt/spout in the topology. - // The following are marked as transient because they are never used after the topology is created so keeping them around just wastes + // @{link org.apache.storm.trident.planner.Node} and several other trident classes inherit from + // DefaultResourceDeclarer + // These classes are serialized out as part of the bolts and spouts of a topology, often for + // each bolt/spout in the topology. + // The following are marked as transient because they are never used after the topology is + // created so keeping them around just wastes // space in the serialized topology private final transient Map resources = new HashMap<>(); private final transient Set sharedMemory = new HashSet<>(); diff --git a/storm-client/src/jvm/org/apache/storm/trident/operation/EachOperation.java b/storm-client/src/jvm/org/apache/storm/trident/operation/EachOperation.java index fab762dde15..ab9c66299a9 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/operation/EachOperation.java +++ b/storm-client/src/jvm/org/apache/storm/trident/operation/EachOperation.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/operation/Filter.java b/storm-client/src/jvm/org/apache/storm/trident/operation/Filter.java index a6f09409571..e698ea6c397 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/operation/Filter.java +++ b/storm-client/src/jvm/org/apache/storm/trident/operation/Filter.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -18,11 +24,14 @@ /** * Filters take in a tuple as input and decide whether or not to keep that tuple or not. * - *

    If the `isKeep()` method of a Filter returns `false` for a tuple, that tuple will be filtered out of the Stream + *

    If the `isKeep()` method of a Filter returns `false` for a tuple, that tuple will be filtered + * out of the Stream * * - *

    ### Configuration If your `Filter` implementation has configuration requirements, you will typically want to extend {@link - * org.apache.storm.trident.operation.BaseFilter} and override the {@link org.apache.storm.trident.operation.Operation#prepare(Map, + *

    ### Configuration If your `Filter` implementation has configuration requirements, you will + * typically want to extend {@link + * org.apache.storm.trident.operation.BaseFilter} and override the {@link + * org.apache.storm.trident.operation.Operation#prepare(Map, * TridentOperationContext)} method to perform your custom initialization. * * @see org.apache.storm.trident.Stream diff --git a/storm-client/src/jvm/org/apache/storm/trident/operation/FlatMapFunction.java b/storm-client/src/jvm/org/apache/storm/trident/operation/FlatMapFunction.java index 341f98b67cc..08566e5a338 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/operation/FlatMapFunction.java +++ b/storm-client/src/jvm/org/apache/storm/trident/operation/FlatMapFunction.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/operation/Function.java b/storm-client/src/jvm/org/apache/storm/trident/operation/Function.java index 5f4dd635953..2f42b22f10f 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/operation/Function.java +++ b/storm-client/src/jvm/org/apache/storm/trident/operation/Function.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -16,19 +22,24 @@ import org.apache.storm.trident.tuple.TridentTuple; /** - * A function takes in a set of input fields and emits zero or more tuples as output. The fields of the output tuple are appended to the - * original input tuple in the stream. If a function emits no tuples, the original input tuple is filtered out. Otherwise, the input tuple + * A function takes in a set of input fields and emits zero or more tuples as output. The fields of + * the output tuple are appended to the + * original input tuple in the stream. If a function emits no tuples, the original input tuple is + * filtered out. Otherwise, the input tuple * is duplicated for each output tuple. * *

    For example, if you have the following function: * - *

    ```java public class MyFunction extends BaseFunction { public void execute(TridentTuple tuple, TridentCollector collector) + *

    ```java public class MyFunction extends BaseFunction { public void execute(TridentTuple tuple, + * TridentCollector collector) * { for(int i=0; i < tuple.getInteger(0); i++) { collector.emit(new Values(i)); } } } * ``` * - *

    Now suppose you have a stream in the variable `mystream` with the fields `["a", "b", "c"]` with the following tuples: + *

    Now suppose you have a stream in the variable `mystream` with the fields `["a", "b", "c"]` + * with the following tuples: * - *

    ``` [1, 2, 3] [4, 1, 6] [3, 0, 8] ``` If you had the following code in your topology definition: + *

    ``` [1, 2, 3] [4, 1, 6] [3, 0, 8] ``` If you had the following code in your topology + * definition: * *

    ```java mystream.each(new Fields("b"), new MyFunction(), new Fields("d"))) ``` * @@ -36,19 +47,28 @@ * *

    ``` [1, 2, 3, 0] [1, 2, 3, 1] [4, 1, 6, 0] ``` * - *

    In this case, the parameter `new Fields("b")` tells Trident that you would like to select the field "b" as input to the function, and - * that will be the only field in the Tuple passed to the `execute()` method. The value of "b" in the first tuple (2) causes the for loop to - * execute twice, so 2 tuples are emitted. similarly the second tuple causes one tuple to be emitted. For the third tuple, the value of 0 - * causes the `for` loop to be skipped, so nothing is emitted and the incoming tuple is filtered out of the stream. + *

    In this case, the parameter `new Fields("b")` tells Trident that you would like to select the + * field "b" as input to the function, and + * that will be the only field in the Tuple passed to the `execute()` method. The value of "b" in + * the first tuple (2) causes the for loop to + * execute twice, so 2 tuples are emitted. similarly the second tuple causes one tuple to be + * emitted. For the third tuple, the value of 0 + * causes the `for` loop to be skipped, so nothing is emitted and the incoming tuple is filtered out + * of the stream. * - *

    ### Configuration If your `Function` implementation has configuration requirements, you will typically want to extend {@link - * org.apache.storm.trident.operation.BaseFunction} and override the {@link org.apache.storm.trident.operation.Operation#prepare(Map, + *

    ### Configuration If your `Function` implementation has configuration requirements, you will + * typically want to extend {@link + * org.apache.storm.trident.operation.BaseFunction} and override the {@link + * org.apache.storm.trident.operation.Operation#prepare(Map, * TridentOperationContext)} method to perform your custom initialization. * - *

    ### Performance Considerations Because Trident Functions perform logic on individual tuples -- as opposed to - * batches -- it is advisable to avoid expensive operations such as database operations in a Function, if possible. For + *

    ### Performance Considerations Because Trident Functions perform logic on individual tuples -- + * as opposed to + * batches -- it is advisable to avoid expensive operations such as database operations in a + * Function, if possible. For * data store interactions it is better to use a {@link org.apache.storm.trident.state.State} or - * {@link org.apache.storm.trident.state.QueryFunction} implementation since Trident states operate on batch partitions + * {@link org.apache.storm.trident.state.QueryFunction} implementation since Trident states operate + * on batch partitions * and can perform bulk updates to a database. */ public interface Function extends EachOperation { diff --git a/storm-client/src/jvm/org/apache/storm/trident/operation/GroupedMultiReducer.java b/storm-client/src/jvm/org/apache/storm/trident/operation/GroupedMultiReducer.java index e0495319511..0b794287fbd 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/operation/GroupedMultiReducer.java +++ b/storm-client/src/jvm/org/apache/storm/trident/operation/GroupedMultiReducer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -16,13 +22,13 @@ import java.util.Map; import org.apache.storm.trident.tuple.TridentTuple; - public interface GroupedMultiReducer extends Serializable { void prepare(Map conf, TridentMultiReducerContext context); T init(TridentCollector collector, TridentTuple group); - void execute(T state, int streamIndex, TridentTuple group, TridentTuple input, TridentCollector collector); + void execute(T state, int streamIndex, TridentTuple group, TridentTuple input, + TridentCollector collector); void complete(T state, TridentTuple group, TridentCollector collector); diff --git a/storm-client/src/jvm/org/apache/storm/trident/operation/ITridentResource.java b/storm-client/src/jvm/org/apache/storm/trident/operation/ITridentResource.java index f6c1d7866ef..3df513fbd79 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/operation/ITridentResource.java +++ b/storm-client/src/jvm/org/apache/storm/trident/operation/ITridentResource.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -17,18 +23,21 @@ import org.apache.storm.generated.SharedMemory; /** - * This interface is implemented by various Trident classes in order to gather and propogate resources that have been set on them. + * This interface is implemented by various Trident classes in order to gather and propogate + * resources that have been set on them. * See {@link org.apache.storm.topology.ResourceDeclarer} */ public interface ITridentResource { /** * Get resource. + * * @return a name of resource name -> amount of that resource. *Return should never be null!* */ Map getResources(); /** * Get shared memory. + * * @return the shared memory region requests */ Set getSharedMemory(); diff --git a/storm-client/src/jvm/org/apache/storm/trident/operation/MapFunction.java b/storm-client/src/jvm/org/apache/storm/trident/operation/MapFunction.java index 300d2b92edc..2887bbfa167 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/operation/MapFunction.java +++ b/storm-client/src/jvm/org/apache/storm/trident/operation/MapFunction.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/operation/MultiReducer.java b/storm-client/src/jvm/org/apache/storm/trident/operation/MultiReducer.java index 7b9d78cac6a..0b46df83ed9 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/operation/MultiReducer.java +++ b/storm-client/src/jvm/org/apache/storm/trident/operation/MultiReducer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -16,7 +22,6 @@ import java.util.Map; import org.apache.storm.trident.tuple.TridentTuple; - public interface MultiReducer extends Serializable { void prepare(Map conf, TridentMultiReducerContext context); diff --git a/storm-client/src/jvm/org/apache/storm/trident/operation/Operation.java b/storm-client/src/jvm/org/apache/storm/trident/operation/Operation.java index d3e1e7af863..2c1a7512f2e 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/operation/Operation.java +++ b/storm-client/src/jvm/org/apache/storm/trident/operation/Operation.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -18,9 +24,12 @@ /** * Parent interface for Trident `Filter`s and `Function`s. * - *

    `Operation` defines two lifecycle methods for Trident components. The `prepare()` method is called once when the `Operation` is first - * initialized. The `cleanup()` method is called in local mode when the local cluster is being shut down. In distributed mode, the - * `cleanup()` method is not guaranteed to be called in every situation, but Storm will make a best effort call `cleanup()` whenever + *

    `Operation` defines two lifecycle methods for Trident components. The `prepare()` method is + * called once when the `Operation` is first + * initialized. The `cleanup()` method is called in local mode when the local cluster is being shut + * down. In distributed mode, the + * `cleanup()` method is not guaranteed to be called in every situation, but Storm will make a best + * effort call `cleanup()` whenever * possible. */ public interface Operation extends Serializable { @@ -28,7 +37,8 @@ public interface Operation extends Serializable { * Called when the `Operation` is first initialized. * * @param conf the Storm configuration map - * @param context the operation context which provides information such as the number of partitions in the stream, and the current + * @param context the operation context which provides information such as the number of + * partitions in the stream, and the current * partition index. It also provides methods for registering operation-specific metrics. * @see org.apache.storm.trident.operation.TridentOperationContext */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/operation/OperationAwareFlatMapFunction.java b/storm-client/src/jvm/org/apache/storm/trident/operation/OperationAwareFlatMapFunction.java index eebcf5d40cc..87745268960 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/operation/OperationAwareFlatMapFunction.java +++ b/storm-client/src/jvm/org/apache/storm/trident/operation/OperationAwareFlatMapFunction.java @@ -1,19 +1,26 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.trident.operation; /** - * A one to many transformation function which is aware of Operation (lifecycle of the Trident component). + * A one to many transformation function which is aware of Operation (lifecycle of the Trident + * component). */ public interface OperationAwareFlatMapFunction extends FlatMapFunction, Operation { } diff --git a/storm-client/src/jvm/org/apache/storm/trident/operation/OperationAwareMapFunction.java b/storm-client/src/jvm/org/apache/storm/trident/operation/OperationAwareMapFunction.java index 7fa4a71ab34..e29c7ea244e 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/operation/OperationAwareMapFunction.java +++ b/storm-client/src/jvm/org/apache/storm/trident/operation/OperationAwareMapFunction.java @@ -1,19 +1,26 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.trident.operation; /** - * A one-one transformation function which is aware of Operation (lifecycle of the Trident component). + * A one-one transformation function which is aware of Operation (lifecycle of the Trident + * component). */ public interface OperationAwareMapFunction extends MapFunction, Operation { } diff --git a/storm-client/src/jvm/org/apache/storm/trident/operation/ReducerAggregator.java b/storm-client/src/jvm/org/apache/storm/trident/operation/ReducerAggregator.java index a23a2b8c27c..903b0106c53 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/operation/ReducerAggregator.java +++ b/storm-client/src/jvm/org/apache/storm/trident/operation/ReducerAggregator.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/operation/TridentCollector.java b/storm-client/src/jvm/org/apache/storm/trident/operation/TridentCollector.java index d61880bdd83..c105717dece 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/operation/TridentCollector.java +++ b/storm-client/src/jvm/org/apache/storm/trident/operation/TridentCollector.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -14,11 +20,12 @@ import java.util.List; - /** - * Interface for publishing tuples to a stream and reporting exceptions (to be displayed in Storm UI). + * Interface for publishing tuples to a stream and reporting exceptions (to be displayed in Storm + * UI). * - *

    Trident components that have the ability to emit tuples to a stream are passed an instance of this interface. + *

    Trident components that have the ability to emit tuples to a stream are passed an instance of + * this interface. * *

    For example, to emit a new tuple to a stream, you would do something like the following: * @@ -43,7 +50,8 @@ public interface TridentCollector { /** * Reports an error. The corresponding stack trace will be visible in the Storm UI. * - *

    Note that calling this method does not alter the processing of a batch. To explicitly fail a batch and trigger + *

    Note that calling this method does not alter the processing of a batch. To explicitly fail + * a batch and trigger * a replay, components should throw {@link org.apache.storm.topology.FailedException}. * * @param t The instance of the error (Throwable) being reported. diff --git a/storm-client/src/jvm/org/apache/storm/trident/operation/TridentMultiReducerContext.java b/storm-client/src/jvm/org/apache/storm/trident/operation/TridentMultiReducerContext.java index 21409d2bc56..b7cc0385ce9 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/operation/TridentMultiReducerContext.java +++ b/storm-client/src/jvm/org/apache/storm/trident/operation/TridentMultiReducerContext.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -17,7 +23,6 @@ import org.apache.storm.trident.tuple.TridentTupleView.ProjectionFactory; import org.apache.storm.tuple.Fields; - public class TridentMultiReducerContext { List factories; diff --git a/storm-client/src/jvm/org/apache/storm/trident/operation/TridentOperationContext.java b/storm-client/src/jvm/org/apache/storm/trident/operation/TridentOperationContext.java index e8029031f75..a46f77ea573 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/operation/TridentOperationContext.java +++ b/storm-client/src/jvm/org/apache/storm/trident/operation/TridentOperationContext.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -65,7 +71,8 @@ public ReducedMetric registerMetric(String name, IReducer reducer, int timeBucke } @Override - public CombinedMetric registerMetric(String name, ICombiner combiner, int timeBucketSizeInSecs) { + public CombinedMetric registerMetric(String name, ICombiner combiner, + int timeBucketSizeInSecs) { return topoContext.registerMetric(name, new CombinedMetric(combiner), timeBucketSizeInSecs); } diff --git a/storm-client/src/jvm/org/apache/storm/trident/operation/builtin/ComparisonAggregator.java b/storm-client/src/jvm/org/apache/storm/trident/operation/builtin/ComparisonAggregator.java index 8b52182efa0..93e56f77311 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/operation/builtin/ComparisonAggregator.java +++ b/storm-client/src/jvm/org/apache/storm/trident/operation/builtin/ComparisonAggregator.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

    - * http://www.apache.org/licenses/LICENSE-2.0 - *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

    http://www.apache.org/licenses/LICENSE-2.0 + * + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -35,7 +41,8 @@ public ComparisonAggregator(String inputFieldName) { @Override public State init(Object batchId, TridentCollector collector) { this.batchId = batchId; - log.debug("Started comparison aggregation for batch: [{}] in operation [{}]", batchId, this); + log.debug("Started comparison aggregation for batch: [{}] in operation [{}]", batchId, + this); return new State(); } @@ -44,7 +51,8 @@ public void aggregate(State state, TridentTuple tuple, TridentCollector collecto T value1 = valueFromTuple(state.previousTuple); T value2 = valueFromTuple(tuple); - log.debug("Aggregated tuple value in state [{}], and received tuple value [{}] in operation [{}]", value1, value2, this); + log.debug("Aggregated tuple value in state [{}], and received tuple value [{}] in " + + "operation [{}]", value1, value2, this); if (value2 == null) { return; @@ -65,14 +73,16 @@ protected T valueFromTuple(TridentTuple tuple) { value = tuple; } - log.debug("value from tuple is [{}] with input field [{}] and tuple [{}]", value, inputFieldName, tuple); + log.debug("value from tuple is [{}] with input field [{}] and tuple [{}]", value, + inputFieldName, tuple); return (T) value; } @Override public void complete(State state, TridentCollector collector) { - log.debug("Completed comparison aggregation for batch [{}] with resultant tuple: [{}] in operation [{}]", batchId, + log.debug("Completed comparison aggregation for batch [{}] with resultant tuple: [{}] in " + + "operation [{}]", batchId, state.previousTuple, this); collector.emit(state.previousTuple != null ? state.previousTuple.getValues() : null); diff --git a/storm-client/src/jvm/org/apache/storm/trident/operation/builtin/Count.java b/storm-client/src/jvm/org/apache/storm/trident/operation/builtin/Count.java index 748d3dd576f..a7630682c23 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/operation/builtin/Count.java +++ b/storm-client/src/jvm/org/apache/storm/trident/operation/builtin/Count.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -15,7 +21,6 @@ import org.apache.storm.trident.operation.CombinerAggregator; import org.apache.storm.trident.tuple.TridentTuple; - public class Count implements CombinerAggregator { @Override diff --git a/storm-client/src/jvm/org/apache/storm/trident/operation/builtin/Debug.java b/storm-client/src/jvm/org/apache/storm/trident/operation/builtin/Debug.java index 4e78ae85520..ad75adf76fe 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/operation/builtin/Debug.java +++ b/storm-client/src/jvm/org/apache/storm/trident/operation/builtin/Debug.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -19,7 +25,8 @@ import org.slf4j.LoggerFactory; /** - * Filter for debugging purposes. The `isKeep()` method simply prints the tuple to `System.out` and returns `true`. + * Filter for debugging purposes. The `isKeep()` method simply prints the tuple to `System.out` and + * returns `true`. */ public class Debug extends BaseFilter { private static final Logger LOG = LoggerFactory.getLogger(Debug.class); diff --git a/storm-client/src/jvm/org/apache/storm/trident/operation/builtin/Equals.java b/storm-client/src/jvm/org/apache/storm/trident/operation/builtin/Equals.java index bd42cd50b0c..fa16e2096a0 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/operation/builtin/Equals.java +++ b/storm-client/src/jvm/org/apache/storm/trident/operation/builtin/Equals.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -15,7 +21,6 @@ import org.apache.storm.trident.operation.BaseFilter; import org.apache.storm.trident.tuple.TridentTuple; - public class Equals extends BaseFilter { @Override diff --git a/storm-client/src/jvm/org/apache/storm/trident/operation/builtin/FilterNull.java b/storm-client/src/jvm/org/apache/storm/trident/operation/builtin/FilterNull.java index 96ebf233aea..f19e15be94b 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/operation/builtin/FilterNull.java +++ b/storm-client/src/jvm/org/apache/storm/trident/operation/builtin/FilterNull.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -16,7 +22,8 @@ import org.apache.storm.trident.tuple.TridentTuple; /** - * Simple `Filter` implementation that filters out any tuples that have fields with a value of `null`. + * Simple `Filter` implementation that filters out any tuples that have fields with a value of + * `null`. */ public class FilterNull extends BaseFilter { @Override diff --git a/storm-client/src/jvm/org/apache/storm/trident/operation/builtin/FirstN.java b/storm-client/src/jvm/org/apache/storm/trident/operation/builtin/FirstN.java index a2b09922811..1e9dd7d8e19 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/operation/builtin/FirstN.java +++ b/storm-client/src/jvm/org/apache/storm/trident/operation/builtin/FirstN.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -22,7 +28,6 @@ import org.apache.storm.trident.tuple.TridentTuple; import org.apache.storm.tuple.Fields; - /** * An {@link org.apache.storm.trident.operation.Assembly} implementation. */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/operation/builtin/MapGet.java b/storm-client/src/jvm/org/apache/storm/trident/operation/builtin/MapGet.java index e39afed46de..3075fb1a424 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/operation/builtin/MapGet.java +++ b/storm-client/src/jvm/org/apache/storm/trident/operation/builtin/MapGet.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -19,7 +25,6 @@ import org.apache.storm.trident.tuple.TridentTuple; import org.apache.storm.tuple.Values; - public class MapGet extends BaseQueryFunction { @Override public List batchRetrieve(ReadOnlyMapState map, List keys) { diff --git a/storm-client/src/jvm/org/apache/storm/trident/operation/builtin/Max.java b/storm-client/src/jvm/org/apache/storm/trident/operation/builtin/Max.java index 5e6045afd57..9b8eab57635 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/operation/builtin/Max.java +++ b/storm-client/src/jvm/org/apache/storm/trident/operation/builtin/Max.java @@ -1,19 +1,26 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

    - * http://www.apache.org/licenses/LICENSE-2.0 - *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

    http://www.apache.org/licenses/LICENSE-2.0 + * + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.trident.operation.builtin; /** - * This aggregator computes the maximum of aggregated tuples in a stream. It assumes that the tuple has one value and it is an instance of + * This aggregator computes the maximum of aggregated tuples in a stream. It assumes that the tuple + * has one value and it is an instance of * {@code Comparable}. */ public class Max extends ComparisonAggregator> { diff --git a/storm-client/src/jvm/org/apache/storm/trident/operation/builtin/MaxWithComparator.java b/storm-client/src/jvm/org/apache/storm/trident/operation/builtin/MaxWithComparator.java index b60d30b7f8e..8c61e27851d 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/operation/builtin/MaxWithComparator.java +++ b/storm-client/src/jvm/org/apache/storm/trident/operation/builtin/MaxWithComparator.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

    - * http://www.apache.org/licenses/LICENSE-2.0 - *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

    http://www.apache.org/licenses/LICENSE-2.0 + * + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/operation/builtin/Min.java b/storm-client/src/jvm/org/apache/storm/trident/operation/builtin/Min.java index 860b8908739..af3d101cbd5 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/operation/builtin/Min.java +++ b/storm-client/src/jvm/org/apache/storm/trident/operation/builtin/Min.java @@ -1,19 +1,26 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

    - * http://www.apache.org/licenses/LICENSE-2.0 - *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

    http://www.apache.org/licenses/LICENSE-2.0 + * + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.trident.operation.builtin; /** - * This aggregator computes the minimum of aggregated tuples in a stream. It assumes that the tuple has one value and it is an instance of + * This aggregator computes the minimum of aggregated tuples in a stream. It assumes that the tuple + * has one value and it is an instance of * {@code Comparable}. */ public class Min extends ComparisonAggregator> { diff --git a/storm-client/src/jvm/org/apache/storm/trident/operation/builtin/MinWithComparator.java b/storm-client/src/jvm/org/apache/storm/trident/operation/builtin/MinWithComparator.java index 1f9eb871ca3..d263eda5b60 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/operation/builtin/MinWithComparator.java +++ b/storm-client/src/jvm/org/apache/storm/trident/operation/builtin/MinWithComparator.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

    - * http://www.apache.org/licenses/LICENSE-2.0 - *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

    http://www.apache.org/licenses/LICENSE-2.0 + * + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -15,7 +21,8 @@ import java.util.Comparator; /** - * This aggregator computes the minimum of aggregated tuples in a stream. It uses given @{code comparator} for comparing two values in a + * This aggregator computes the minimum of aggregated tuples in a stream. It uses given @{code + * comparator} for comparing two values in a * stream. */ public class MinWithComparator extends ComparisonAggregator { diff --git a/storm-client/src/jvm/org/apache/storm/trident/operation/builtin/Negate.java b/storm-client/src/jvm/org/apache/storm/trident/operation/builtin/Negate.java index a2f1380863d..476de8c30a7 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/operation/builtin/Negate.java +++ b/storm-client/src/jvm/org/apache/storm/trident/operation/builtin/Negate.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -26,15 +32,19 @@ * *

    The `Negate` filter is useful for dividing a Stream in two based on some boolean condition. * - *

    Suppose we had a Stream named `userStream` containing information about users, and a custom `Filter` implementation, - * `RegisteredUserFilter` that filtered out unregistered users. We could divide the `userStream` Stream into two separate Streams -- one for + *

    Suppose we had a Stream named `userStream` containing information about users, and a custom + * `Filter` implementation, + * `RegisteredUserFilter` that filtered out unregistered users. We could divide the `userStream` + * Stream into two separate Streams -- one for * registered users, and one for unregistered users -- by doing the following: * *

    ```java Stream userStream = ... * - *

    Filter registeredFilter = new ResisteredUserFilter(); Filter unregisteredFilter = new Negate(registeredFilter); + *

    Filter registeredFilter = new ResisteredUserFilter(); Filter unregisteredFilter = new + * Negate(registeredFilter); * - *

    Stream registeredUserStream = userStream.each(userStream.getOutputFields(), registeredFilter); Stream unregisteredUserStream = + *

    Stream registeredUserStream = userStream.each(userStream.getOutputFields(), registeredFilter); + * Stream unregisteredUserStream = * userStream.each(userStream.getOutputFields(), unregisteredFilter); ``` */ public class Negate implements Filter { diff --git a/storm-client/src/jvm/org/apache/storm/trident/operation/builtin/SnapshotGet.java b/storm-client/src/jvm/org/apache/storm/trident/operation/builtin/SnapshotGet.java index 0c25e98613f..bf62e2098a5 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/operation/builtin/SnapshotGet.java +++ b/storm-client/src/jvm/org/apache/storm/trident/operation/builtin/SnapshotGet.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/operation/builtin/Sum.java b/storm-client/src/jvm/org/apache/storm/trident/operation/builtin/Sum.java index 9c52064b414..012f13c2d45 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/operation/builtin/Sum.java +++ b/storm-client/src/jvm/org/apache/storm/trident/operation/builtin/Sum.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -16,7 +22,6 @@ import org.apache.storm.trident.operation.CombinerAggregator; import org.apache.storm.trident.tuple.TridentTuple; - public class Sum implements CombinerAggregator { private static BigDecimal asBigDecimal(Number val) { diff --git a/storm-client/src/jvm/org/apache/storm/trident/operation/builtin/TupleCollectionGet.java b/storm-client/src/jvm/org/apache/storm/trident/operation/builtin/TupleCollectionGet.java index 6ceb21164fd..37dd9d97a35 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/operation/builtin/TupleCollectionGet.java +++ b/storm-client/src/jvm/org/apache/storm/trident/operation/builtin/TupleCollectionGet.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -33,7 +39,8 @@ public List>> batchRetrieve(State state, List> tuplesIterator, TridentCollector collector) { + public void execute(TridentTuple tuple, Iterator> tuplesIterator, + TridentCollector collector) { while (tuplesIterator.hasNext()) { collector.emit(tuplesIterator.next()); } diff --git a/storm-client/src/jvm/org/apache/storm/trident/operation/impl/CaptureCollector.java b/storm-client/src/jvm/org/apache/storm/trident/operation/impl/CaptureCollector.java index 9f9605cb509..9869e6e8f06 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/operation/impl/CaptureCollector.java +++ b/storm-client/src/jvm/org/apache/storm/trident/operation/impl/CaptureCollector.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/operation/impl/ChainedAggregatorImpl.java b/storm-client/src/jvm/org/apache/storm/trident/operation/impl/ChainedAggregatorImpl.java index ce45291737c..cd7bbd94b71 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/operation/impl/ChainedAggregatorImpl.java +++ b/storm-client/src/jvm/org/apache/storm/trident/operation/impl/ChainedAggregatorImpl.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -19,8 +25,8 @@ import org.apache.storm.trident.operation.TridentOperationContext; import org.apache.storm.trident.tuple.ComboList; import org.apache.storm.trident.tuple.TridentTuple; -import org.apache.storm.trident.tuple.TridentTupleView; import org.apache.storm.trident.tuple.TridentTupleView.ProjectionFactory; +import org.apache.storm.trident.tuple.TridentTupleView; import org.apache.storm.tuple.Fields; public class ChainedAggregatorImpl implements Aggregator { @@ -29,7 +35,6 @@ public class ChainedAggregatorImpl implements Aggregator { ComboList.Factory fact; Fields[] inputFields; - public ChainedAggregatorImpl(Aggregator[] aggs, Fields[] inputFields, ComboList.Factory fact) { this.aggs = aggs; this.inputFields = inputFields; @@ -78,7 +83,7 @@ public void complete(ChainedResult val, TridentCollector collector) { indices[i] = 0; } boolean keepGoing = true; - //emit cross-join of all emitted tuples + // emit cross-join of all emitted tuples while (keepGoing) { List[] combined = new List[aggs.length]; for (int i = 0; i < aggs.length; i++) { @@ -91,7 +96,7 @@ public void complete(ChainedResult val, TridentCollector collector) { } } - //return false if can't increment anymore + // return false if can't increment anymore private boolean increment(TridentCollector[] lengths, int[] indices, int j) { if (j == -1) { return false; diff --git a/storm-client/src/jvm/org/apache/storm/trident/operation/impl/ChainedResult.java b/storm-client/src/jvm/org/apache/storm/trident/operation/impl/ChainedResult.java index b5d73e4ed97..74eff4c18ca 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/operation/impl/ChainedResult.java +++ b/storm-client/src/jvm/org/apache/storm/trident/operation/impl/ChainedResult.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -15,8 +21,7 @@ import org.apache.storm.shade.org.apache.commons.lang3.builder.ToStringBuilder; import org.apache.storm.trident.operation.TridentCollector; - -//for ChainedAggregator +// for ChainedAggregator public class ChainedResult { Object[] objs; TridentCollector[] collectors; diff --git a/storm-client/src/jvm/org/apache/storm/trident/operation/impl/CombinerAggStateUpdater.java b/storm-client/src/jvm/org/apache/storm/trident/operation/impl/CombinerAggStateUpdater.java index 44e4f039bdd..c34d2ba477b 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/operation/impl/CombinerAggStateUpdater.java +++ b/storm-client/src/jvm/org/apache/storm/trident/operation/impl/CombinerAggStateUpdater.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -30,11 +36,12 @@ public CombinerAggStateUpdater(CombinerAggregator agg) { this.agg = agg; } - @Override - public void updateState(Snapshottable state, List tuples, TridentCollector collector) { + public void updateState(Snapshottable state, List tuples, + TridentCollector collector) { if (tuples.size() != 1) { - throw new IllegalArgumentException("Combiner state updater should receive a single tuple. Received: " + tuples.toString()); + throw new IllegalArgumentException("Combiner state updater should receive a single " + + "tuple. Received: " + tuples.toString()); } Object newVal = state.update(new CombinerValueUpdater(agg, tuples.get(0).getValue(0))); collector.emit(new Values(newVal)); diff --git a/storm-client/src/jvm/org/apache/storm/trident/operation/impl/CombinerAggregatorCombineImpl.java b/storm-client/src/jvm/org/apache/storm/trident/operation/impl/CombinerAggregatorCombineImpl.java index 4427263c682..ee9de16154a 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/operation/impl/CombinerAggregatorCombineImpl.java +++ b/storm-client/src/jvm/org/apache/storm/trident/operation/impl/CombinerAggregatorCombineImpl.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/operation/impl/CombinerAggregatorInitImpl.java b/storm-client/src/jvm/org/apache/storm/trident/operation/impl/CombinerAggregatorInitImpl.java index 085797adcf3..05b60b5d81b 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/operation/impl/CombinerAggregatorInitImpl.java +++ b/storm-client/src/jvm/org/apache/storm/trident/operation/impl/CombinerAggregatorInitImpl.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/operation/impl/ConsumerExecutor.java b/storm-client/src/jvm/org/apache/storm/trident/operation/impl/ConsumerExecutor.java index 5d813040600..db2c1a51ce2 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/operation/impl/ConsumerExecutor.java +++ b/storm-client/src/jvm/org/apache/storm/trident/operation/impl/ConsumerExecutor.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/operation/impl/FilterExecutor.java b/storm-client/src/jvm/org/apache/storm/trident/operation/impl/FilterExecutor.java index 09774bb5d5c..98c3d3b25a0 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/operation/impl/FilterExecutor.java +++ b/storm-client/src/jvm/org/apache/storm/trident/operation/impl/FilterExecutor.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/operation/impl/FlatMapFunctionExecutor.java b/storm-client/src/jvm/org/apache/storm/trident/operation/impl/FlatMapFunctionExecutor.java index df3fcf1c25e..94bd11c7af4 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/operation/impl/FlatMapFunctionExecutor.java +++ b/storm-client/src/jvm/org/apache/storm/trident/operation/impl/FlatMapFunctionExecutor.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/operation/impl/GlobalBatchToPartition.java b/storm-client/src/jvm/org/apache/storm/trident/operation/impl/GlobalBatchToPartition.java index 6657835c231..4a8292cec5d 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/operation/impl/GlobalBatchToPartition.java +++ b/storm-client/src/jvm/org/apache/storm/trident/operation/impl/GlobalBatchToPartition.java @@ -1,18 +1,23 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.trident.operation.impl; - public class GlobalBatchToPartition implements SingleEmitAggregator.BatchToPartition { @Override diff --git a/storm-client/src/jvm/org/apache/storm/trident/operation/impl/GroupCollector.java b/storm-client/src/jvm/org/apache/storm/trident/operation/impl/GroupCollector.java index f70e488959e..2527d6bb694 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/operation/impl/GroupCollector.java +++ b/storm-client/src/jvm/org/apache/storm/trident/operation/impl/GroupCollector.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/operation/impl/GroupedAggregator.java b/storm-client/src/jvm/org/apache/storm/trident/operation/impl/GroupedAggregator.java index 8598ffb6e42..f0af511ed56 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/operation/impl/GroupedAggregator.java +++ b/storm-client/src/jvm/org/apache/storm/trident/operation/impl/GroupedAggregator.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -14,15 +20,15 @@ import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Map.Entry; +import java.util.Map; import org.apache.storm.trident.operation.Aggregator; import org.apache.storm.trident.operation.TridentCollector; import org.apache.storm.trident.operation.TridentOperationContext; import org.apache.storm.trident.tuple.ComboList; import org.apache.storm.trident.tuple.TridentTuple; -import org.apache.storm.trident.tuple.TridentTupleView; import org.apache.storm.trident.tuple.TridentTupleView.ProjectionFactory; +import org.apache.storm.trident.tuple.TridentTupleView; import org.apache.storm.tuple.Fields; public class GroupedAggregator implements Aggregator { diff --git a/storm-client/src/jvm/org/apache/storm/trident/operation/impl/GroupedMultiReducerExecutor.java b/storm-client/src/jvm/org/apache/storm/trident/operation/impl/GroupedMultiReducerExecutor.java index f37c4629a78..e5a442af937 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/operation/impl/GroupedMultiReducerExecutor.java +++ b/storm-client/src/jvm/org/apache/storm/trident/operation/impl/GroupedMultiReducerExecutor.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -24,7 +30,6 @@ import org.apache.storm.trident.tuple.TridentTupleView.ProjectionFactory; import org.apache.storm.tuple.Fields; - public class GroupedMultiReducerExecutor implements MultiReducer> { GroupedMultiReducer reducer; List groupFields; @@ -32,9 +37,11 @@ public class GroupedMultiReducerExecutor implements MultiReducer groupFactories = new ArrayList(); List inputFactories = new ArrayList(); - public GroupedMultiReducerExecutor(GroupedMultiReducer reducer, List groupFields, List inputFields) { + public GroupedMultiReducerExecutor(GroupedMultiReducer reducer, List groupFields, + List inputFields) { if (inputFields.size() != groupFields.size()) { - throw new IllegalArgumentException("Multireducer groupFields and inputFields must be the same size"); + throw new IllegalArgumentException("Multireducer groupFields and inputFields must be " + + "the same size"); } this.groupFields = groupFields; this.inputFields = inputFields; @@ -56,7 +63,8 @@ public Map init(TridentCollector collector) { } @Override - public void execute(Map state, int streamIndex, TridentTuple full, TridentCollector collector) { + public void execute(Map state, int streamIndex, TridentTuple full, + TridentCollector collector) { ProjectionFactory groupFactory = groupFactories.get(streamIndex); ProjectionFactory inputFactory = inputFactories.get(streamIndex); diff --git a/storm-client/src/jvm/org/apache/storm/trident/operation/impl/IdentityMultiReducer.java b/storm-client/src/jvm/org/apache/storm/trident/operation/impl/IdentityMultiReducer.java index de07b7a5eee..93b8f4ae2e5 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/operation/impl/IdentityMultiReducer.java +++ b/storm-client/src/jvm/org/apache/storm/trident/operation/impl/IdentityMultiReducer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -18,7 +24,6 @@ import org.apache.storm.trident.operation.TridentMultiReducerContext; import org.apache.storm.trident.tuple.TridentTuple; - public class IdentityMultiReducer implements MultiReducer { @Override diff --git a/storm-client/src/jvm/org/apache/storm/trident/operation/impl/IndexHashBatchToPartition.java b/storm-client/src/jvm/org/apache/storm/trident/operation/impl/IndexHashBatchToPartition.java index a067d81e2d8..a1c48787a9d 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/operation/impl/IndexHashBatchToPartition.java +++ b/storm-client/src/jvm/org/apache/storm/trident/operation/impl/IndexHashBatchToPartition.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/operation/impl/JoinState.java b/storm-client/src/jvm/org/apache/storm/trident/operation/impl/JoinState.java index a82f0d13bcd..57ecfd7313a 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/operation/impl/JoinState.java +++ b/storm-client/src/jvm/org/apache/storm/trident/operation/impl/JoinState.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/operation/impl/JoinerMultiReducer.java b/storm-client/src/jvm/org/apache/storm/trident/operation/impl/JoinerMultiReducer.java index 6eb33d8049b..fbae6763898 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/operation/impl/JoinerMultiReducer.java +++ b/storm-client/src/jvm/org/apache/storm/trident/operation/impl/JoinerMultiReducer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -30,7 +36,6 @@ public class JoinerMultiReducer implements GroupedMultiReducer { int numGroupFields; ComboList.Factory factory; - public JoinerMultiReducer(List types, int numGroupFields, List sides) { this.types = types; sideFields = sides; @@ -53,9 +58,11 @@ public JoinState init(TridentCollector collector, TridentTuple group) { } @Override - public void execute(JoinState state, int streamIndex, TridentTuple group, TridentTuple input, TridentCollector collector) { - //TODO: do the inner join incrementally, emitting the cross join with this tuple, against all other sides - //TODO: only do cross join if at least one tuple in each side + public void execute(JoinState state, int streamIndex, TridentTuple group, TridentTuple input, + TridentCollector collector) { + // TODO: do the inner join incrementally, emitting the cross join with this tuple, against + // all other sides + // TODO: only do cross join if at least one tuple in each side List side = state.sides[streamIndex]; if (side.isEmpty()) { state.numSidesReceived++; @@ -94,7 +101,8 @@ private List makeNullList(int size) { return ret; } - private void emitCrossJoin(JoinState state, TridentCollector collector, int overrideIndex, TridentTuple overrideTuple) { + private void emitCrossJoin(JoinState state, TridentCollector collector, int overrideIndex, + TridentTuple overrideTuple) { List[] sides = state.sides; int[] indices = state.indices; for (int i = 0; i < indices.length; i++) { @@ -102,7 +110,7 @@ private void emitCrossJoin(JoinState state, TridentCollector collector, int over } boolean keepGoing = true; - //emit cross-join of all emitted tuples + // emit cross-join of all emitted tuples while (keepGoing) { List[] combined = new List[sides.length + 1]; combined[0] = state.group; @@ -118,9 +126,8 @@ private void emitCrossJoin(JoinState state, TridentCollector collector, int over } } - - //return false if can't increment anymore - //TODO: DRY this code up with what's in ChainedAggregatorImpl + // return false if can't increment anymore + // TODO: DRY this code up with what's in ChainedAggregatorImpl private boolean increment(List[] lengths, int[] indices, int j, int overrideIndex) { if (j == -1) { return false; diff --git a/storm-client/src/jvm/org/apache/storm/trident/operation/impl/MapFunctionExecutor.java b/storm-client/src/jvm/org/apache/storm/trident/operation/impl/MapFunctionExecutor.java index 5df04e48f4c..d800d6ea7ef 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/operation/impl/MapFunctionExecutor.java +++ b/storm-client/src/jvm/org/apache/storm/trident/operation/impl/MapFunctionExecutor.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/operation/impl/PreservingFieldsOrderJoinerMultiReducer.java b/storm-client/src/jvm/org/apache/storm/trident/operation/impl/PreservingFieldsOrderJoinerMultiReducer.java index 08a45fc6d53..76f0b2424bd 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/operation/impl/PreservingFieldsOrderJoinerMultiReducer.java +++ b/storm-client/src/jvm/org/apache/storm/trident/operation/impl/PreservingFieldsOrderJoinerMultiReducer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -31,8 +37,8 @@ public class PreservingFieldsOrderJoinerMultiReducer implements GroupedMultiRedu int numGroupFields; ComboList.Factory factory; - - public PreservingFieldsOrderJoinerMultiReducer(List types, int numGroupFields, List origins, + public PreservingFieldsOrderJoinerMultiReducer(List types, int numGroupFields, + List origins, List joins, List sides) { this.types = types; originFields = origins; @@ -58,9 +64,11 @@ public JoinState init(TridentCollector collector, TridentTuple group) { } @Override - public void execute(JoinState state, int streamIndex, TridentTuple group, TridentTuple input, TridentCollector collector) { - //TODO: do the inner join incrementally, emitting the cross join with this tuple, against all other sides - //TODO: only do cross join if at least one tuple in each side + public void execute(JoinState state, int streamIndex, TridentTuple group, TridentTuple input, + TridentCollector collector) { + // TODO: do the inner join incrementally, emitting the cross join with this tuple, against + // all other sides + // TODO: only do cross join if at least one tuple in each side List side = state.sides[streamIndex]; if (side.isEmpty()) { state.numSidesReceived++; @@ -99,7 +107,8 @@ private List makeNullList(int size) { return ret; } - private void emitCrossJoin(JoinState state, TridentCollector collector, int overrideIndex, TridentTuple overrideTuple) { + private void emitCrossJoin(JoinState state, TridentCollector collector, int overrideIndex, + TridentTuple overrideTuple) { List[] sides = state.sides; int[] indices = state.indices; for (int i = 0; i < indices.length; i++) { @@ -107,12 +116,13 @@ private void emitCrossJoin(JoinState state, TridentCollector collector, int over } boolean keepGoing = true; - //emit cross-join of all emitted tuples + // emit cross-join of all emitted tuples while (keepGoing) { List[] combined = new List[sides.length]; for (int i = 0; i < sides.length; i++) { - List values = buildValuesForStream(state, overrideIndex, overrideTuple, sides, indices, combined, i); + List values = buildValuesForStream(state, overrideIndex, overrideTuple, + sides, indices, combined, i); combined[i] = values; } collector.emit(factory.create(combined)); @@ -120,7 +130,8 @@ private void emitCrossJoin(JoinState state, TridentCollector collector, int over } } - private List buildValuesForStream(JoinState state, int overrideIndex, TridentTuple overrideTuple, List[] sides, + private List buildValuesForStream(JoinState state, int overrideIndex, + TridentTuple overrideTuple, List[] sides, int[] indices, List[] combined, int streamIdx) { List sideValues; if (streamIdx == overrideIndex) { @@ -150,9 +161,8 @@ private List buildValuesForStream(JoinState state, int overrideIndex, Tr } } - - //return false if can't increment anymore - //TODO: DRY this code up with what's in ChainedAggregatorImpl + // return false if can't increment anymore + // TODO: DRY this code up with what's in ChainedAggregatorImpl private boolean increment(List[] lengths, int[] indices, int j, int overrideIndex) { if (j == -1) { return false; diff --git a/storm-client/src/jvm/org/apache/storm/trident/operation/impl/ReducerAggStateUpdater.java b/storm-client/src/jvm/org/apache/storm/trident/operation/impl/ReducerAggStateUpdater.java index 84caa757c97..f81be4b7428 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/operation/impl/ReducerAggStateUpdater.java +++ b/storm-client/src/jvm/org/apache/storm/trident/operation/impl/ReducerAggStateUpdater.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -30,9 +36,9 @@ public ReducerAggStateUpdater(ReducerAggregator agg) { this.agg = agg; } - @Override - public void updateState(Snapshottable state, List tuples, TridentCollector collector) { + public void updateState(Snapshottable state, List tuples, + TridentCollector collector) { Object newVal = state.update(new ReducerValueUpdater(agg, tuples)); collector.emit(new Values(newVal)); } diff --git a/storm-client/src/jvm/org/apache/storm/trident/operation/impl/ReducerAggregatorImpl.java b/storm-client/src/jvm/org/apache/storm/trident/operation/impl/ReducerAggregatorImpl.java index bf208cb35b3..4bb6c59bb30 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/operation/impl/ReducerAggregatorImpl.java +++ b/storm-client/src/jvm/org/apache/storm/trident/operation/impl/ReducerAggregatorImpl.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/operation/impl/Result.java b/storm-client/src/jvm/org/apache/storm/trident/operation/impl/Result.java index 84bbc0da6db..453c1edf21e 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/operation/impl/Result.java +++ b/storm-client/src/jvm/org/apache/storm/trident/operation/impl/Result.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/operation/impl/SingleEmitAggregator.java b/storm-client/src/jvm/org/apache/storm/trident/operation/impl/SingleEmitAggregator.java index bc579b3e758..4f490b97f33 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/operation/impl/SingleEmitAggregator.java +++ b/storm-client/src/jvm/org/apache/storm/trident/operation/impl/SingleEmitAggregator.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -20,7 +26,6 @@ import org.apache.storm.trident.operation.impl.SingleEmitAggregator.SingleEmitState; import org.apache.storm.trident.tuple.TridentTuple; - public class SingleEmitAggregator implements Aggregator { Aggregator agg; BatchToPartition batchToPartition; @@ -32,7 +37,6 @@ public SingleEmitAggregator(Aggregator agg, BatchToPartition batchToPartition) { this.batchToPartition = batchToPartition; } - @Override public SingleEmitState init(Object batchId, TridentCollector collector) { return new SingleEmitState(batchId); @@ -50,7 +54,8 @@ public void aggregate(SingleEmitState val, TridentTuple tuple, TridentCollector @Override public void complete(SingleEmitState val, TridentCollector collector) { if (!val.received) { - if (this.myPartitionIndex == batchToPartition.partitionIndex(val.batchId, this.totalPartitions)) { + if (this.myPartitionIndex == batchToPartition.partitionIndex(val.batchId, + this.totalPartitions)) { val.state = agg.init(val.batchId, collector); agg.complete(val.state, collector); } diff --git a/storm-client/src/jvm/org/apache/storm/trident/operation/impl/TrueFilter.java b/storm-client/src/jvm/org/apache/storm/trident/operation/impl/TrueFilter.java index ddb7ea51580..0faaaf087b5 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/operation/impl/TrueFilter.java +++ b/storm-client/src/jvm/org/apache/storm/trident/operation/impl/TrueFilter.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/partition/GlobalGrouping.java b/storm-client/src/jvm/org/apache/storm/trident/partition/GlobalGrouping.java index 666b70ae4ed..5714bad6cd6 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/partition/GlobalGrouping.java +++ b/storm-client/src/jvm/org/apache/storm/trident/partition/GlobalGrouping.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -24,7 +30,8 @@ public class GlobalGrouping implements CustomStreamGrouping { List target; @Override - public void prepare(WorkerTopologyContext context, GlobalStreamId stream, List targets) { + public void prepare(WorkerTopologyContext context, GlobalStreamId stream, + List targets) { List sorted = new ArrayList<>(targets); Collections.sort(sorted); target = Arrays.asList(sorted.get(0)); diff --git a/storm-client/src/jvm/org/apache/storm/trident/partition/IdentityGrouping.java b/storm-client/src/jvm/org/apache/storm/trident/partition/IdentityGrouping.java index 9617ea8e635..d027b9cf3bb 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/partition/IdentityGrouping.java +++ b/storm-client/src/jvm/org/apache/storm/trident/partition/IdentityGrouping.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -22,16 +28,17 @@ import org.apache.storm.grouping.CustomStreamGrouping; import org.apache.storm.task.WorkerTopologyContext; - public class IdentityGrouping implements CustomStreamGrouping { final Map> precomputed = new HashMap<>(); @Override public void prepare(WorkerTopologyContext context, GlobalStreamId stream, List tasks) { - List sourceTasks = new ArrayList<>(context.getComponentTasks(stream.get_componentId())); + List sourceTasks = new ArrayList<>(context.getComponentTasks(stream + .get_componentId())); Collections.sort(sourceTasks); if (sourceTasks.size() != tasks.size()) { - throw new RuntimeException("Can only do an identity grouping when source and target have same number of tasks"); + throw new RuntimeException("Can only do an identity grouping when source and target " + + "have same number of tasks"); } tasks = new ArrayList<>(tasks); Collections.sort(tasks); @@ -46,7 +53,8 @@ public void prepare(WorkerTopologyContext context, GlobalStreamId stream, List chooseTasks(int task, List values) { List ret = precomputed.get(task); if (ret == null) { - throw new RuntimeException("Tuple emitted by task that's not part of this component. Should be impossible"); + throw new RuntimeException("Tuple emitted by task that's not part of this component. " + + "Should be impossible"); } return ret; } diff --git a/storm-client/src/jvm/org/apache/storm/trident/partition/IndexHashGrouping.java b/storm-client/src/jvm/org/apache/storm/trident/partition/IndexHashGrouping.java index 7a2cfc1ea15..a11920bc6f4 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/partition/IndexHashGrouping.java +++ b/storm-client/src/jvm/org/apache/storm/trident/partition/IndexHashGrouping.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -35,7 +41,8 @@ public static int objectToIndex(Object val, int numPartitions) { } @Override - public void prepare(WorkerTopologyContext context, GlobalStreamId stream, List targetTasks) { + public void prepare(WorkerTopologyContext context, GlobalStreamId stream, + List targetTasks) { targets = targetTasks; } diff --git a/storm-client/src/jvm/org/apache/storm/trident/planner/BridgeReceiver.java b/storm-client/src/jvm/org/apache/storm/trident/planner/BridgeReceiver.java index 601a64b9a34..3d773803983 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/planner/BridgeReceiver.java +++ b/storm-client/src/jvm/org/apache/storm/trident/planner/BridgeReceiver.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -16,7 +22,6 @@ import org.apache.storm.trident.tuple.ConsList; import org.apache.storm.trident.tuple.TridentTuple; - public class BridgeReceiver implements TupleReceiver { BatchOutputCollector collector; diff --git a/storm-client/src/jvm/org/apache/storm/trident/planner/Node.java b/storm-client/src/jvm/org/apache/storm/trident/planner/Node.java index 076514a835f..6d8b7e9d565 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/planner/Node.java +++ b/storm-client/src/jvm/org/apache/storm/trident/planner/Node.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/planner/NodeStateInfo.java b/storm-client/src/jvm/org/apache/storm/trident/planner/NodeStateInfo.java index fd5efc0d6ba..3581993f050 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/planner/NodeStateInfo.java +++ b/storm-client/src/jvm/org/apache/storm/trident/planner/NodeStateInfo.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/planner/PartitionNode.java b/storm-client/src/jvm/org/apache/storm/trident/planner/PartitionNode.java index e5323eb4858..c168978ce03 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/planner/PartitionNode.java +++ b/storm-client/src/jvm/org/apache/storm/trident/planner/PartitionNode.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -19,11 +25,10 @@ import org.apache.storm.trident.util.TridentUtils; import org.apache.storm.tuple.Fields; - public class PartitionNode extends Node { public transient Grouping thriftGrouping; - //has the streamid/outputFields of the node it's doing the partitioning on + // has the streamid/outputFields of the node it's doing the partitioning on public PartitionNode(String streamId, String name, Fields allOutputFields, Grouping grouping) { super(streamId, name, allOutputFields); this.thriftGrouping = grouping; diff --git a/storm-client/src/jvm/org/apache/storm/trident/planner/ProcessorContext.java b/storm-client/src/jvm/org/apache/storm/trident/planner/ProcessorContext.java index 1a41d363c28..29fed2398e7 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/planner/ProcessorContext.java +++ b/storm-client/src/jvm/org/apache/storm/trident/planner/ProcessorContext.java @@ -1,18 +1,23 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.trident.planner; - public class ProcessorContext { public Object batchId; public Object[] state; diff --git a/storm-client/src/jvm/org/apache/storm/trident/planner/ProcessorNode.java b/storm-client/src/jvm/org/apache/storm/trident/planner/ProcessorNode.java index 11d7c47240a..af4b414d93c 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/planner/ProcessorNode.java +++ b/storm-client/src/jvm/org/apache/storm/trident/planner/ProcessorNode.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -20,7 +26,8 @@ public class ProcessorNode extends Node { public TridentProcessor processor; public Fields selfOutFields; - public ProcessorNode(String streamId, String name, Fields allOutputFields, Fields selfOutFields, TridentProcessor processor) { + public ProcessorNode(String streamId, String name, Fields allOutputFields, Fields selfOutFields, + TridentProcessor processor) { super(streamId, name, allOutputFields); this.processor = processor; this.selfOutFields = selfOutFields; @@ -28,6 +35,7 @@ public ProcessorNode(String streamId, String name, Fields allOutputFields, Field @Override public String shortString() { - return super.shortString() + ", processor: " + processor + ", selfOutFields: " + selfOutFields; + return super.shortString() + ", processor: " + processor + ", selfOutFields: " + + selfOutFields; } } diff --git a/storm-client/src/jvm/org/apache/storm/trident/planner/SpoutNode.java b/storm-client/src/jvm/org/apache/storm/trident/planner/SpoutNode.java index 59f543cc3d3..5f17f6a47bf 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/planner/SpoutNode.java +++ b/storm-client/src/jvm/org/apache/storm/trident/planner/SpoutNode.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -14,13 +20,13 @@ import org.apache.storm.tuple.Fields; - public class SpoutNode extends Node { public Object spout; - public String txId; //where state is stored in zookeeper (only for batch spout types) + public String txId; // where state is stored in zookeeper (only for batch spout types) public SpoutType type; - public SpoutNode(String streamId, Fields allOutputFields, String txid, Object spout, SpoutType type) { + public SpoutNode(String streamId, Fields allOutputFields, String txid, Object spout, + SpoutType type) { super(streamId, null, allOutputFields); this.txId = txid; this.spout = spout; diff --git a/storm-client/src/jvm/org/apache/storm/trident/planner/SubtopologyBolt.java b/storm-client/src/jvm/org/apache/storm/trident/planner/SubtopologyBolt.java index 59b3f20b0bc..69580a3b3c4 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/planner/SubtopologyBolt.java +++ b/storm-client/src/jvm/org/apache/storm/trident/planner/SubtopologyBolt.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -30,8 +36,8 @@ import org.apache.storm.trident.state.State; import org.apache.storm.trident.topology.BatchInfo; import org.apache.storm.trident.topology.ITridentBatchBolt; -import org.apache.storm.trident.tuple.TridentTuple; import org.apache.storm.trident.tuple.TridentTuple.Factory; +import org.apache.storm.trident.tuple.TridentTuple; import org.apache.storm.trident.tuple.TridentTupleView.ProjectionFactory; import org.apache.storm.trident.tuple.TridentTupleView.RootFactory; import org.apache.storm.trident.util.IndexedEdge; @@ -52,15 +58,17 @@ public class SubtopologyBolt implements ITridentBatchBolt { final Map> myTopologicallyOrdered = new HashMap<>(); final Map batchGroups; - //given processornodes and static state nodes + // given processornodes and static state nodes @SuppressWarnings({ "unchecked", "rawtypes" }) - public SubtopologyBolt(DefaultDirectedGraph graph, Set nodes, Map batchGroups) { + public SubtopologyBolt(DefaultDirectedGraph graph, Set nodes, Map batchGroups) { this.nodes = nodes; this.graph = (Graph) graph.clone(); this.batchGroups = copyAndOnlyKeep(batchGroups, nodes); - //Remove the unneeded entries from the graph - //We want to keep all of our nodes, and the nodes that they are connected directly to (parents and children). + // Remove the unneeded entries from the graph + // We want to keep all of our nodes, and the nodes that they are connected directly to + // (parents and children). Set nodesToKeep = new HashSet<>(); for (IndexedEdge edge : this.graph.edgeSet()) { Node s = this.graph.getEdgeSource(edge); @@ -76,7 +84,8 @@ public SubtopologyBolt(DefaultDirectedGraph graph, Set this.graph.removeAllVertices(nodesToRemove); } - private static Map copyAndOnlyKeep(Map batchGroups, Set nodes) { + private static Map copyAndOnlyKeep(Map batchGroups, + Set nodes) { Map ret = new HashMap<>(nodes.size()); for (Map.Entry entry : batchGroups.entrySet()) { if (nodes.contains(entry.getKey())) { @@ -87,11 +96,13 @@ private static Map copyAndOnlyKeep(Map batchGroups, } @Override - public void prepare(Map conf, TopologyContext context, BatchOutputCollector batchCollector) { + public void prepare(Map conf, TopologyContext context, + BatchOutputCollector batchCollector) { int thisComponentNumTasks = context.getComponentTasks(context.getThisComponentId()).size(); for (Node n : nodes) { if (n.stateInfo != null) { - State s = n.stateInfo.spec.stateFactory.makeState(conf, context, context.getThisTaskIndex(), thisComponentNumTasks); + State s = n.stateInfo.spec.stateFactory.makeState(conf, context, context + .getThisTaskIndex(), thisComponentNumTasks); context.setTaskData(n.stateInfo.id, s); } } @@ -115,7 +126,8 @@ public void prepare(Map conf, TopologyContext context, BatchOutp parentFactories.add(outputFactories.get(p)); } else { if (!roots.containsKey(p.streamId)) { - roots.put(p.streamId, new InitialReceiver(p.streamId, getSourceOutputFields(context, p.streamId))); + roots.put(p.streamId, new InitialReceiver(p.streamId, + getSourceOutputFields(context, p.streamId))); } roots.get(p.streamId).addReceiver(pn.processor); parentFactories.add(roots.get(p.streamId).getOutputFactory()); @@ -197,7 +209,8 @@ public void cleanup() { @Override public void declareOutputFields(OutputFieldsDeclarer declarer) { for (Node n : nodes) { - declarer.declareStream(n.streamId, TridentUtils.fieldsConcat(new Fields("$batchId"), n.allOutputFields)); + declarer.declareStream(n.streamId, TridentUtils.fieldsConcat(new Fields("$batchId"), + n.allOutputFields)); } } @@ -206,7 +219,6 @@ public Map getComponentConfiguration() { return null; } - protected static class InitialReceiver { List receivers = new ArrayList<>(); RootFactory factory; diff --git a/storm-client/src/jvm/org/apache/storm/trident/planner/TridentProcessor.java b/storm-client/src/jvm/org/apache/storm/trident/planner/TridentProcessor.java index 1cd03ca29dc..de02f6e2d59 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/planner/TridentProcessor.java +++ b/storm-client/src/jvm/org/apache/storm/trident/planner/TridentProcessor.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/planner/TupleReceiver.java b/storm-client/src/jvm/org/apache/storm/trident/planner/TupleReceiver.java index 1d5befa7c1a..0b113b86cdc 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/planner/TupleReceiver.java +++ b/storm-client/src/jvm/org/apache/storm/trident/planner/TupleReceiver.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -14,9 +20,8 @@ import org.apache.storm.trident.tuple.TridentTuple; - public interface TupleReceiver { - //streaId indicates where tuple came from + // streaId indicates where tuple came from void execute(ProcessorContext processorContext, String streamId, TridentTuple tuple); void flush(); diff --git a/storm-client/src/jvm/org/apache/storm/trident/planner/processor/AggregateProcessor.java b/storm-client/src/jvm/org/apache/storm/trident/planner/processor/AggregateProcessor.java index 61476eeef41..430d333b2f4 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/planner/processor/AggregateProcessor.java +++ b/storm-client/src/jvm/org/apache/storm/trident/planner/processor/AggregateProcessor.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -19,12 +25,11 @@ import org.apache.storm.trident.operation.TridentOperationContext; import org.apache.storm.trident.planner.ProcessorContext; import org.apache.storm.trident.planner.TridentProcessor; -import org.apache.storm.trident.tuple.TridentTuple; import org.apache.storm.trident.tuple.TridentTuple.Factory; +import org.apache.storm.trident.tuple.TridentTuple; import org.apache.storm.trident.tuple.TridentTupleView.ProjectionFactory; import org.apache.storm.tuple.Fields; - public class AggregateProcessor implements TridentProcessor { Aggregator agg; TridentContext context; @@ -38,7 +43,8 @@ public AggregateProcessor(Fields inputFields, Aggregator agg) { } @Override - public void prepare(Map conf, TopologyContext context, TridentContext tridentContext) { + public void prepare(Map conf, TopologyContext context, + TridentContext tridentContext) { List parents = tridentContext.getParentTupleFactories(); if (parents.size() != 1) { throw new RuntimeException("Aggregate operation can only have one parent"); @@ -57,13 +63,15 @@ public void cleanup() { @Override public void startBatch(ProcessorContext processorContext) { collector.setContext(processorContext); - processorContext.state[context.getStateIndex()] = agg.init(processorContext.batchId, collector); + processorContext.state[context.getStateIndex()] = agg.init(processorContext.batchId, + collector); } @Override public void execute(ProcessorContext processorContext, String streamId, TridentTuple tuple) { collector.setContext(processorContext); - agg.aggregate(processorContext.state[context.getStateIndex()], projection.create(tuple), collector); + agg.aggregate(processorContext.state[context.getStateIndex()], projection.create(tuple), + collector); } @Override diff --git a/storm-client/src/jvm/org/apache/storm/trident/planner/processor/AppendCollector.java b/storm-client/src/jvm/org/apache/storm/trident/planner/processor/AppendCollector.java index cde0a5e75fc..84ecb204b04 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/planner/processor/AppendCollector.java +++ b/storm-client/src/jvm/org/apache/storm/trident/planner/processor/AppendCollector.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -16,11 +22,10 @@ import org.apache.storm.trident.operation.TridentCollector; import org.apache.storm.trident.planner.ProcessorContext; import org.apache.storm.trident.planner.TupleReceiver; -import org.apache.storm.trident.tuple.TridentTuple; import org.apache.storm.trident.tuple.TridentTuple.Factory; -import org.apache.storm.trident.tuple.TridentTupleView; +import org.apache.storm.trident.tuple.TridentTuple; import org.apache.storm.trident.tuple.TridentTupleView.OperationOutputFactory; - +import org.apache.storm.trident.tuple.TridentTupleView; public class AppendCollector implements TridentCollector { OperationOutputFactory factory; @@ -30,7 +35,8 @@ public class AppendCollector implements TridentCollector { public AppendCollector(TridentContext context) { triContext = context; - factory = new OperationOutputFactory(context.getParentTupleFactories().get(0), context.getSelfOutputFields()); + factory = new OperationOutputFactory(context.getParentTupleFactories().get(0), context + .getSelfOutputFields()); } public void setContext(ProcessorContext pc, TridentTuple t) { diff --git a/storm-client/src/jvm/org/apache/storm/trident/planner/processor/EachProcessor.java b/storm-client/src/jvm/org/apache/storm/trident/planner/processor/EachProcessor.java index 386df46eaeb..7087a1b70ac 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/planner/processor/EachProcessor.java +++ b/storm-client/src/jvm/org/apache/storm/trident/planner/processor/EachProcessor.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -19,12 +25,11 @@ import org.apache.storm.trident.operation.TridentOperationContext; import org.apache.storm.trident.planner.ProcessorContext; import org.apache.storm.trident.planner.TridentProcessor; -import org.apache.storm.trident.tuple.TridentTuple; import org.apache.storm.trident.tuple.TridentTuple.Factory; +import org.apache.storm.trident.tuple.TridentTuple; import org.apache.storm.trident.tuple.TridentTupleView.ProjectionFactory; import org.apache.storm.tuple.Fields; - public class EachProcessor implements TridentProcessor { Function function; TridentContext context; @@ -38,7 +43,8 @@ public EachProcessor(Fields inputFields, Function function) { } @Override - public void prepare(Map conf, TopologyContext context, TridentContext tridentContext) { + public void prepare(Map conf, TopologyContext context, + TridentContext tridentContext) { List parents = tridentContext.getParentTupleFactories(); if (parents.size() != 1) { throw new RuntimeException("Each operation can only have one parent"); @@ -65,7 +71,6 @@ public void execute(ProcessorContext processorContext, String streamId, TridentT function.execute(projection.create(tuple), collector); } - @Override public void startBatch(ProcessorContext processorContext) { } diff --git a/storm-client/src/jvm/org/apache/storm/trident/planner/processor/FreshCollector.java b/storm-client/src/jvm/org/apache/storm/trident/planner/processor/FreshCollector.java index 21843cc3980..399f7971ebb 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/planner/processor/FreshCollector.java +++ b/storm-client/src/jvm/org/apache/storm/trident/planner/processor/FreshCollector.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -16,11 +22,10 @@ import org.apache.storm.trident.operation.TridentCollector; import org.apache.storm.trident.planner.ProcessorContext; import org.apache.storm.trident.planner.TupleReceiver; -import org.apache.storm.trident.tuple.TridentTuple; import org.apache.storm.trident.tuple.TridentTuple.Factory; +import org.apache.storm.trident.tuple.TridentTuple; import org.apache.storm.trident.tuple.TridentTupleView.FreshOutputFactory; - public class FreshCollector implements TridentCollector { FreshOutputFactory factory; TridentContext triContext; diff --git a/storm-client/src/jvm/org/apache/storm/trident/planner/processor/MapProcessor.java b/storm-client/src/jvm/org/apache/storm/trident/planner/processor/MapProcessor.java index 289546305b3..8883a580354 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/planner/processor/MapProcessor.java +++ b/storm-client/src/jvm/org/apache/storm/trident/planner/processor/MapProcessor.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -43,7 +49,8 @@ public MapProcessor(Fields inputFields, Function function) { } @Override - public void prepare(Map conf, TopologyContext context, TridentContext tridentContext) { + public void prepare(Map conf, TopologyContext context, + TridentContext tridentContext) { List parents = tridentContext.getParentTupleFactories(); if (parents.size() != 1) { throw new RuntimeException("Map operation can only have one parent"); diff --git a/storm-client/src/jvm/org/apache/storm/trident/planner/processor/MultiReducerProcessor.java b/storm-client/src/jvm/org/apache/storm/trident/planner/processor/MultiReducerProcessor.java index 32b0076d009..e23a9e81525 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/planner/processor/MultiReducerProcessor.java +++ b/storm-client/src/jvm/org/apache/storm/trident/planner/processor/MultiReducerProcessor.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -21,12 +27,11 @@ import org.apache.storm.trident.operation.TridentMultiReducerContext; import org.apache.storm.trident.planner.ProcessorContext; import org.apache.storm.trident.planner.TridentProcessor; -import org.apache.storm.trident.tuple.TridentTuple; import org.apache.storm.trident.tuple.TridentTuple.Factory; +import org.apache.storm.trident.tuple.TridentTuple; import org.apache.storm.trident.tuple.TridentTupleView.ProjectionFactory; import org.apache.storm.tuple.Fields; - public class MultiReducerProcessor implements TridentProcessor { MultiReducer reducer; TridentContext context; @@ -41,7 +46,8 @@ public MultiReducerProcessor(List inputFields, MultiReducer reducer) { } @Override - public void prepare(Map conf, TopologyContext context, TridentContext tridentContext) { + public void prepare(Map conf, TopologyContext context, + TridentContext tridentContext) { List parents = tridentContext.getParentTupleFactories(); this.context = tridentContext; streamToIndex = new HashMap<>(); @@ -54,7 +60,8 @@ public void prepare(Map conf, TopologyContext context, TridentCo projectionFactories[i] = new ProjectionFactory(parents.get(i), projectFields.get(i)); } collector = new FreshCollector(tridentContext); - reducer.prepare(conf, new TridentMultiReducerContext((List) Arrays.asList(projectionFactories))); + reducer.prepare(conf, new TridentMultiReducerContext((List) Arrays + .asList(projectionFactories))); } @Override @@ -72,7 +79,8 @@ public void startBatch(ProcessorContext processorContext) { public void execute(ProcessorContext processorContext, String streamId, TridentTuple tuple) { collector.setContext(processorContext); int i = streamToIndex.get(streamId); - reducer.execute(processorContext.state[context.getStateIndex()], i, projectionFactories[i].create(tuple), collector); + reducer.execute(processorContext.state[context.getStateIndex()], i, + projectionFactories[i].create(tuple), collector); } @Override diff --git a/storm-client/src/jvm/org/apache/storm/trident/planner/processor/PartitionPersistProcessor.java b/storm-client/src/jvm/org/apache/storm/trident/planner/processor/PartitionPersistProcessor.java index a82f39bd1f5..7ae37882e08 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/planner/processor/PartitionPersistProcessor.java +++ b/storm-client/src/jvm/org/apache/storm/trident/planner/processor/PartitionPersistProcessor.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -22,12 +28,11 @@ import org.apache.storm.trident.state.State; import org.apache.storm.trident.state.StateUpdater; import org.apache.storm.trident.topology.TransactionAttempt; -import org.apache.storm.trident.tuple.TridentTuple; import org.apache.storm.trident.tuple.TridentTuple.Factory; +import org.apache.storm.trident.tuple.TridentTuple; import org.apache.storm.trident.tuple.TridentTupleView.ProjectionFactory; import org.apache.storm.tuple.Fields; - public class PartitionPersistProcessor implements TridentProcessor { StateUpdater updater; State state; @@ -44,7 +49,8 @@ public PartitionPersistProcessor(String stateId, Fields inputFields, StateUpdate } @Override - public void prepare(Map conf, TopologyContext context, TridentContext tridentContext) { + public void prepare(Map conf, TopologyContext context, + TridentContext tridentContext) { List parents = tridentContext.getParentTupleFactories(); if (parents.size() != 1) { throw new RuntimeException("Partition persist operation can only have one parent"); @@ -89,7 +95,8 @@ public void finishBatch(ProcessorContext processorContext) { // this is also a helpful optimization that state implementations don't need to manually do if (buffer.size() > 0) { Long txid = null; - // this is to support things like persisting off of drpc stream, which is inherently unreliable + // this is to support things like persisting off of drpc stream, which is inherently + // unreliable // and won't have a tx attempt if (batchId instanceof TransactionAttempt) { txid = ((TransactionAttempt) batchId).getTransactionId(); diff --git a/storm-client/src/jvm/org/apache/storm/trident/planner/processor/ProjectedProcessor.java b/storm-client/src/jvm/org/apache/storm/trident/planner/processor/ProjectedProcessor.java index 1ed9012ec8d..bb9c70e7814 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/planner/processor/ProjectedProcessor.java +++ b/storm-client/src/jvm/org/apache/storm/trident/planner/processor/ProjectedProcessor.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -17,12 +23,11 @@ import org.apache.storm.trident.planner.ProcessorContext; import org.apache.storm.trident.planner.TridentProcessor; import org.apache.storm.trident.planner.TupleReceiver; -import org.apache.storm.trident.tuple.TridentTuple; import org.apache.storm.trident.tuple.TridentTuple.Factory; +import org.apache.storm.trident.tuple.TridentTuple; import org.apache.storm.trident.tuple.TridentTupleView.ProjectionFactory; import org.apache.storm.tuple.Fields; - public class ProjectedProcessor implements TridentProcessor { Fields projectFields; ProjectionFactory factory; @@ -33,12 +38,14 @@ public ProjectedProcessor(Fields projectFields) { } @Override - public void prepare(Map conf, TopologyContext context, TridentContext tridentContext) { + public void prepare(Map conf, TopologyContext context, + TridentContext tridentContext) { if (tridentContext.getParentTupleFactories().size() != 1) { throw new RuntimeException("Projection processor can only have one parent"); } this.context = tridentContext; - factory = new ProjectionFactory(tridentContext.getParentTupleFactories().get(0), projectFields); + factory = new ProjectionFactory(tridentContext.getParentTupleFactories().get(0), + projectFields); } @Override diff --git a/storm-client/src/jvm/org/apache/storm/trident/planner/processor/StateQueryProcessor.java b/storm-client/src/jvm/org/apache/storm/trident/planner/processor/StateQueryProcessor.java index d9126c1420e..88f1c5450c8 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/planner/processor/StateQueryProcessor.java +++ b/storm-client/src/jvm/org/apache/storm/trident/planner/processor/StateQueryProcessor.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -22,12 +28,11 @@ import org.apache.storm.trident.planner.TridentProcessor; import org.apache.storm.trident.state.QueryFunction; import org.apache.storm.trident.state.State; -import org.apache.storm.trident.tuple.TridentTuple; import org.apache.storm.trident.tuple.TridentTuple.Factory; +import org.apache.storm.trident.tuple.TridentTuple; import org.apache.storm.trident.tuple.TridentTupleView.ProjectionFactory; import org.apache.storm.tuple.Fields; - public class StateQueryProcessor implements TridentProcessor { QueryFunction function; State state; @@ -44,7 +49,8 @@ public StateQueryProcessor(String stateId, Fields inputFields, QueryFunction fun } @Override - public void prepare(Map conf, TopologyContext context, TridentContext tridentContext) { + public void prepare(Map conf, TopologyContext context, + TridentContext tridentContext) { List parents = tridentContext.getParentTupleFactories(); if (parents.size() != 1) { throw new RuntimeException("State query operation can only have one parent"); @@ -82,10 +88,12 @@ public void flush() { public void finishBatch(ProcessorContext processorContext) { BatchState state = (BatchState) processorContext.state[context.getStateIndex()]; if (!state.tuples.isEmpty()) { - List results = function.batchRetrieve(this.state, Collections.unmodifiableList(state.args)); + List results = function.batchRetrieve(this.state, Collections + .unmodifiableList(state.args)); if (results.size() != state.tuples.size()) { throw new RuntimeException( - "Results size is different than argument size: " + results.size() + " vs " + state.tuples.size()); + "Results size is different than argument size: " + results.size() + " vs " + + state.tuples.size()); } for (int i = 0; i < state.tuples.size(); i++) { TridentTuple tuple = state.tuples.get(i); diff --git a/storm-client/src/jvm/org/apache/storm/trident/planner/processor/TridentContext.java b/storm-client/src/jvm/org/apache/storm/trident/planner/processor/TridentContext.java index 62270a34221..b1964214cda 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/planner/processor/TridentContext.java +++ b/storm-client/src/jvm/org/apache/storm/trident/planner/processor/TridentContext.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -18,7 +24,6 @@ import org.apache.storm.trident.tuple.TridentTuple.Factory; import org.apache.storm.tuple.Fields; - public class TridentContext { Fields selfFields; List parentFactories; @@ -64,7 +69,7 @@ public int getStateIndex() { return stateIndex; } - //for reporting errors + // for reporting errors public BatchOutputCollector getDelegateCollector() { return collector; } diff --git a/storm-client/src/jvm/org/apache/storm/trident/spout/BatchSpoutExecutor.java b/storm-client/src/jvm/org/apache/storm/trident/spout/BatchSpoutExecutor.java index d86269ac81c..7f1755b1c9a 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/spout/BatchSpoutExecutor.java +++ b/storm-client/src/jvm/org/apache/storm/trident/spout/BatchSpoutExecutor.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -26,12 +32,14 @@ public BatchSpoutExecutor(IBatchSpout spout) { } @Override - public BatchCoordinator getCoordinator(String txStateId, Map conf, TopologyContext context) { + public BatchCoordinator getCoordinator(String txStateId, Map conf, + TopologyContext context) { return new EmptyCoordinator(); } @Override - public Emitter getEmitter(String txStateId, Map conf, TopologyContext context) { + public Emitter getEmitter(String txStateId, Map conf, + TopologyContext context) { spout.open(conf, context); return new BatchSpoutEmitter(); } @@ -69,7 +77,8 @@ public boolean isReady(long txid) { public class BatchSpoutEmitter implements Emitter { @Override - public void emitBatch(TransactionAttempt tx, Object coordinatorMeta, TridentCollector collector) { + public void emitBatch(TransactionAttempt tx, Object coordinatorMeta, + TridentCollector collector) { spout.emitBatch(tx.getTransactionId(), collector); } diff --git a/storm-client/src/jvm/org/apache/storm/trident/spout/IBatchID.java b/storm-client/src/jvm/org/apache/storm/trident/spout/IBatchID.java index f4f5e4f6343..6887845832f 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/spout/IBatchID.java +++ b/storm-client/src/jvm/org/apache/storm/trident/spout/IBatchID.java @@ -1,18 +1,23 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.trident.spout; - @SuppressWarnings("checkstyle:AbbreviationAsWordInName") public interface IBatchID { Object getId(); diff --git a/storm-client/src/jvm/org/apache/storm/trident/spout/IBatchSpout.java b/storm-client/src/jvm/org/apache/storm/trident/spout/IBatchSpout.java index cba8dcc0850..8dd5a41175b 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/spout/IBatchSpout.java +++ b/storm-client/src/jvm/org/apache/storm/trident/spout/IBatchSpout.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/spout/ICommitterTridentSpout.java b/storm-client/src/jvm/org/apache/storm/trident/spout/ICommitterTridentSpout.java index ec4081a9fb5..cfe80236d6c 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/spout/ICommitterTridentSpout.java +++ b/storm-client/src/jvm/org/apache/storm/trident/spout/ICommitterTridentSpout.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/spout/IOpaquePartitionedTridentSpout.java b/storm-client/src/jvm/org/apache/storm/trident/spout/IOpaquePartitionedTridentSpout.java index 240a99b2256..c0729dc0784 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/spout/IOpaquePartitionedTridentSpout.java +++ b/storm-client/src/jvm/org/apache/storm/trident/spout/IOpaquePartitionedTridentSpout.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -16,24 +22,27 @@ import java.util.List; import java.util.Map; import java.util.Set; - import org.apache.storm.task.TopologyContext; import org.apache.storm.trident.operation.TridentCollector; import org.apache.storm.trident.topology.TransactionAttempt; import org.apache.storm.tuple.Fields; /** - * This defines a transactional spout which does *not* necessarily replay the same batch every time it emits a batch for a transaction id. + * This defines a transactional spout which does *not* necessarily replay the same batch every time + * it emits a batch for a transaction id. * - * @param The type of metadata object passed to the Emitter when emitting a new batch based on a previous batch. This type must + * @param The type of metadata object passed to the Emitter when emitting a new batch based on a + * previous batch. This type must * be JSON serializable by json-simple. - * @param The type of metadata object used by the coordinator to describe partitions. This type must be JSON serializable by + * @param The type of metadata object used by the coordinator to describe partitions. + * This type must be JSON serializable by * json-simple. */ public interface IOpaquePartitionedTridentSpout extends ITridentDataSource { - Emitter getEmitter(Map conf, TopologyContext context); + Emitter getEmitter(Map conf, + TopologyContext context); Coordinator getCoordinator(Map conf, TopologyContext context); @@ -42,14 +51,17 @@ public interface IOpaquePartitionedTridentSpout The type of metadata object used by the coordinator to describe partitions. This type must be JSON serializable + * @param The type of metadata object used by the coordinator to describe + * partitions. This type must be JSON serializable * by json-simple. */ interface Coordinator { /** - * Indicates whether this coordinator is ready to commit the given transaction. The master batch coordinator will only begin + * Indicates whether this coordinator is ready to commit the given transaction. The master + * batch coordinator will only begin * committing if at least one coordinator indicates it is ready to commit. * * @param txid The transaction id @@ -58,7 +70,8 @@ interface Coordinator { boolean isReady(long txid); /** - * Gets the partitions for the following batches. The emitter will be asked to refresh partitions when this value changes. + * Gets the partitions for the following batches. The emitter will be asked to refresh + * partitions when this value changes. * * @return The partitions for the following batches. */ @@ -71,14 +84,16 @@ interface Emitter { /** * Emit a batch of tuples for a list of partitions/transactions. * - *

    Return the map of metadata describing this batch that will be used as lastPartitionMeta for defining the + *

    Return the map of metadata describing this batch that will be used as + * lastPartitionMeta for defining the * parameters of the next batch for each partition. */ Map emitBatchNew(TransactionAttempt tx, TridentCollector collector, Set partitions, Map lastBatchMetaMap); /** - * This method is called when this task is responsible for a new set of partitions. Should be used to manage things like connections + * This method is called when this task is responsible for a new set of partitions. Should + * be used to manage things like connections * to brokers. * * @param partitionResponsibilities The partitions assigned to this task @@ -88,8 +103,10 @@ Map emitBatchNew(TransactionAttempt tx, TridentCollector collecto /** * Sorts the partition info to produce an ordered list of partition. * - * @param allPartitionInfo The partition info for all partitions being processed by all spout tasks - * @return The ordered list of partitions being processed by all the tasks. The ordering must be consistent for all tasks. + * @param allPartitionInfo The partition info for all partitions being processed by all + * spout tasks + * @return The ordered list of partitions being processed by all the tasks. The ordering + * must be consistent for all tasks. */ List getOrderedPartitions(PartitionsT allPartitionInfo); @@ -98,12 +115,15 @@ Map emitBatchNew(TransactionAttempt tx, TridentCollector collecto * * @param taskId The id of this task * @param numTasks The number of tasks for this spout - * @param allPartitionInfoSorted The partition info for all partitions being processed by all spout tasks, sorted according to + * @param allPartitionInfoSorted The partition info for all partitions being processed by + * all spout tasks, sorted according to * {@link #getOrderedPartitions(java.lang.Object)} * @return The list of partitions that are to be processed by the task with id {@code taskId} */ - default List getPartitionsForTask(int taskId, int numTasks, List allPartitionInfoSorted) { - final List taskPartitions = new ArrayList<>(allPartitionInfoSorted == null ? 0 : allPartitionInfoSorted.size()); + default List getPartitionsForTask(int taskId, int numTasks, + List allPartitionInfoSorted) { + final List taskPartitions = new ArrayList<>(allPartitionInfoSorted == null + ? 0 : allPartitionInfoSorted.size()); if (allPartitionInfoSorted != null) { for (int i = taskId; i < allPartitionInfoSorted.size(); i += numTasks) { taskPartitions.add(allPartitionInfoSorted.get(i)); diff --git a/storm-client/src/jvm/org/apache/storm/trident/spout/IPartitionedTridentSpout.java b/storm-client/src/jvm/org/apache/storm/trident/spout/IPartitionedTridentSpout.java index ac9eb4d9b50..32b96a0032c 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/spout/IPartitionedTridentSpout.java +++ b/storm-client/src/jvm/org/apache/storm/trident/spout/IPartitionedTridentSpout.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -22,14 +28,17 @@ import org.apache.storm.tuple.Fields; /** - * This interface defines a transactional spout that reads its tuples from a partitioned set of brokers. It automates the storing of - * metadata for each partition to ensure that the same batch is always emitted for the same transaction id. The partition metadata is stored + * This interface defines a transactional spout that reads its tuples from a partitioned set of + * brokers. It automates the storing of + * metadata for each partition to ensure that the same batch is always emitted for the same + * transaction id. The partition metadata is stored * in Zookeeper. */ public interface IPartitionedTridentSpout extends ITridentDataSource { Coordinator getCoordinator(Map conf, TopologyContext context); - Emitter getEmitter(Map conf, TopologyContext context); + Emitter getEmitter(Map conf, + TopologyContext context); Map getComponentConfiguration(); @@ -37,8 +46,10 @@ public interface IPartitionedTridentSpout { /** - * Return the partitions currently in the source of data. The idea is is that if a new partition is added and a prior transaction is - * replayed, it doesn't emit tuples for the new partition because it knows what partitions were in that transaction. + * Return the partitions currently in the source of data. The idea is is that if a new + * partition is added and a prior transaction is + * replayed, it doesn't emit tuples for the new partition because it knows what partitions + * were in that transaction. */ PartitionsT getPartitionsForBatch(); @@ -52,41 +63,51 @@ interface Emitter { /** * Sorts given partition info to produce an ordered list of partitions. * - * @param allPartitionInfo The partition info for all partitions being processed by all spout tasks - * @return sorted list of partitions being processed by all the tasks. The ordering must be consistent for all tasks. + * @param allPartitionInfo The partition info for all partitions being processed by all + * spout tasks + * @return sorted list of partitions being processed by all the tasks. The ordering must be + * consistent for all tasks. */ List getOrderedPartitions(PartitionsT allPartitionInfo); /** - * Emit a batch of tuples for the partitions that's never been emitted before. Return the metadata that can be used to + * Emit a batch of tuples for the partitions that's never been emitted before. Return the + * metadata that can be used to * reconstruct this partition/batch in the future. */ - Map emitBatchNew(TransactionAttempt tx, TridentCollector collector, Set partitions, + Map emitBatchNew(TransactionAttempt tx, TridentCollector collector, + Set partitions, Map lastPartitionMetaMap); /** - * This method is called when this task is responsible for a new set of partitions. Should be used to manage things like connections + * This method is called when this task is responsible for a new set of partitions. Should + * be used to manage things like connections * to brokers. */ void refreshPartitions(List partitionResponsibilities); /** - * Emit a batch of tuples for a partition/transaction that has been emitted before, using the metadata created when it was first + * Emit a batch of tuples for a partition/transaction that has been emitted before, using + * the metadata created when it was first * emitted. */ - void reEmitPartitionBatch(TransactionAttempt tx, TridentCollector collector, PartitionT partition, X partitionMeta); + void reEmitPartitionBatch(TransactionAttempt tx, TridentCollector collector, + PartitionT partition, X partitionMeta); /** * Get the partitions assigned to the given task. * * @param taskId The id of the task * @param numTasks The number of tasks for the spout - * @param allPartitionInfoSorted The partition info of all partitions being processed by all spout tasks, sorted according to + * @param allPartitionInfoSorted The partition info of all partitions being processed by all + * spout tasks, sorted according to * {@link #getOrderedPartitions(java.lang.Object)} * @return The list of partitions that are to be processed by the task with {@code taskId} */ - default List getPartitionsForTask(int taskId, int numTasks, List allPartitionInfoSorted) { - List taskPartitions = new ArrayList<>(allPartitionInfoSorted == null ? 0 : allPartitionInfoSorted.size()); + default List getPartitionsForTask(int taskId, int numTasks, + List allPartitionInfoSorted) { + List taskPartitions = new ArrayList<>(allPartitionInfoSorted == null + ? 0 : allPartitionInfoSorted.size()); if (allPartitionInfoSorted != null) { for (int i = taskId; i < allPartitionInfoSorted.size(); i += numTasks) { taskPartitions.add(allPartitionInfoSorted.get(i)); diff --git a/storm-client/src/jvm/org/apache/storm/trident/spout/ISpoutPartition.java b/storm-client/src/jvm/org/apache/storm/trident/spout/ISpoutPartition.java index bf7d5bd692c..9a4b6097f92 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/spout/ISpoutPartition.java +++ b/storm-client/src/jvm/org/apache/storm/trident/spout/ISpoutPartition.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/spout/ITridentSpout.java b/storm-client/src/jvm/org/apache/storm/trident/spout/ITridentSpout.java index b9ab8d549b3..f15a1f6e276 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/spout/ITridentSpout.java +++ b/storm-client/src/jvm/org/apache/storm/trident/spout/ITridentSpout.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -18,15 +24,18 @@ import org.apache.storm.trident.topology.TransactionAttempt; import org.apache.storm.tuple.Fields; - public interface ITridentSpout extends ITridentDataSource { /** - * The coordinator for a TransactionalSpout runs in a single thread and indicates when batches of tuples should be - * emitted. The Coordinator that you provide in a TransactionalSpout provides metadata for each transaction so that + * The coordinator for a TransactionalSpout runs in a single thread and indicates when batches + * of tuples should be + * emitted. The Coordinator that you provide in a TransactionalSpout provides metadata for each + * transaction so that * the transactions can be replayed in case of failure. * - *

    Two instances are requested, one on the master batch coordinator where isReady() is called, and an instance in - * the coordinator bolt which is used for all other operations. The two instances do not necessarily share a worker + *

    Two instances are requested, one on the master batch coordinator where isReady() is + * called, and an instance in + * the coordinator bolt which is used for all other operations. The two instances do not + * necessarily share a worker * JVM. * * @param txStateId stream id @@ -34,14 +43,18 @@ public interface ITridentSpout extends ITridentDataSource { * @param context topology context * @return spout coordinator instance */ - BatchCoordinator getCoordinator(String txStateId, Map conf, TopologyContext context); + BatchCoordinator getCoordinator(String txStateId, Map conf, + TopologyContext context); /** - * The emitter for a TransactionalSpout runs as many tasks across the cluster. Emitters are responsible for emitting - * batches of tuples for a transaction and must ensure that the same batch of tuples is always emitted for the same + * The emitter for a TransactionalSpout runs as many tasks across the cluster. Emitters are + * responsible for emitting + * batches of tuples for a transaction and must ensure that the same batch of tuples is always + * emitted for the same * transaction id. * - *

    All emitter tasks get the same transaction metadata. The topology context parameter contains the instance task + *

    All emitter tasks get the same transaction metadata. The topology context parameter + * contains the instance task * id that can be used to distribute the work across the tasks. * * @param txStateId stream id @@ -57,30 +70,35 @@ public interface ITridentSpout extends ITridentDataSource { interface BatchCoordinator { /** - * Create metadata for this particular transaction id which has never been emitted before. The metadata should - * contain whatever is necessary to be able to replay the exact batch for the transaction at a later point. + * Create metadata for this particular transaction id which has never been emitted before. + * The metadata should + * contain whatever is necessary to be able to replay the exact batch for the transaction at + * a later point. * *

    The metadata is stored in Zookeeper. * - *

    Storm uses JSON encoding to store the metadata. Only simple types such as numbers, booleans, strings, + *

    Storm uses JSON encoding to store the metadata. Only simple types such as numbers, + * booleans, strings, * lists, and maps should be used. * * @param txid The id of the transaction. * @param prevMetadata The metadata of the previous transaction - * @param currMetadata The metadata for this transaction the last time it was initialized. null if this is the first attempt + * @param currMetadata The metadata for this transaction the last time it was initialized. + * null if this is the first attempt * @return the metadata for this new transaction */ X initializeTransaction(long txid, X prevMetadata, X currMetadata); /** - * This attempt committed successfully, so all state for this commit and before can be safely cleaned up. + * This attempt committed successfully, so all state for this commit and before can be + * safely cleaned up. * * @param txid transaction id that completed */ void success(long txid); /** - * hint to Storm if the spout is ready for the transaction id. + * Hint to Storm if the spout is ready for the transaction id. * * @param txid the id of the transaction * @return true, if the spout is ready for the given transaction id @@ -95,8 +113,10 @@ interface BatchCoordinator { interface Emitter { /** - * Emit a batch for the specified transaction attempt and metadata for the transaction. The metadata was created by the Coordinator - * in the initializeTransaction method. This method must always emit the same batch of tuples across all tasks for the same + * Emit a batch for the specified transaction attempt and metadata for the transaction. The + * metadata was created by the Coordinator + * in the initializeTransaction method. This method must always emit the same batch of + * tuples across all tasks for the same * transaction id. * * @param tx transaction id @@ -106,7 +126,8 @@ interface Emitter { void emitBatch(TransactionAttempt tx, X coordinatorMeta, TridentCollector collector); /** - * This attempt committed successfully, so all state for this commit and before can be safely cleaned up. + * This attempt committed successfully, so all state for this commit and before can be + * safely cleaned up. * * @param tx attempt object containing transaction id and attempt number */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/spout/OpaquePartitionedTridentSpoutExecutor.java b/storm-client/src/jvm/org/apache/storm/trident/spout/OpaquePartitionedTridentSpoutExecutor.java index 141d55f6b78..55f9a05f5bd 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/spout/OpaquePartitionedTridentSpoutExecutor.java +++ b/storm-client/src/jvm/org/apache/storm/trident/spout/OpaquePartitionedTridentSpoutExecutor.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -15,11 +21,10 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Map.Entry; +import java.util.Map; import java.util.Set; import java.util.TreeMap; - import org.apache.storm.task.TopologyContext; import org.apache.storm.trident.operation.TridentCollector; import org.apache.storm.trident.topology.TransactionAttempt; @@ -29,23 +34,26 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; - public class OpaquePartitionedTridentSpoutExecutor implements ICommitterTridentSpout { - protected static final Logger LOG = LoggerFactory.getLogger(OpaquePartitionedTridentSpoutExecutor.class); + protected static final Logger LOG = LoggerFactory + .getLogger(OpaquePartitionedTridentSpoutExecutor.class); IOpaquePartitionedTridentSpout spout; - public OpaquePartitionedTridentSpoutExecutor(IOpaquePartitionedTridentSpout spout) { + public OpaquePartitionedTridentSpoutExecutor(IOpaquePartitionedTridentSpout spout) { this.spout = spout; } @Override - public ITridentSpout.BatchCoordinator getCoordinator(String txStateId, Map conf, TopologyContext context) { + public ITridentSpout.BatchCoordinator getCoordinator(String txStateId, Map conf, TopologyContext context) { return new Coordinator(conf, context); } @Override - public ICommitterTridentSpout.Emitter getEmitter(String txStateId, Map conf, TopologyContext context) { + public ICommitterTridentSpout.Emitter getEmitter(String txStateId, Map conf, + TopologyContext context) { return new Emitter(txStateId, conf, context); } @@ -78,11 +86,11 @@ public Coordinator(Map conf, TopologyContext context) { @Override public Object initializeTransaction(long txid, Object prevMetadata, Object currMetadata) { - LOG.debug("Initialize Transaction. [txid = {}], [prevMetadata = {}], [currMetadata = {}]", txid, prevMetadata, currMetadata); + LOG.debug("Initialize Transaction. [txid = {}], [prevMetadata = {}], [currMetadata = " + + "{}]", txid, prevMetadata, currMetadata); return coordinator.getPartitionsForBatch(); } - @Override public void close() { LOG.debug("Closing"); @@ -122,14 +130,18 @@ public Emitter(String txStateId, Map conf, TopologyContext conte } @Override - public void emitBatch(TransactionAttempt tx, Object coordinatorMeta, TridentCollector collector) { - LOG.debug("Emitting Batch. [transaction = {}], [coordinatorMeta = {}], [collector = {}], [{}]", + public void emitBatch(TransactionAttempt tx, Object coordinatorMeta, + TridentCollector collector) { + LOG.debug("Emitting Batch. [transaction = {}], [coordinatorMeta = {}], [collector = " + + "{}], [{}]", tx, coordinatorMeta, collector, this); if (savedCoordinatorMeta == null || !savedCoordinatorMeta.equals(coordinatorMeta)) { partitionStates.clear(); - final List sortedPartitions = emitter.getOrderedPartitions(coordinatorMeta); - final List taskPartitions = emitter.getPartitionsForTask(index, numTasks, sortedPartitions); + final List sortedPartitions = emitter + .getOrderedPartitions(coordinatorMeta); + final List taskPartitions = emitter.getPartitionsForTask(index, + numTasks, sortedPartitions); for (ISpoutPartition partition : taskPartitions) { partitionStates.put(partition.getId(), new EmitterPartitionState(new RotatingTransactionalState(state, partition.getId()), partition)); @@ -163,11 +175,13 @@ public void emitBatch(TransactionAttempt tx, Object coordinatorMeta, TridentColl partitions.add(s.partition); } - Map partitionMetaMap = emitter.emitBatchNew(tx, collector, partitions, prevStateMap); + Map partitionMetaMap = emitter.emitBatchNew(tx, collector, + partitions, prevStateMap); for (Map.Entry partitionMeta : partitionMetaMap.entrySet()) { metas.put(partitionMeta.getKey().getId(), partitionMeta.getValue()); } - LOG.debug("Emitted Batch. [transaction = {}], [coordinatorMeta = {}], [collector = {}], [{}]", + LOG.debug("Emitted Batch. [transaction = {}], [coordinatorMeta = {}], [collector = " + + "{}], [{}]", tx, coordinatorMeta, collector, this); } @@ -188,8 +202,10 @@ public void commit(TransactionAttempt attempt) { // we make sure only a single task ever does this. we're also guaranteed that // it's impossible for there to be another writer to the directory for that partition // because only a single commit can be happening at once. this is because in order for - // another attempt of the batch to commit, the batch phase must have succeeded in between. - // hence, all tasks for the prior commit must have finished committing (whether successfully or not) + // another attempt of the batch to commit, the batch phase must have succeeded in + // between. + // hence, all tasks for the prior commit must have finished committing (whether + // successfully or not) if (changedMeta && index == 0) { Set validIds = new HashSet<>(); for (ISpoutPartition p : emitter.getOrderedPartitions(savedCoordinatorMeta)) { @@ -197,7 +213,8 @@ public void commit(TransactionAttempt attempt) { } for (String existingPartition : state.list("")) { if (!validIds.contains(existingPartition)) { - RotatingTransactionalState s = new RotatingTransactionalState(state, existingPartition); + RotatingTransactionalState s = new RotatingTransactionalState(state, + existingPartition); s.removeState(attempt.getTransactionId()); } } @@ -207,7 +224,8 @@ public void commit(TransactionAttempt attempt) { Long txid = attempt.getTransactionId(); Map metas = cachedMetas.remove(txid); for (Entry entry : metas.entrySet()) { - partitionStates.get(entry.getKey()).rotatingState.overrideState(txid, entry.getValue()); + partitionStates.get(entry.getKey()).rotatingState.overrideState(txid, entry + .getValue()); } LOG.debug("Exiting commit method for transaction {}. [{}]", attempt, this); } diff --git a/storm-client/src/jvm/org/apache/storm/trident/spout/PartitionedTridentSpoutExecutor.java b/storm-client/src/jvm/org/apache/storm/trident/spout/PartitionedTridentSpoutExecutor.java index da51563cbfd..a21de73216e 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/spout/PartitionedTridentSpoutExecutor.java +++ b/storm-client/src/jvm/org/apache/storm/trident/spout/PartitionedTridentSpoutExecutor.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -26,13 +32,14 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; - public class PartitionedTridentSpoutExecutor implements ITridentSpout { - private static final Logger LOG = LoggerFactory.getLogger(PartitionedTridentSpoutExecutor.class); + private static final Logger LOG = LoggerFactory + .getLogger(PartitionedTridentSpoutExecutor.class); IPartitionedTridentSpout spout; - public PartitionedTridentSpoutExecutor(IPartitionedTridentSpout spout) { + public PartitionedTridentSpoutExecutor(IPartitionedTridentSpout spout) { this.spout = spout; } @@ -41,12 +48,14 @@ public IPartitionedTridentSpout getPartitionedS } @Override - public ITridentSpout.BatchCoordinator getCoordinator(String txStateId, Map conf, TopologyContext context) { + public ITridentSpout.BatchCoordinator getCoordinator(String txStateId, Map conf, TopologyContext context) { return new Coordinator(conf, context); } @Override - public ITridentSpout.Emitter getEmitter(String txStateId, Map conf, TopologyContext context) { + public ITridentSpout.Emitter getEmitter(String txStateId, Map conf, + TopologyContext context) { return new Emitter(txStateId, conf, context); } @@ -79,7 +88,8 @@ class Coordinator implements ITridentSpout.BatchCoordinator { @Override public Object initializeTransaction(long txid, Object prevMetadata, Object currMetadata) { - LOG.debug("Initialize Transaction. txid = {}, prevMetadata = {}, currMetadata = {}", txid, prevMetadata, currMetadata); + LOG.debug("Initialize Transaction. txid = {}, prevMetadata = {}, currMetadata = {}", + txid, prevMetadata, currMetadata); if (currMetadata != null) { return currMetadata; @@ -88,7 +98,6 @@ public Object initializeTransaction(long txid, Object prevMetadata, Object currM } } - @Override public void close() { LOG.debug("Closing"); @@ -125,8 +134,10 @@ class Emitter implements ITridentSpout.Emitter { } @Override - public void emitBatch(final TransactionAttempt tx, final Object coordinatorMeta, final TridentCollector collector) { - LOG.debug("Emitting Batch. [transaction = {}], [coordinatorMeta = {}], [collector = {}]", tx, coordinatorMeta, collector); + public void emitBatch(final TransactionAttempt tx, final Object coordinatorMeta, + final TridentCollector collector) { + LOG.debug("Emitting Batch. [transaction = {}], [coordinatorMeta = {}], [collector = " + + "{}]", tx, coordinatorMeta, collector); if (savedCoordinatorMeta == null || !savedCoordinatorMeta.equals(coordinatorMeta)) { partitionStates.clear(); @@ -134,7 +145,8 @@ public void emitBatch(final TransactionAttempt tx, final Object coordinatorMeta, emitter.getOrderedPartitions(coordinatorMeta)); for (ISpoutPartition partition : taskPartitions) { partitionStates.put(partition.getId(), - new EmitterPartitionState(new RotatingTransactionalState(state, partition.getId()), partition)); + new EmitterPartitionState(new RotatingTransactionalState(state, + partition.getId()), partition)); } emitter.refreshPartitions(taskPartitions); @@ -143,7 +155,8 @@ public void emitBatch(final TransactionAttempt tx, final Object coordinatorMeta, Map prevStateMap = new HashMap<>(); Set partitions = new HashSet<>(); for (EmitterPartitionState s : partitionStates.values()) { - prevStateMap.put(s.partition, s.rotatingState.getPreviousState(tx.getTransactionId())); + prevStateMap.put(s.partition, s.rotatingState.getPreviousState(tx + .getTransactionId())); partitions.add(s.partition); } boolean isNewBatch = true; @@ -154,11 +167,13 @@ public void emitBatch(final TransactionAttempt tx, final Object coordinatorMeta, } } if (isNewBatch) { - Map partitionToBatchMeta = emitter.emitBatchNew(tx, collector, partitions, prevStateMap); + Map partitionToBatchMeta = emitter.emitBatchNew(tx, + collector, partitions, prevStateMap); for (Map.Entry entry : partitionToBatchMeta.entrySet()) { ISpoutPartition partition = entry.getKey(); Object batchMeta = entry.getValue(); - partitionStates.get(partition.getId()).rotatingState.getStateOrCreate(tx.getTransactionId(), + partitionStates.get(partition.getId()).rotatingState.getStateOrCreate(tx + .getTransactionId(), new RotatingTransactionalState.StateInitializer() { @Override public Object init(long txid, Object lastState) { @@ -173,7 +188,8 @@ public Object init(long txid, Object lastState) { emitter.reEmitPartitionBatch(tx, collector, s.partition, partitionBatchMeta); } } - LOG.debug("Emitted Batch. [tx = {}], [coordinatorMeta = {}], [collector = {}]", tx, coordinatorMeta, collector); + LOG.debug("Emitted Batch. [tx = {}], [coordinatorMeta = {}], [collector = {}]", tx, + coordinatorMeta, collector); } @Override diff --git a/storm-client/src/jvm/org/apache/storm/trident/spout/RichSpoutBatchExecutor.java b/storm-client/src/jvm/org/apache/storm/trident/spout/RichSpoutBatchExecutor.java index 7e87dee3c60..6eecdb5f3f9 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/spout/RichSpoutBatchExecutor.java +++ b/storm-client/src/jvm/org/apache/storm/trident/spout/RichSpoutBatchExecutor.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -47,12 +53,14 @@ public Fields getOutputFields() { } @Override - public BatchCoordinator getCoordinator(String txStateId, Map conf, TopologyContext context) { + public BatchCoordinator getCoordinator(String txStateId, Map conf, + TopologyContext context) { return new RichSpoutCoordinator(); } @Override - public Emitter getEmitter(String txStateId, Map conf, TopologyContext context) { + public Emitter getEmitter(String txStateId, Map conf, + TopologyContext context) { return new RichSpoutEmitter(conf, context); } @@ -139,18 +147,20 @@ class RichSpoutEmitter implements ITridentSpout.Emitter { maxBatchSize = batchSize.intValue(); collector = new CaptureCollector(); idsMap = new RotatingMap<>(3); - rotateTime = 1000L * ((Number) conf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS)).intValue(); + rotateTime = 1000L * ((Number) conf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS)) + .intValue(); } @Override - public void emitBatch(TransactionAttempt tx, Object coordinatorMeta, TridentCollector collector) { + public void emitBatch(TransactionAttempt tx, Object coordinatorMeta, + TridentCollector collector) { long txid = tx.getTransactionId(); long now = System.currentTimeMillis(); if (now - lastRotate > rotateTime) { Map> failed = idsMap.rotate(); for (Long id : failed.keySet()) { - //TODO: this isn't right... it's not in the map anymore + // TODO: this isn't right... it's not in the map anymore fail(id); } lastRotate = now; diff --git a/storm-client/src/jvm/org/apache/storm/trident/spout/RichSpoutBatchId.java b/storm-client/src/jvm/org/apache/storm/trident/spout/RichSpoutBatchId.java index 22b4441cfe1..a2d3fe4b941 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/spout/RichSpoutBatchId.java +++ b/storm-client/src/jvm/org/apache/storm/trident/spout/RichSpoutBatchId.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/spout/RichSpoutBatchIdSerializer.java b/storm-client/src/jvm/org/apache/storm/trident/spout/RichSpoutBatchIdSerializer.java index 511a0eb680e..df32647e87e 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/spout/RichSpoutBatchIdSerializer.java +++ b/storm-client/src/jvm/org/apache/storm/trident/spout/RichSpoutBatchIdSerializer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -17,7 +23,6 @@ import com.esotericsoftware.kryo.io.Input; import com.esotericsoftware.kryo.io.Output; - public class RichSpoutBatchIdSerializer extends Serializer { @Override diff --git a/storm-client/src/jvm/org/apache/storm/trident/spout/RichSpoutBatchTriggerer.java b/storm-client/src/jvm/org/apache/storm/trident/spout/RichSpoutBatchTriggerer.java index 378824d91b7..3c4624b0453 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/spout/RichSpoutBatchTriggerer.java +++ b/storm-client/src/jvm/org/apache/storm/trident/spout/RichSpoutBatchTriggerer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -33,7 +39,6 @@ import org.apache.storm.tuple.Values; import org.apache.storm.utils.Utils; - public class RichSpoutBatchTriggerer implements IRichSpout { String stream; @@ -51,8 +56,10 @@ public RichSpoutBatchTriggerer(IRichSpout delegate, String streamName, String ba } @Override - public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { - delegate.open(conf, context, new SpoutOutputCollector(new StreamOverrideCollector(collector))); + public void open(Map conf, TopologyContext context, + SpoutOutputCollector collector) { + delegate.open(conf, context, + new SpoutOutputCollector(new StreamOverrideCollector(collector))); outputTasks = new ArrayList<>(); for (String component : Utils.get(context.getThisTargets(), coordStream, @@ -121,7 +128,8 @@ public Map getComponentConfiguration() { } else { conf = new HashMap<>(conf); } - Config.registerSerialization(conf, RichSpoutBatchId.class, RichSpoutBatchIdSerializer.class); + Config.registerSerialization(conf, RichSpoutBatchId.class, + RichSpoutBatchIdSerializer.class); return conf; } diff --git a/storm-client/src/jvm/org/apache/storm/trident/spout/TridentSpoutCoordinator.java b/storm-client/src/jvm/org/apache/storm/trident/spout/TridentSpoutCoordinator.java index 7230726c9b7..f6a500a022e 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/spout/TridentSpoutCoordinator.java +++ b/storm-client/src/jvm/org/apache/storm/trident/spout/TridentSpoutCoordinator.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -28,7 +34,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; - public class TridentSpoutCoordinator implements IBasicBolt { public static final Logger LOG = LoggerFactory.getLogger(TridentSpoutCoordinator.class); private static final String META_DIR = "meta"; @@ -39,7 +44,6 @@ public class TridentSpoutCoordinator implements IBasicBolt { TransactionalState underlyingState; String id; - public TridentSpoutCoordinator(String id, ITridentSpout spout) { this.spout = spout; this.id = id; @@ -77,7 +81,8 @@ public void cleanup() { @Override public void declareOutputFields(OutputFieldsDeclarer declarer) { - declarer.declareStream(MasterBatchCoordinator.BATCH_STREAM_ID, new Fields("tx", "metadata")); + declarer.declareStream(MasterBatchCoordinator.BATCH_STREAM_ID, new Fields("tx", + "metadata")); } @Override diff --git a/storm-client/src/jvm/org/apache/storm/trident/spout/TridentSpoutExecutor.java b/storm-client/src/jvm/org/apache/storm/trident/spout/TridentSpoutExecutor.java index e45ec3ec9e7..5e088d92f9e 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/spout/TridentSpoutExecutor.java +++ b/storm-client/src/jvm/org/apache/storm/trident/spout/TridentSpoutExecutor.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -51,7 +57,8 @@ public TridentSpoutExecutor(String txStateId, String streamName, ITridentSpout conf, TopologyContext context, BatchOutputCollector collector) { + public void prepare(Map conf, TopologyContext context, + BatchOutputCollector collector) { emitter = spout.getEmitter(txStateId, conf, context); this.collector = new AddIdCollector(streamName, collector); } @@ -115,7 +122,6 @@ private static class AddIdCollector implements TridentCollector { this.stream = stream; } - public void setBatch(Object id) { this.id = id; } diff --git a/storm-client/src/jvm/org/apache/storm/trident/state/BaseQueryFunction.java b/storm-client/src/jvm/org/apache/storm/trident/state/BaseQueryFunction.java index 3179fc6bda0..e9571ba8f46 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/state/BaseQueryFunction.java +++ b/storm-client/src/jvm/org/apache/storm/trident/state/BaseQueryFunction.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -14,7 +20,6 @@ import org.apache.storm.trident.operation.BaseOperation; - public abstract class BaseQueryFunction extends BaseOperation implements QueryFunction { } diff --git a/storm-client/src/jvm/org/apache/storm/trident/state/BaseStateUpdater.java b/storm-client/src/jvm/org/apache/storm/trident/state/BaseStateUpdater.java index ebcd769600c..06cd3a57607 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/state/BaseStateUpdater.java +++ b/storm-client/src/jvm/org/apache/storm/trident/state/BaseStateUpdater.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -14,7 +20,6 @@ import org.apache.storm.trident.operation.BaseOperation; - public abstract class BaseStateUpdater extends BaseOperation implements StateUpdater { } diff --git a/storm-client/src/jvm/org/apache/storm/trident/state/CombinerValueUpdater.java b/storm-client/src/jvm/org/apache/storm/trident/state/CombinerValueUpdater.java index 7827ec7264c..a570f2b5c35 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/state/CombinerValueUpdater.java +++ b/storm-client/src/jvm/org/apache/storm/trident/state/CombinerValueUpdater.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/state/ITupleCollection.java b/storm-client/src/jvm/org/apache/storm/trident/state/ITupleCollection.java index 61108b82b41..eadf31c9d32 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/state/ITupleCollection.java +++ b/storm-client/src/jvm/org/apache/storm/trident/state/ITupleCollection.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/state/JSONNonTransactionalSerializer.java b/storm-client/src/jvm/org/apache/storm/trident/state/JSONNonTransactionalSerializer.java index 3eae7dd6a1c..f1177cbc6b2 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/state/JSONNonTransactionalSerializer.java +++ b/storm-client/src/jvm/org/apache/storm/trident/state/JSONNonTransactionalSerializer.java @@ -1,19 +1,24 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.trident.state; import java.nio.charset.StandardCharsets; - import org.apache.storm.shade.net.minidev.json.JSONValue; import org.apache.storm.shade.net.minidev.json.parser.ParseException; diff --git a/storm-client/src/jvm/org/apache/storm/trident/state/JSONOpaqueSerializer.java b/storm-client/src/jvm/org/apache/storm/trident/state/JSONOpaqueSerializer.java index 97f9da65700..4c44583699f 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/state/JSONOpaqueSerializer.java +++ b/storm-client/src/jvm/org/apache/storm/trident/state/JSONOpaqueSerializer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/state/JSONTransactionalSerializer.java b/storm-client/src/jvm/org/apache/storm/trident/state/JSONTransactionalSerializer.java index f04ed36df6d..5059599b56b 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/state/JSONTransactionalSerializer.java +++ b/storm-client/src/jvm/org/apache/storm/trident/state/JSONTransactionalSerializer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/state/OpaqueValue.java b/storm-client/src/jvm/org/apache/storm/trident/state/OpaqueValue.java index fc22b6b30d3..030e10ed2c4 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/state/OpaqueValue.java +++ b/storm-client/src/jvm/org/apache/storm/trident/state/OpaqueValue.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -36,7 +42,8 @@ public OpaqueValue update(Long batchTxid, T newVal) { } else if (batchTxid.equals(this.currTxid)) { prev = this.prev; } else { - throw new RuntimeException("Current batch (" + batchTxid + ") is behind state's batch: " + this.toString()); + throw new RuntimeException("Current batch (" + batchTxid + + ") is behind state's batch: " + this.toString()); } return new OpaqueValue(batchTxid, newVal, prev); } @@ -47,7 +54,8 @@ public T get(Long batchTxid) { } else if (batchTxid.equals(this.currTxid)) { return prev; } else { - throw new RuntimeException("Current batch (" + batchTxid + ") is behind state's batch: " + this.toString()); + throw new RuntimeException("Current batch (" + batchTxid + + ") is behind state's batch: " + this.toString()); } } diff --git a/storm-client/src/jvm/org/apache/storm/trident/state/QueryFunction.java b/storm-client/src/jvm/org/apache/storm/trident/state/QueryFunction.java index 416ab954913..0d315ddacfe 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/state/QueryFunction.java +++ b/storm-client/src/jvm/org/apache/storm/trident/state/QueryFunction.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/state/ReadOnlyState.java b/storm-client/src/jvm/org/apache/storm/trident/state/ReadOnlyState.java index 73c68cb4b89..206ed791dc8 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/state/ReadOnlyState.java +++ b/storm-client/src/jvm/org/apache/storm/trident/state/ReadOnlyState.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -16,11 +22,13 @@ public class ReadOnlyState implements State { @Override public void beginCommit(Long txid) { - throw new UnsupportedOperationException("This state is read-only and does not support updates"); + throw new UnsupportedOperationException("This state is read-only and does not support " + + "updates"); } @Override public void commit(Long txid) { - throw new UnsupportedOperationException("This state is read-only and does not support updates"); + throw new UnsupportedOperationException("This state is read-only and does not support " + + "updates"); } } diff --git a/storm-client/src/jvm/org/apache/storm/trident/state/ReducerValueUpdater.java b/storm-client/src/jvm/org/apache/storm/trident/state/ReducerValueUpdater.java index 7ba4f0f0af5..d29ec3c52e5 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/state/ReducerValueUpdater.java +++ b/storm-client/src/jvm/org/apache/storm/trident/state/ReducerValueUpdater.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/state/Serializer.java b/storm-client/src/jvm/org/apache/storm/trident/state/Serializer.java index 955a0d81bc5..e1c17716b89 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/state/Serializer.java +++ b/storm-client/src/jvm/org/apache/storm/trident/state/Serializer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -14,7 +20,6 @@ import java.io.Serializable; - public interface Serializer extends Serializable { byte[] serialize(T obj); diff --git a/storm-client/src/jvm/org/apache/storm/trident/state/State.java b/storm-client/src/jvm/org/apache/storm/trident/state/State.java index 19cf17d94dd..1b59aef8614 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/state/State.java +++ b/storm-client/src/jvm/org/apache/storm/trident/state/State.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -15,14 +21,19 @@ /** * There's 3 different kinds of state: * - *

    1. non-transactional: ignores commits, updates are permanent. no rollback. a cassandra incrementing state would be like this 2. - * repeat-transactional: idempotent as long as all batches for a txid are identical 3. opaque-transactional: the most general kind of state. - * updates are always done based on the previous version of the value if the current commit = latest stored commit Idempotent even if the + *

    1. non-transactional: ignores commits, updates are permanent. no rollback. a cassandra + * incrementing state would be like this 2. + * repeat-transactional: idempotent as long as all batches for a txid are identical 3. + * opaque-transactional: the most general kind of state. + * updates are always done based on the previous version of the value if the current commit = latest + * stored commit Idempotent even if the * batch for a txid can change. * - *

    repeat transactional is idempotent for transactional spouts opaque transactional is idempotent for opaque or transactional spouts + *

    repeat transactional is idempotent for transactional spouts opaque transactional is idempotent + * for opaque or transactional spouts * - *

    Trident should log warnings when state is idempotent but updates will not be idempotent because of spout + *

    Trident should log warnings when state is idempotent but updates will not be idempotent + * because of spout */ // retrieving is encapsulated in Retrieval interface public interface State { diff --git a/storm-client/src/jvm/org/apache/storm/trident/state/StateFactory.java b/storm-client/src/jvm/org/apache/storm/trident/state/StateFactory.java index 3b5b4671223..77b8d945762 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/state/StateFactory.java +++ b/storm-client/src/jvm/org/apache/storm/trident/state/StateFactory.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -17,5 +23,6 @@ import org.apache.storm.task.IMetricsContext; public interface StateFactory extends Serializable { - State makeState(Map conf, IMetricsContext metrics, int partitionIndex, int numPartitions); + State makeState(Map conf, IMetricsContext metrics, int partitionIndex, + int numPartitions); } diff --git a/storm-client/src/jvm/org/apache/storm/trident/state/StateSpec.java b/storm-client/src/jvm/org/apache/storm/trident/state/StateSpec.java index a3793a7fd73..1a6c3e55af7 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/state/StateSpec.java +++ b/storm-client/src/jvm/org/apache/storm/trident/state/StateSpec.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -14,7 +20,6 @@ import java.io.Serializable; - public class StateSpec implements Serializable { public StateFactory stateFactory; public Integer requiredNumPartitions = null; diff --git a/storm-client/src/jvm/org/apache/storm/trident/state/StateType.java b/storm-client/src/jvm/org/apache/storm/trident/state/StateType.java index 3ae6137b396..9ff68898320 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/state/StateType.java +++ b/storm-client/src/jvm/org/apache/storm/trident/state/StateType.java @@ -1,18 +1,23 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.trident.state; - public enum StateType { NON_TRANSACTIONAL, TRANSACTIONAL, diff --git a/storm-client/src/jvm/org/apache/storm/trident/state/StateUpdater.java b/storm-client/src/jvm/org/apache/storm/trident/state/StateUpdater.java index 0342cd659ce..dcf980939dc 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/state/StateUpdater.java +++ b/storm-client/src/jvm/org/apache/storm/trident/state/StateUpdater.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -17,9 +23,9 @@ import org.apache.storm.trident.operation.TridentCollector; import org.apache.storm.trident.tuple.TridentTuple; - public interface StateUpdater extends Operation { - // maybe it needs a start phase (where it can do a retrieval, an update phase, and then a finish phase...? + // maybe it needs a start phase (where it can do a retrieval, an update phase, and then a finish + // phase...? // shouldn't really be a one-at-a-time interface, since we have all the tuples already? // TOOD: used for the new values stream // the list is needed to be able to get reduceragg and combineragg persistentaggregate diff --git a/storm-client/src/jvm/org/apache/storm/trident/state/TransactionalValue.java b/storm-client/src/jvm/org/apache/storm/trident/state/TransactionalValue.java index 7d1bfb2de4a..4b37cd907c0 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/state/TransactionalValue.java +++ b/storm-client/src/jvm/org/apache/storm/trident/state/TransactionalValue.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/state/ValueUpdater.java b/storm-client/src/jvm/org/apache/storm/trident/state/ValueUpdater.java index f116b45ed39..1cc74d5c706 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/state/ValueUpdater.java +++ b/storm-client/src/jvm/org/apache/storm/trident/state/ValueUpdater.java @@ -1,18 +1,23 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.trident.state; - public interface ValueUpdater { T update(T stored); } diff --git a/storm-client/src/jvm/org/apache/storm/trident/state/map/CachedBatchReadsMap.java b/storm-client/src/jvm/org/apache/storm/trident/state/map/CachedBatchReadsMap.java index bed36df97ae..58610e6616d 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/state/map/CachedBatchReadsMap.java +++ b/storm-client/src/jvm/org/apache/storm/trident/state/map/CachedBatchReadsMap.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -17,7 +23,6 @@ import java.util.List; import java.util.Map; - public class CachedBatchReadsMap { public IBackingMap delegate; Map, T> cached = new HashMap, T>(); diff --git a/storm-client/src/jvm/org/apache/storm/trident/state/map/CachedMap.java b/storm-client/src/jvm/org/apache/storm/trident/state/map/CachedMap.java index a0134a46afd..98ad8daecb2 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/state/map/CachedMap.java +++ b/storm-client/src/jvm/org/apache/storm/trident/state/map/CachedMap.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -19,7 +25,8 @@ import org.apache.storm.trident.util.LRUMap; /** - * Useful to layer over a map that communicates with a database. you generally layer opaque map over this over your database store. + * Useful to layer over a map that communicates with a database. you generally layer opaque map over + * this over your database store. */ public class CachedMap implements IBackingMap { LRUMap, T> cache; diff --git a/storm-client/src/jvm/org/apache/storm/trident/state/map/IBackingMap.java b/storm-client/src/jvm/org/apache/storm/trident/state/map/IBackingMap.java index 0d7bda3cebd..92b91876bb1 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/state/map/IBackingMap.java +++ b/storm-client/src/jvm/org/apache/storm/trident/state/map/IBackingMap.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -14,7 +20,6 @@ import java.util.List; - public interface IBackingMap { List multiGet(List> keys); diff --git a/storm-client/src/jvm/org/apache/storm/trident/state/map/MapCombinerAggStateUpdater.java b/storm-client/src/jvm/org/apache/storm/trident/state/map/MapCombinerAggStateUpdater.java index 330ce2f01a5..c0f5f731efd 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/state/map/MapCombinerAggStateUpdater.java +++ b/storm-client/src/jvm/org/apache/storm/trident/state/map/MapCombinerAggStateUpdater.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -28,7 +34,7 @@ import org.apache.storm.tuple.Values; public class MapCombinerAggStateUpdater implements StateUpdater { - //ANY CHANGE TO THIS CODE MUST BE SERIALIZABLE COMPATIBLE OR THERE WILL BE PROBLEMS + // ANY CHANGE TO THIS CODE MUST BE SERIALIZABLE COMPATIBLE OR THERE WILL BE PROBLEMS private static final long serialVersionUID = -3960578785572592092L; CombinerAggregator agg; @@ -38,13 +44,15 @@ public class MapCombinerAggStateUpdater implements StateUpdater { transient ProjectionFactory inputFactory; ComboList.Factory factory; - public MapCombinerAggStateUpdater(CombinerAggregator agg, Fields groupFields, Fields inputFields) { + public MapCombinerAggStateUpdater(CombinerAggregator agg, Fields groupFields, + Fields inputFields) { this.agg = agg; this.groupFields = groupFields; this.inputFields = inputFields; if (inputFields.size() != 1) { throw new IllegalArgumentException( - "Combiner aggs only take a single field as input. Got this instead: " + inputFields.toString()); + "Combiner aggs only take a single field as input. Got this instead: " + inputFields + .toString()); } factory = new ComboList.Factory(groupFields.size(), inputFields.size()); } diff --git a/storm-client/src/jvm/org/apache/storm/trident/state/map/MapReducerAggStateUpdater.java b/storm-client/src/jvm/org/apache/storm/trident/state/map/MapReducerAggStateUpdater.java index 1b06c5c74e0..1470648a842 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/state/map/MapReducerAggStateUpdater.java +++ b/storm-client/src/jvm/org/apache/storm/trident/state/map/MapReducerAggStateUpdater.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -29,7 +35,7 @@ import org.apache.storm.tuple.Values; public class MapReducerAggStateUpdater implements StateUpdater { - //ANY CHANGE TO THIS CODE MUST BE SERIALIZABLE COMPATIBLE OR THERE WILL BE PROBLEMS + // ANY CHANGE TO THIS CODE MUST BE SERIALIZABLE COMPATIBLE OR THERE WILL BE PROBLEMS private static final long serialVersionUID = 8667174018978959987L; ReducerAggregator agg; @@ -39,7 +45,8 @@ public class MapReducerAggStateUpdater implements StateUpdater { transient ProjectionFactory inputFactory; ComboList.Factory factory; - public MapReducerAggStateUpdater(ReducerAggregator agg, Fields groupFields, Fields inputFields) { + public MapReducerAggStateUpdater(ReducerAggregator agg, Fields groupFields, + Fields inputFields) { this.agg = agg; this.groupFields = groupFields; this.inputFields = inputFields; diff --git a/storm-client/src/jvm/org/apache/storm/trident/state/map/MapState.java b/storm-client/src/jvm/org/apache/storm/trident/state/map/MapState.java index 7aa0b17e4bc..b2a56ae1979 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/state/map/MapState.java +++ b/storm-client/src/jvm/org/apache/storm/trident/state/map/MapState.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/state/map/MicroBatchIBackingMap.java b/storm-client/src/jvm/org/apache/storm/trident/state/map/MicroBatchIBackingMap.java index 73270a884a5..476ac8bd08c 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/state/map/MicroBatchIBackingMap.java +++ b/storm-client/src/jvm/org/apache/storm/trident/state/map/MicroBatchIBackingMap.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -21,7 +27,6 @@ public class MicroBatchIBackingMap implements IBackingMap { IBackingMap delegate; Options options; - public MicroBatchIBackingMap(final Options options, final IBackingMap delegate) { this.options = options; this.delegate = delegate; diff --git a/storm-client/src/jvm/org/apache/storm/trident/state/map/NonTransactionalMap.java b/storm-client/src/jvm/org/apache/storm/trident/state/map/NonTransactionalMap.java index a574054000e..1e6d2302222 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/state/map/NonTransactionalMap.java +++ b/storm-client/src/jvm/org/apache/storm/trident/state/map/NonTransactionalMap.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -16,7 +22,6 @@ import java.util.List; import org.apache.storm.trident.state.ValueUpdater; - public class NonTransactionalMap implements MapState { IBackingMap backing; diff --git a/storm-client/src/jvm/org/apache/storm/trident/state/map/OpaqueMap.java b/storm-client/src/jvm/org/apache/storm/trident/state/map/OpaqueMap.java index 5775365a1c9..70e8a330a85 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/state/map/OpaqueMap.java +++ b/storm-client/src/jvm/org/apache/storm/trident/state/map/OpaqueMap.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -17,7 +23,6 @@ import org.apache.storm.trident.state.OpaqueValue; import org.apache.storm.trident.state.ValueUpdater; - public class OpaqueMap implements MapState { CachedBatchReadsMap backing; Long currTx; diff --git a/storm-client/src/jvm/org/apache/storm/trident/state/map/ReadOnlyMapState.java b/storm-client/src/jvm/org/apache/storm/trident/state/map/ReadOnlyMapState.java index 1d48c0b29fa..f7e229b7de2 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/state/map/ReadOnlyMapState.java +++ b/storm-client/src/jvm/org/apache/storm/trident/state/map/ReadOnlyMapState.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/state/map/RemovableMapState.java b/storm-client/src/jvm/org/apache/storm/trident/state/map/RemovableMapState.java index 1658f968faa..e7851a47558 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/state/map/RemovableMapState.java +++ b/storm-client/src/jvm/org/apache/storm/trident/state/map/RemovableMapState.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/state/map/SnapshottableMap.java b/storm-client/src/jvm/org/apache/storm/trident/state/map/SnapshottableMap.java index 7b144ca6c10..32f22f81eec 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/state/map/SnapshottableMap.java +++ b/storm-client/src/jvm/org/apache/storm/trident/state/map/SnapshottableMap.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -17,7 +23,6 @@ import org.apache.storm.trident.state.ValueUpdater; import org.apache.storm.trident.state.snapshot.Snapshottable; - public class SnapshottableMap implements MapState, Snapshottable { MapState delegate; List> keys; diff --git a/storm-client/src/jvm/org/apache/storm/trident/state/map/TransactionalMap.java b/storm-client/src/jvm/org/apache/storm/trident/state/map/TransactionalMap.java index ef1a08cd612..a0ad4356ced 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/state/map/TransactionalMap.java +++ b/storm-client/src/jvm/org/apache/storm/trident/state/map/TransactionalMap.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -17,7 +23,6 @@ import org.apache.storm.trident.state.TransactionalValue; import org.apache.storm.trident.state.ValueUpdater; - public class TransactionalMap implements MapState { CachedBatchReadsMap backing; Long currTx; diff --git a/storm-client/src/jvm/org/apache/storm/trident/state/snapshot/ReadOnlySnapshottable.java b/storm-client/src/jvm/org/apache/storm/trident/state/snapshot/ReadOnlySnapshottable.java index d177d0073dc..a0ac5427152 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/state/snapshot/ReadOnlySnapshottable.java +++ b/storm-client/src/jvm/org/apache/storm/trident/state/snapshot/ReadOnlySnapshottable.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/state/snapshot/Snapshottable.java b/storm-client/src/jvm/org/apache/storm/trident/state/snapshot/Snapshottable.java index c8001f4ae42..10c98907c75 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/state/snapshot/Snapshottable.java +++ b/storm-client/src/jvm/org/apache/storm/trident/state/snapshot/Snapshottable.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -14,7 +20,6 @@ import org.apache.storm.trident.state.ValueUpdater; - // used by Stream#persistentAggregate public interface Snapshottable extends ReadOnlySnapshottable { T update(ValueUpdater updater); diff --git a/storm-client/src/jvm/org/apache/storm/trident/testing/CountAsAggregator.java b/storm-client/src/jvm/org/apache/storm/trident/testing/CountAsAggregator.java index 06f11eaf413..18f3650308d 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/testing/CountAsAggregator.java +++ b/storm-client/src/jvm/org/apache/storm/trident/testing/CountAsAggregator.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -17,7 +23,6 @@ import org.apache.storm.trident.tuple.TridentTuple; import org.apache.storm.tuple.Values; - public class CountAsAggregator extends BaseAggregator { @Override diff --git a/storm-client/src/jvm/org/apache/storm/trident/testing/FeederBatchSpout.java b/storm-client/src/jvm/org/apache/storm/trident/testing/FeederBatchSpout.java index 2ae6a4327f9..162a851e156 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/testing/FeederBatchSpout.java +++ b/storm-client/src/jvm/org/apache/storm/trident/testing/FeederBatchSpout.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -34,7 +40,6 @@ public class FeederBatchSpout implements ITridentSpout fields) { outFields = new Fields(fields); id = RegisteredGlobalState.registerState(new CopyOnWriteArrayList()); @@ -68,7 +73,8 @@ public Fields getOutputFields() { } @Override - public BatchCoordinator>>> getCoordinator(String txStateId, Map conf, + public BatchCoordinator>>> getCoordinator(String txStateId, + Map conf, TopologyContext context) { int numTasks = context.getComponentTasks( TridentTopologyBuilder.spoutIdFromCoordinatorId( @@ -78,7 +84,8 @@ public BatchCoordinator>>> getCoordinator(String } @Override - public Emitter>>> getEmitter(String txStateId, Map conf, TopologyContext context) { + public Emitter>>> getEmitter(String txStateId, Map conf, TopologyContext context) { return new FeederEmitter(context.getThisTaskIndex()); } @@ -91,7 +98,8 @@ private static class FeederEmitter implements ITridentSpout.Emitter>> coordinatorMeta, TridentCollector collector) { + public void emitBatch(TransactionAttempt tx, Map>> coordinatorMeta, TridentCollector collector) { List> tuples = coordinatorMeta.get(index); if (tuples != null) { for (List t : tuples) { @@ -121,7 +129,8 @@ public FeederCoordinator(int numPartitions) { } @Override - public Map>> initializeTransaction(long txid, Map>> prevMetadata, + public Map>> initializeTransaction(long txid, Map>> prevMetadata, Map>> currMetadata) { if (currMetadata != null) { return currMetadata; @@ -158,7 +167,8 @@ public void close() { public void success(long txid) { Integer index = txIndices.get(txid); if (index != null) { - Semaphore sem = (Semaphore) ((List) RegisteredGlobalState.getState(semaphoreId)).get(index); + Semaphore sem = (Semaphore) ((List) RegisteredGlobalState.getState(semaphoreId)) + .get(index); sem.release(); } } diff --git a/storm-client/src/jvm/org/apache/storm/trident/testing/FeederCommitterBatchSpout.java b/storm-client/src/jvm/org/apache/storm/trident/testing/FeederCommitterBatchSpout.java index 6a71ad57a2d..9d0051b102a 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/testing/FeederCommitterBatchSpout.java +++ b/storm-client/src/jvm/org/apache/storm/trident/testing/FeederCommitterBatchSpout.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -21,7 +27,6 @@ import org.apache.storm.trident.topology.TransactionAttempt; import org.apache.storm.tuple.Fields; - public class FeederCommitterBatchSpout implements ICommitterTridentSpout>>>, IFeeder { FeederBatchSpout spout; @@ -40,7 +45,8 @@ public Emitter getEmitter(String txStateId, Map conf, TopologyCo } @Override - public BatchCoordinator>>> getCoordinator(String txStateId, Map conf, + public BatchCoordinator>>> getCoordinator(String txStateId, + Map conf, TopologyContext context) { return spout.getCoordinator(txStateId, conf, context); } @@ -63,7 +69,6 @@ public void feed(Object tuples) { static class CommitterEmitter implements ICommitterTridentSpout.Emitter { ITridentSpout.Emitter emitter; - CommitterEmitter(ITridentSpout.Emitter e) { emitter = e; } @@ -73,7 +78,8 @@ public void commit(TransactionAttempt attempt) { } @Override - public void emitBatch(TransactionAttempt tx, Object coordinatorMeta, TridentCollector collector) { + public void emitBatch(TransactionAttempt tx, Object coordinatorMeta, + TridentCollector collector) { emitter.emitBatch(tx, coordinatorMeta, collector); } diff --git a/storm-client/src/jvm/org/apache/storm/trident/testing/FixedBatchSpout.java b/storm-client/src/jvm/org/apache/storm/trident/testing/FixedBatchSpout.java index 1961b335ef5..b9ee69edfea 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/testing/FixedBatchSpout.java +++ b/storm-client/src/jvm/org/apache/storm/trident/testing/FixedBatchSpout.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -22,7 +28,6 @@ import org.apache.storm.trident.spout.IBatchSpout; import org.apache.storm.tuple.Fields; - public class FixedBatchSpout implements IBatchSpout { Fields fields; diff --git a/storm-client/src/jvm/org/apache/storm/trident/testing/IFeeder.java b/storm-client/src/jvm/org/apache/storm/trident/testing/IFeeder.java index ef717727a5a..0f49d1668db 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/testing/IFeeder.java +++ b/storm-client/src/jvm/org/apache/storm/trident/testing/IFeeder.java @@ -1,18 +1,23 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.trident.testing; - public interface IFeeder { void feed(Object tuples); } diff --git a/storm-client/src/jvm/org/apache/storm/trident/testing/LRUMemoryMapState.java b/storm-client/src/jvm/org/apache/storm/trident/testing/LRUMemoryMapState.java index 02edbb638d0..90bb289f05a 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/testing/LRUMemoryMapState.java +++ b/storm-client/src/jvm/org/apache/storm/trident/testing/LRUMemoryMapState.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -35,13 +41,15 @@ @SuppressWarnings("checkstyle:AbbreviationAsWordInName") public class LRUMemoryMapState implements Snapshottable, ITupleCollection, MapState { - static ConcurrentHashMap, Object>> dbs = new ConcurrentHashMap, Object>>(); + static ConcurrentHashMap, Object>> dbs = + new ConcurrentHashMap, Object>>(); LRUMemoryMapStateBacking backing; SnapshottableMap delegate; public LRUMemoryMapState(int cacheSize, String id) { backing = new LRUMemoryMapStateBacking(cacheSize, id); - delegate = new SnapshottableMap(OpaqueMap.build(backing), new Values("$MEMORY-MAP-STATE-GLOBAL$")); + delegate = new SnapshottableMap(OpaqueMap.build(backing), + new Values("$MEMORY-MAP-STATE-GLOBAL$")); } @Override @@ -100,7 +108,8 @@ public Factory(int maxSize) { } @Override - public State makeState(Map conf, IMetricsContext metrics, int partitionIndex, int numPartitions) { + public State makeState(Map conf, IMetricsContext metrics, + int partitionIndex, int numPartitions) { return new LRUMemoryMapState(maxSize, id + partitionIndex); } } diff --git a/storm-client/src/jvm/org/apache/storm/trident/testing/MemoryBackingMap.java b/storm-client/src/jvm/org/apache/storm/trident/testing/MemoryBackingMap.java index f9ebce0d449..a470341548d 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/testing/MemoryBackingMap.java +++ b/storm-client/src/jvm/org/apache/storm/trident/testing/MemoryBackingMap.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/testing/MemoryMapState.java b/storm-client/src/jvm/org/apache/storm/trident/testing/MemoryMapState.java index 64dfc60a38d..4b547049530 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/testing/MemoryMapState.java +++ b/storm-client/src/jvm/org/apache/storm/trident/testing/MemoryMapState.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -35,7 +41,8 @@ public class MemoryMapState implements Snapshottable, ITupleCollection, MapState, RemovableMapState { - static ConcurrentHashMap, Object>> dbs = new ConcurrentHashMap, Object>>(); + static ConcurrentHashMap, Object>> dbs = + new ConcurrentHashMap, Object>>(); MemoryMapStateBacking backing; SnapshottableMap delegate; List> removed = new ArrayList(); @@ -43,7 +50,8 @@ public class MemoryMapState implements Snapshottable, ITupleCollection, Ma public MemoryMapState(String id) { backing = new MemoryMapStateBacking(id); - delegate = new SnapshottableMap(OpaqueMap.build(backing), new Values("$MEMORY-MAP-STATE-GLOBAL$")); + delegate = new SnapshottableMap(OpaqueMap.build(backing), + new Values("$MEMORY-MAP-STATE-GLOBAL$")); } @Override @@ -102,7 +110,8 @@ public void multiRemove(List> keys) { for (int i = 0; i < keys.size(); i++) { nulls.add(null); } - // first just set the keys to null, then flag to remove them at beginning of next commit when we know the current and last value + // first just set the keys to null, then flag to remove them at beginning of next commit + // when we know the current and last value // are both null multiPut(keys, nulls); removed.addAll(keys); @@ -117,7 +126,8 @@ public Factory() { } @Override - public State makeState(Map conf, IMetricsContext metrics, int partitionIndex, int numPartitions) { + public State makeState(Map conf, IMetricsContext metrics, + int partitionIndex, int numPartitions) { return new MemoryMapState(id + partitionIndex); } } diff --git a/storm-client/src/jvm/org/apache/storm/trident/testing/Split.java b/storm-client/src/jvm/org/apache/storm/trident/testing/Split.java index a768281b8db..4f498093128 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/testing/Split.java +++ b/storm-client/src/jvm/org/apache/storm/trident/testing/Split.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/testing/StringLength.java b/storm-client/src/jvm/org/apache/storm/trident/testing/StringLength.java index dcf7b7b1007..49bf1974466 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/testing/StringLength.java +++ b/storm-client/src/jvm/org/apache/storm/trident/testing/StringLength.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/testing/TrueFilter.java b/storm-client/src/jvm/org/apache/storm/trident/testing/TrueFilter.java index 52165f5ba52..ce167e4ebec 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/testing/TrueFilter.java +++ b/storm-client/src/jvm/org/apache/storm/trident/testing/TrueFilter.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/testing/TuplifyArgs.java b/storm-client/src/jvm/org/apache/storm/trident/testing/TuplifyArgs.java index 056d2a1efd4..b835002239f 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/testing/TuplifyArgs.java +++ b/storm-client/src/jvm/org/apache/storm/trident/testing/TuplifyArgs.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/topology/BatchInfo.java b/storm-client/src/jvm/org/apache/storm/trident/topology/BatchInfo.java index 107f0590fcd..d72568a1da0 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/topology/BatchInfo.java +++ b/storm-client/src/jvm/org/apache/storm/trident/topology/BatchInfo.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -14,7 +20,6 @@ import org.apache.storm.trident.spout.IBatchID; - public class BatchInfo { public IBatchID batchId; public Object state; diff --git a/storm-client/src/jvm/org/apache/storm/trident/topology/ITridentBatchBolt.java b/storm-client/src/jvm/org/apache/storm/trident/topology/ITridentBatchBolt.java index 0d2c71d3b4b..fdd98fb5a5a 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/topology/ITridentBatchBolt.java +++ b/storm-client/src/jvm/org/apache/storm/trident/topology/ITridentBatchBolt.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/topology/MasterBatchCoordinator.java b/storm-client/src/jvm/org/apache/storm/trident/topology/MasterBatchCoordinator.java index aed1045a97c..7aeaed14636 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/topology/MasterBatchCoordinator.java +++ b/storm-client/src/jvm/org/apache/storm/trident/topology/MasterBatchCoordinator.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -15,8 +21,8 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Map.Entry; +import java.util.Map; import java.util.TreeMap; import org.apache.storm.Config; import org.apache.storm.spout.SpoutOutputCollector; @@ -36,7 +42,6 @@ public class MasterBatchCoordinator extends BaseRichSpout { public static final long INIT_TXID = 1L; - public static final String BATCH_STREAM_ID = "$batch"; public static final String COMMIT_STREAM_ID = "$commit"; public static final String SUCCESS_STREAM_ID = "$success"; @@ -79,8 +84,10 @@ public void deactivate() { } @Override - public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { - throttler = new WindowedTimeThrottler((Number) conf.get(Config.TOPOLOGY_TRIDENT_BATCH_EMIT_INTERVAL_MILLIS), 1); + public void open(Map conf, TopologyContext context, + SpoutOutputCollector collector) { + throttler = new WindowedTimeThrottler((Number) conf + .get(Config.TOPOLOGY_TRIDENT_BATCH_EMIT_INTERVAL_MILLIS), 1); for (String spoutId : managedSpoutIds) { states.add(TransactionalState.newCoordinatorState(conf, spoutId)); } @@ -133,7 +140,8 @@ public void ack(Object msgId) { for (TransactionalState state : states) { state.setData(CURRENT_TX, currTransaction); } - LOG.debug("Emitted on [stream = {}], [tx_attempt = {}], [tx_status = {}], [{}]", SUCCESS_STREAM_ID, tx, status, this); + LOG.debug("Emitted on [stream = {}], [tx_attempt = {}], [tx_status = {}], [{}]", + SUCCESS_STREAM_ID, tx, status, this); } sync(); } @@ -152,7 +160,8 @@ public void fail(Object msgId) { @Override public void declareOutputFields(OutputFieldsDeclarer declarer) { - // in partitioned example, in case an emitter task receives a later transaction than it's emitted so far, + // in partitioned example, in case an emitter task receives a later transaction than it's + // emitted so far, // when it sees the earlier txid it should know to emit nothing declarer.declareStream(BATCH_STREAM_ID, new Fields("tx")); declarer.declareStream(COMMIT_STREAM_ID, new Fields("tx")); @@ -162,13 +171,15 @@ public void declareOutputFields(OutputFieldsDeclarer declarer) { private void sync() { // note that sometimes the tuples active may be less than max_spout_pending, e.g. // max_spout_pending = 3 - // tx 1, 2, 3 active, tx 2 is acked. there won't be a commit for tx 2 (because tx 1 isn't committed yet), + // tx 1, 2, 3 active, tx 2 is acked. there won't be a commit for tx 2 (because tx 1 isn't + // committed yet), // and there won't be a batch for tx 4 because there's max_spout_pending tx active TransactionStatus maybeCommit = activeTx.get(currTransaction); if (maybeCommit != null && maybeCommit.status == AttemptStatus.PROCESSED) { maybeCommit.status = AttemptStatus.COMMITTING; collector.emit(COMMIT_STREAM_ID, new Values(maybeCommit.attempt), maybeCommit.attempt); - LOG.debug("Emitted on [stream = {}], [tx_status = {}], [{}]", COMMIT_STREAM_ID, maybeCommit, this); + LOG.debug("Emitted on [stream = {}], [tx_status = {}], [{}]", COMMIT_STREAM_ID, + maybeCommit, this); } if (active) { @@ -191,10 +202,12 @@ private void sync() { } TransactionAttempt attempt = new TransactionAttempt(curr, attemptId); - final TransactionStatus newTransactionStatus = new TransactionStatus(attempt); + final TransactionStatus newTransactionStatus = + new TransactionStatus(attempt); activeTx.put(curr, newTransactionStatus); collector.emit(BATCH_STREAM_ID, new Values(attempt), attempt); - LOG.debug("Emitted on [stream = {}], [tx_attempt = {}], [tx_status = {}], [{}]", BATCH_STREAM_ID, attempt, + LOG.debug("Emitted on [stream = {}], [tx_attempt = {}], [tx_status = {}], " + + "[{}]", BATCH_STREAM_ID, attempt, newTransactionStatus, this); throttler.markEvent(); } @@ -208,7 +221,7 @@ private boolean isReady(long txid) { if (throttler.isThrottled()) { return false; } - //TODO: make this strategy configurable?... right now it goes if anyone is ready + // TODO: make this strategy configurable?... right now it goes if anyone is ready for (ITridentSpout.BatchCoordinator coord : coordinators) { if (coord.isReady(txid)) { return true; diff --git a/storm-client/src/jvm/org/apache/storm/trident/topology/TransactionAttempt.java b/storm-client/src/jvm/org/apache/storm/trident/topology/TransactionAttempt.java index 710599e8a1d..544d252fb92 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/topology/TransactionAttempt.java +++ b/storm-client/src/jvm/org/apache/storm/trident/topology/TransactionAttempt.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -14,12 +20,10 @@ import org.apache.storm.trident.spout.IBatchID; - public class TransactionAttempt implements IBatchID { Long txid; int attemptId; - // for kryo compatibility public TransactionAttempt() { diff --git a/storm-client/src/jvm/org/apache/storm/trident/topology/TridentBoltExecutor.java b/storm-client/src/jvm/org/apache/storm/trident/topology/TridentBoltExecutor.java index bacb42d2451..879e146e0a4 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/topology/TridentBoltExecutor.java +++ b/storm-client/src/jvm/org/apache/storm/trident/topology/TridentBoltExecutor.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -72,7 +78,8 @@ public static String coordStream(String batch) { } @Override - public void prepare(Map conf, TopologyContext context, OutputCollector collector) { + public void prepare(Map conf, TopologyContext context, + OutputCollector collector) { messageTimeoutMs = context.maxTopologyMessageTimeout() * 1000L; lastRotate = System.currentTimeMillis(); batches = new RotatingMap<>(2); @@ -132,7 +139,8 @@ private boolean finishBatch(TrackedBatch tracked, Tuple finishTuple) { String stream = coordStream(tracked.info.batchGroup); for (Integer task : tracked.condition.targetTasks) { collector - .emitDirect(task, stream, finishTuple, new Values(tracked.info.batchId, Utils.get(tracked.taskEmittedTuples, task, 0))); + .emitDirect(task, stream, finishTuple, new Values(tracked.info.batchId, Utils + .get(tracked.taskEmittedTuples, task, 0))); } if (tracked.delayedAck != null) { collector.ack(tracked.delayedAck); @@ -190,15 +198,16 @@ public void execute(Tuple tuple) { } String batchGroup = batchGroupIds.get(tuple.getSourceGlobalStreamId()); if (batchGroup == null) { - // this is so we can do things like have simple DRPC that doesn't need to use batch processing + // this is so we can do things like have simple DRPC that doesn't need to use batch + // processing coordCollector.setCurrBatch(null); bolt.execute(null, tuple); collector.ack(tuple); return; } IBatchID id = (IBatchID) tuple.getValue(0); - //get transaction id - //if it already exists and attempt id is greater than the attempt there + // get transaction id + // if it already exists and attempt id is greater than the attempt there TrackedBatch tracked = (TrackedBatch) batches.get(id.getId()); @@ -216,13 +225,14 @@ public void execute(Tuple tuple) { if (tracked == null) { tracked = - new TrackedBatch(new BatchInfo(batchGroup, id, bolt.initBatchState(batchGroup, id)), coordConditions.get(batchGroup), + new TrackedBatch(new BatchInfo(batchGroup, id, bolt.initBatchState(batchGroup, id)), + coordConditions.get(batchGroup), id.getAttemptId()); batches.put(id.getId(), tracked); } coordCollector.setCurrBatch(tracked); - //System.out.println("TRACKED: " + tracked + " " + tuple); + // System.out.println("TRACKED: " + tracked + " " + tuple); TupleType t = getTupleType(tuple, tracked); if (t == TupleType.COMMIT) { @@ -273,7 +283,8 @@ public Map getComponentConfiguration() { ret = new HashMap<>(); } ret.put(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, 5); - // TODO: Need to be able to set the tick tuple time to the message timeout, ideally without parameterization + // TODO: Need to be able to set the tick tuple time to the message timeout, ideally without + // parameterization return ret; } @@ -401,7 +412,8 @@ public List emit(String stream, Collection anchors, List } @Override - public void emitDirect(int task, String stream, Collection anchors, List tuple) { + public void emitDirect(int task, String stream, Collection anchors, + List tuple) { updateTaskCounts(Arrays.asList(task)); delegate.emitDirect(task, stream, anchors, tuple); } @@ -431,7 +443,6 @@ public void reportError(Throwable error) { delegate.reportError(error); } - private void updateTaskCounts(List tasks) { if (currBatch != null) { Map taskEmittedTuples = currBatch.taskEmittedTuples; diff --git a/storm-client/src/jvm/org/apache/storm/trident/topology/TridentTopologyBuilder.java b/storm-client/src/jvm/org/apache/storm/trident/topology/TridentTopologyBuilder.java index 6373effb895..d43c090bc4a 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/topology/TridentTopologyBuilder.java +++ b/storm-client/src/jvm/org/apache/storm/trident/topology/TridentTopologyBuilder.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -60,7 +66,8 @@ public static String spoutIdFromCoordinatorId(String coordId) { return coordId.substring(SPOUT_COORD_PREFIX.length()); } - public SpoutDeclarer setBatchPerTupleSpout(String id, String streamName, IRichSpout spout, Integer parallelism, String batchGroup) { + public SpoutDeclarer setBatchPerTupleSpout(String id, String streamName, IRichSpout spout, + Integer parallelism, String batchGroup) { Map batchGroups = new HashMap<>(); batchGroups.put(streamName, batchGroup); markBatchGroups(id, batchGroups); @@ -69,24 +76,29 @@ public SpoutDeclarer setBatchPerTupleSpout(String id, String streamName, IRichSp return new SpoutDeclarerImpl(c); } - public SpoutDeclarer setSpout(String id, String streamName, String txStateId, IBatchSpout spout, Integer parallelism, + public SpoutDeclarer setSpout(String id, String streamName, String txStateId, IBatchSpout spout, + Integer parallelism, String batchGroup) { - return setSpout(id, streamName, txStateId, new BatchSpoutExecutor(spout), parallelism, batchGroup); + return setSpout(id, streamName, txStateId, new BatchSpoutExecutor(spout), parallelism, + batchGroup); } - public SpoutDeclarer setSpout(String id, String streamName, String txStateId, ITridentSpout spout, Integer parallelism, + public SpoutDeclarer setSpout(String id, String streamName, String txStateId, + ITridentSpout spout, Integer parallelism, String batchGroup) { Map batchGroups = new HashMap<>(); batchGroups.put(streamName, batchGroup); markBatchGroups(id, batchGroups); - TransactionalSpoutComponent c = new TransactionalSpoutComponent(spout, streamName, parallelism, txStateId, batchGroup); + TransactionalSpoutComponent c = new TransactionalSpoutComponent(spout, streamName, + parallelism, txStateId, batchGroup); spouts.put(id, c); return new SpoutDeclarerImpl(c); } // map from stream name to batch id - public BoltDeclarer setBolt(String id, ITridentBatchBolt bolt, Integer parallelism, Set committerBatches, + public BoltDeclarer setBolt(String id, ITridentBatchBolt bolt, Integer parallelism, + Set committerBatches, Map batchGroups) { markBatchGroups(id, batchGroups); Component c = new Component(bolt, parallelism, committerBatches); @@ -103,22 +115,26 @@ Map fleshOutStreamBatchIds(boolean includeCommitStream) Map ret = new HashMap<>(batchIds); Set allBatches = new HashSet<>(batchIds.values()); for (String b : allBatches) { - ret.put(new GlobalStreamId(masterCoordinator(b), MasterBatchCoordinator.BATCH_STREAM_ID), b); + ret.put(new GlobalStreamId(masterCoordinator(b), + MasterBatchCoordinator.BATCH_STREAM_ID), b); if (includeCommitStream) { - ret.put(new GlobalStreamId(masterCoordinator(b), MasterBatchCoordinator.COMMIT_STREAM_ID), b); + ret.put(new GlobalStreamId(masterCoordinator(b), + MasterBatchCoordinator.COMMIT_STREAM_ID), b); } - // DO NOT include the success stream as part of the batch. it should not trigger coordination tuples, + // DO NOT include the success stream as part of the batch. it should not trigger + // coordination tuples, // and is just a metadata tuple to assist in cleanup, should not trigger batch tracking } for (String id : spouts.keySet()) { TransactionalSpoutComponent c = spouts.get(id); if (c.batchGroupId != null) { - ret.put(new GlobalStreamId(spoutCoordinator(id), MasterBatchCoordinator.BATCH_STREAM_ID), c.batchGroupId); + ret.put(new GlobalStreamId(spoutCoordinator(id), + MasterBatchCoordinator.BATCH_STREAM_ID), c.batchGroupId); } } - //this takes care of setting up coord streams for spouts and bolts + // this takes care of setting up coord streams for spouts and bolts for (GlobalStreamId s : batchIds.keySet()) { String b = batchIds.get(s); ret.put(new GlobalStreamId(s.get_componentId(), TridentBoltExecutor.coordStream(b)), b); @@ -139,7 +155,7 @@ public StormTopology buildTopology(Map masterCoordResources) { TransactionalSpoutComponent c = spouts.get(id); if (c.spout instanceof IRichSpout) { - //TODO: wrap this to set the stream name + // TODO: wrap this to set the stream name builder.setSpout(id, (IRichSpout) c.spout, c.parallelism); } else { String batchGroup = c.batchGroupId; @@ -155,9 +171,12 @@ public StormTopology buildTopology(Map masterCoordResources) { BoltDeclarer scd = - builder.setBolt(spoutCoordinator(id), new TridentSpoutCoordinator(c.commitStateId, (ITridentSpout) c.spout)) - .globalGrouping(masterCoordinator(c.batchGroupId), MasterBatchCoordinator.BATCH_STREAM_ID) - .globalGrouping(masterCoordinator(c.batchGroupId), MasterBatchCoordinator.SUCCESS_STREAM_ID); + builder.setBolt(spoutCoordinator(id), + new TridentSpoutCoordinator(c.commitStateId, (ITridentSpout) c.spout)) + .globalGrouping(masterCoordinator(c.batchGroupId), + MasterBatchCoordinator.BATCH_STREAM_ID) + .globalGrouping(masterCoordinator(c.batchGroupId), + MasterBatchCoordinator.SUCCESS_STREAM_ID); for (SharedMemory request : c.sharedMemory) { scd.addSharedMemory(request); } @@ -175,9 +194,11 @@ public StormTopology buildTopology(Map masterCoordResources) { specs), c.parallelism); bd.allGrouping(spoutCoordinator(id), MasterBatchCoordinator.BATCH_STREAM_ID); - bd.allGrouping(masterCoordinator(batchGroup), MasterBatchCoordinator.SUCCESS_STREAM_ID); + bd.allGrouping(masterCoordinator(batchGroup), + MasterBatchCoordinator.SUCCESS_STREAM_ID); if (c.spout instanceof ICommitterTridentSpout) { - bd.allGrouping(masterCoordinator(batchGroup), MasterBatchCoordinator.COMMIT_STREAM_ID); + bd.allGrouping(masterCoordinator(batchGroup), + MasterBatchCoordinator.COMMIT_STREAM_ID); } bd.addConfigurations(c.componentConf); } @@ -186,19 +207,23 @@ public StormTopology buildTopology(Map masterCoordResources) { for (String id : batchPerTupleSpouts.keySet()) { SpoutComponent c = batchPerTupleSpouts.get(id); SpoutDeclarer d = - builder.setSpout(id, new RichSpoutBatchTriggerer((IRichSpout) c.spout, c.streamName, c.batchGroupId), c.parallelism); + builder.setSpout(id, new RichSpoutBatchTriggerer((IRichSpout) c.spout, c.streamName, + c.batchGroupId), c.parallelism); d.addConfigurations(c.componentConf); } - Number onHeap = masterCoordResources.get(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB); - Number offHeap = masterCoordResources.get(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB); + Number onHeap = masterCoordResources + .get(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB); + Number offHeap = masterCoordResources + .get(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB); Number cpuLoad = masterCoordResources.get(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT); for (String batch : batchesToCommitIds.keySet()) { List commitIds = batchesToCommitIds.get(batch); SpoutDeclarer masterCoord = - builder.setSpout(masterCoordinator(batch), new MasterBatchCoordinator(commitIds, batchesToSpouts.get(batch))); + builder.setSpout(masterCoordinator(batch), new MasterBatchCoordinator(commitIds, + batchesToSpouts.get(batch))); if (onHeap != null) { if (offHeap != null) { @@ -234,10 +259,12 @@ public StormTopology buildTopology(Map masterCoordResources) { } for (String b : c.committerBatches) { - specs.get(b).commitStream = new GlobalStreamId(masterCoordinator(b), MasterBatchCoordinator.COMMIT_STREAM_ID); + specs.get(b).commitStream = new GlobalStreamId(masterCoordinator(b), + MasterBatchCoordinator.COMMIT_STREAM_ID); } - BoltDeclarer d = builder.setBolt(id, new TridentBoltExecutor(c.bolt, batchIdsForBolts, specs), c.parallelism); + BoltDeclarer d = builder.setBolt(id, new TridentBoltExecutor(c.bolt, batchIdsForBolts, + specs), c.parallelism); for (SharedMemory request : c.sharedMemory) { d.addSharedMemory(request); } @@ -321,7 +348,8 @@ public String toString() { private static class TransactionalSpoutComponent extends SpoutComponent { public String commitStateId; - TransactionalSpoutComponent(Object spout, String streamName, Integer parallelism, String commitStateId, + TransactionalSpoutComponent(Object spout, String streamName, Integer parallelism, + String commitStateId, String batchGroupId) { super(spout, streamName, parallelism, batchGroupId); this.commitStateId = commitStateId; @@ -369,7 +397,7 @@ public SpoutDeclarer addConfigurations(Map conf) { } /** - * return the current component configuration. + * Return the current component configuration. * * @return the current configuration. */ @@ -414,7 +442,8 @@ public String getStream() { } @Override - public BoltDeclarer fieldsGrouping(final String component, final String streamId, final Fields fields) { + public BoltDeclarer fieldsGrouping(final String component, final String streamId, + final Fields fields) { addDeclaration(new InputDeclaration() { @Override public void declare(InputDeclarer declarer) { @@ -697,7 +726,8 @@ public BoltDeclarer partialKeyGrouping(String componentId, String streamId, Fiel } @Override - public BoltDeclarer customGrouping(final String component, final CustomStreamGrouping grouping) { + public BoltDeclarer customGrouping(final String component, + final CustomStreamGrouping grouping) { addDeclaration(new InputDeclaration() { @Override public void declare(InputDeclarer declarer) { @@ -718,7 +748,8 @@ public String getStream() { } @Override - public BoltDeclarer customGrouping(final String component, final String streamId, final CustomStreamGrouping grouping) { + public BoltDeclarer customGrouping(final String component, final String streamId, + final CustomStreamGrouping grouping) { addDeclaration(new InputDeclaration() { @Override public void declare(InputDeclarer declarer) { @@ -772,7 +803,7 @@ public BoltDeclarer addConfigurations(Map conf) { } /** - * return the current component configuration. + * Return the current component configuration. * * @return the current configuration. */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/topology/state/RotatingTransactionalState.java b/storm-client/src/jvm/org/apache/storm/trident/topology/state/RotatingTransactionalState.java index 424ce4a3f75..d2fd56bd3c8 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/topology/state/RotatingTransactionalState.java +++ b/storm-client/src/jvm/org/apache/storm/trident/topology/state/RotatingTransactionalState.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -141,7 +147,8 @@ public void cleanupBefore(long txid) { try { state.delete(txPath(tx)); } catch (RuntimeException e) { - // Ignore NoNodeExists exceptions because when sync() it may populate curr with stale data since + // Ignore NoNodeExists exceptions because when sync() it may populate curr with + // stale data since // zookeeper reads are eventually consistent. if (!Utils.exceptionCauseIsInstanceOf(KeeperException.NoNodeException.class, e)) { throw e; diff --git a/storm-client/src/jvm/org/apache/storm/trident/topology/state/TestTransactionalState.java b/storm-client/src/jvm/org/apache/storm/trident/topology/state/TestTransactionalState.java index ebcabaa5a9f..5ae55cc1749 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/topology/state/TestTransactionalState.java +++ b/storm-client/src/jvm/org/apache/storm/trident/topology/state/TestTransactionalState.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/topology/state/TransactionalState.java b/storm-client/src/jvm/org/apache/storm/trident/topology/state/TransactionalState.java index fc3bc8549b4..6bb381909e1 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/topology/state/TransactionalState.java +++ b/storm-client/src/jvm/org/apache/storm/trident/topology/state/TransactionalState.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -17,7 +23,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; - import org.apache.storm.Config; import org.apache.storm.cluster.DaemonType; import org.apache.storm.shade.net.minidev.json.JSONValue; @@ -31,12 +36,12 @@ import org.apache.storm.utils.CuratorUtils; import org.apache.storm.utils.Utils; import org.apache.storm.utils.ZookeeperAuthInfo; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * Class that contains the logic to extract the transactional state info from zookeeper. All transactional state is kept in zookeeper. This + * Class that contains the logic to extract the transactional state info from zookeeper. All + * transactional state is kept in zookeeper. This * class only contains references to Curator, which is used to get all info from zookeeper. */ public class TransactionalState { @@ -51,24 +56,28 @@ protected TransactionalState(Map conf, String id, String subroot String transactionalRoot = (String) conf.get(Config.TRANSACTIONAL_ZOOKEEPER_ROOT); String rootDir = transactionalRoot + "/" + id + "/" + subroot; List servers = - (List) getWithBackup(conf, Config.TRANSACTIONAL_ZOOKEEPER_SERVERS, Config.STORM_ZOOKEEPER_SERVERS); - Object port = getWithBackup(conf, Config.TRANSACTIONAL_ZOOKEEPER_PORT, Config.STORM_ZOOKEEPER_PORT); + (List) getWithBackup(conf, Config.TRANSACTIONAL_ZOOKEEPER_SERVERS, + Config.STORM_ZOOKEEPER_SERVERS); + Object port = getWithBackup(conf, Config.TRANSACTIONAL_ZOOKEEPER_PORT, + Config.STORM_ZOOKEEPER_PORT); ZookeeperAuthInfo auth = new ZookeeperAuthInfo(conf); - CuratorFramework initter = CuratorUtils.newCuratorStarted(conf, servers, port, auth, DaemonType.WORKER.getDefaultZkAcls(conf)); + CuratorFramework initter = CuratorUtils.newCuratorStarted(conf, servers, port, auth, + DaemonType.WORKER.getDefaultZkAcls(conf)); zkAcls = Utils.getWorkerACL(conf); try { TransactionalState.createNode(initter, transactionalRoot, null, null, null); } catch (KeeperException.NodeExistsException e) { - //ignore + // ignore } try { TransactionalState.createNode(initter, rootDir, null, zkAcls, null); } catch (KeeperException.NodeExistsException e) { - //ignore + // ignore } initter.close(); - curator = CuratorUtils.newCuratorStarted(conf, servers, port, rootDir, auth, DaemonType.WORKER.getDefaultZkAcls(conf)); + curator = CuratorUtils.newCuratorStarted(conf, servers, port, rootDir, auth, + DaemonType.WORKER.getDefaultZkAcls(conf)); } catch (Exception e) { throw new RuntimeException(e); } @@ -93,7 +102,8 @@ protected static void createNode(CuratorFramework curator, String path, byte[] data, List acls, CreateMode mode) throws Exception { ProtectACLCreateModePathAndBytesable builder = curator.create().creatingParentsIfNeeded(); - LOG.debug("Creating node [path = {}], [data = {}], [acls = {}], [mode = {}]", path, asString(data), acls, mode); + LOG.debug("Creating node [path = {}], [data = {}], [acls = {}], [mode = {}]", path, + asString(data), acls, mode); if (acls == null) { if (mode == null) { @@ -170,10 +180,12 @@ public Object getData(String path) { try { Object data; if (curator.checkExists().forPath(path) != null) { - // Use parseWithException instead of parse so we can capture deserialization errors in the log. + // Use parseWithException instead of parse so we can capture deserialization errors + // in the log. // They are likely to be bugs in the spout code. try { - data = JSONValue.parseWithException(new String(curator.getData().forPath(path), "UTF-8")); + data = JSONValue.parseWithException(new String(curator.getData().forPath(path), + "UTF-8")); } catch (ParseException e) { LOG.warn("Failed to deserialize zookeeper data for path {}", path, e); data = null; diff --git a/storm-client/src/jvm/org/apache/storm/trident/tuple/ComboList.java b/storm-client/src/jvm/org/apache/storm/trident/tuple/ComboList.java index 2e84f6f28a7..339f44ed1bb 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/tuple/ComboList.java +++ b/storm-client/src/jvm/org/apache/storm/trident/tuple/ComboList.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -61,12 +67,14 @@ public Factory(int... sizes) { public ComboList create(List[] delegates) { if (delegates.length != sizes.length) { - throw new RuntimeException("Expected " + sizes.length + " lists, but instead got " + delegates.length + " lists"); + throw new RuntimeException("Expected " + sizes.length + " lists, but instead got " + + delegates.length + " lists"); } for (int i = 0; i < delegates.length; i++) { List l = delegates[i]; if (l == null || l.size() != sizes[i]) { - throw new RuntimeException("Got unexpected delegates to ComboList: " + ToStringBuilder.reflectionToString(delegates)); + throw new RuntimeException("Got unexpected delegates to ComboList: " + + ToStringBuilder.reflectionToString(delegates)); } } return new ComboList(delegates, index); diff --git a/storm-client/src/jvm/org/apache/storm/trident/tuple/ConsList.java b/storm-client/src/jvm/org/apache/storm/trident/tuple/ConsList.java index 91647aaf7cd..41da22f5fb0 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/tuple/ConsList.java +++ b/storm-client/src/jvm/org/apache/storm/trident/tuple/ConsList.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/tuple/TridentTuple.java b/storm-client/src/jvm/org/apache/storm/trident/tuple/TridentTuple.java index d0ec01e0401..dc3f7059004 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/tuple/TridentTuple.java +++ b/storm-client/src/jvm/org/apache/storm/trident/tuple/TridentTuple.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/tuple/TridentTupleView.java b/storm-client/src/jvm/org/apache/storm/trident/tuple/TridentTupleView.java index 6385477a048..bd07e2ee12d 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/tuple/TridentTupleView.java +++ b/storm-client/src/jvm/org/apache/storm/trident/tuple/TridentTupleView.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -27,13 +33,16 @@ * Extends AbstractList so that it can be emitted directly as Storm tuples. */ public class TridentTupleView extends AbstractList implements TridentTuple { - public static final TridentTupleView EMPTY_TUPLE = new TridentTupleView(null, new ValuePointer[0], new HashMap()); + public static final TridentTupleView EMPTY_TUPLE = new TridentTupleView(null, + new ValuePointer[0], new HashMap()); private final ValuePointer[] index; private final Map fieldIndex; private final List> delegates; - // index and fieldIndex are precomputed, delegates built up over many operations using persistent data structures - public TridentTupleView(List delegates, ValuePointer[] index, Map fieldIndex) { + // index and fieldIndex are precomputed, delegates built up over many operations using + // persistent data structures + public TridentTupleView(List delegates, ValuePointer[] index, Map fieldIndex) { this.delegates = delegates; this.index = index; this.fieldIndex = fieldIndex; @@ -218,7 +227,8 @@ public TridentTuple create(TridentTuple parent) { if (index.length == 0) { return EMPTY_TUPLE; } else { - return new TridentTupleView(((TridentTupleView) parent).delegates, index, fieldIndex); + return new TridentTupleView(((TridentTupleView) parent).delegates, index, + fieldIndex); } } diff --git a/storm-client/src/jvm/org/apache/storm/trident/tuple/ValuePointer.java b/storm-client/src/jvm/org/apache/storm/trident/tuple/ValuePointer.java index c67e2ade339..f386ab01d70 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/tuple/ValuePointer.java +++ b/storm-client/src/jvm/org/apache/storm/trident/tuple/ValuePointer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -36,7 +42,8 @@ public static Map buildFieldIndex(ValuePointer[] pointers) return ret; } - public static ValuePointer[] buildIndex(Fields fieldsOrder, Map pointers) { + public static ValuePointer[] buildIndex(Fields fieldsOrder, Map pointers) { if (fieldsOrder.size() != pointers.size()) { throw new IllegalArgumentException("Fields order must be same length as pointers map"); } diff --git a/storm-client/src/jvm/org/apache/storm/trident/util/ErrorEdgeFactory.java b/storm-client/src/jvm/org/apache/storm/trident/util/ErrorEdgeFactory.java index 1af43912ed0..cd4093a737d 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/util/ErrorEdgeFactory.java +++ b/storm-client/src/jvm/org/apache/storm/trident/util/ErrorEdgeFactory.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/util/IndexedEdge.java b/storm-client/src/jvm/org/apache/storm/trident/util/IndexedEdge.java index a3d52173b77..386afe75060 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/util/IndexedEdge.java +++ b/storm-client/src/jvm/org/apache/storm/trident/util/IndexedEdge.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -14,7 +20,6 @@ import java.io.Serializable; - public class IndexedEdge implements Comparable, Serializable { public T source; public T target; diff --git a/storm-client/src/jvm/org/apache/storm/trident/util/LRUMap.java b/storm-client/src/jvm/org/apache/storm/trident/util/LRUMap.java index 3f26dd30461..710e7a8879f 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/util/LRUMap.java +++ b/storm-client/src/jvm/org/apache/storm/trident/util/LRUMap.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/util/TridentUtils.java b/storm-client/src/jvm/org/apache/storm/trident/util/TridentUtils.java index 4c32017b09b..74d6f4d7dcf 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/util/TridentUtils.java +++ b/storm-client/src/jvm/org/apache/storm/trident/util/TridentUtils.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -94,7 +100,6 @@ public static List getChildren(Graph g, T n) { return ret; } - public static T getParent(Graph g, T n) { List parents = getParents(g, n); if (parents.size() != 1) { diff --git a/storm-client/src/jvm/org/apache/storm/trident/windowing/AbstractTridentWindowManager.java b/storm-client/src/jvm/org/apache/storm/trident/windowing/AbstractTridentWindowManager.java index 81f218eda6a..f9454582d3e 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/windowing/AbstractTridentWindowManager.java +++ b/storm-client/src/jvm/org/apache/storm/trident/windowing/AbstractTridentWindowManager.java @@ -1,12 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -49,7 +54,8 @@ public abstract class AbstractTridentWindowManager implements ITridentWindowM private final String windowTriggerCountId; private final TriggerPolicy triggerPolicy; - public AbstractTridentWindowManager(WindowConfig windowConfig, String windowTaskId, WindowsStore windowStore, + public AbstractTridentWindowManager(WindowConfig windowConfig, String windowTaskId, + WindowsStore windowStore, Aggregator aggregator, BatchOutputCollector delegateCollector) { this.windowTaskId = windowTaskId; this.windowStore = windowStore; @@ -96,7 +102,8 @@ private void postInitialize() { } /** - * Load and initialize any resources into window manager before windowing for component/task is activated. + * Load and initialize any resources into window manager before windowing for component/task is + * activated. */ protected abstract void initialize(); @@ -118,8 +125,10 @@ private void execAggregatorAndStoreResult(int currentTriggerId, List tupleEve List> resultantAggregatedValue = collector.values; - ArrayList entries = Lists.newArrayList(new WindowsStore.Entry(windowTriggerCountId, currentTriggerId + 1), - new WindowsStore.Entry(WindowTridentProcessor + ArrayList entries = Lists.newArrayList(new WindowsStore + .Entry(windowTriggerCountId, currentTriggerId + 1), + new WindowsStore + .Entry(WindowTridentProcessor .generateWindowTriggerKey(windowTaskId, currentTriggerId), resultantAggregatedValue)); @@ -228,7 +237,8 @@ public void onExpiry(List expiredEvents) { } @Override - public void onActivation(List events, List newEvents, List expired, Long timestamp) { + public void onActivation(List events, List newEvents, List expired, + Long timestamp) { LOG.debug("onActivation is invoked with events size: [{}]", events.size()); // trigger occurred, create an aggregation and keep them in store int currentTriggerId = triggerId.incrementAndGet(); diff --git a/storm-client/src/jvm/org/apache/storm/trident/windowing/ITridentWindowManager.java b/storm-client/src/jvm/org/apache/storm/trident/windowing/ITridentWindowManager.java index 167e444b9c3..d3e98d3db94 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/windowing/ITridentWindowManager.java +++ b/storm-client/src/jvm/org/apache/storm/trident/windowing/ITridentWindowManager.java @@ -1,12 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -23,7 +28,8 @@ public interface ITridentWindowManager { /** * This is invoked from {@code org.apache.storm.trident.planner.TridentProcessor}'s prepare method. So any initialization tasks can be - * done before the topology starts accepting tuples. For ex: initialize window manager with any earlier stored tuples/triggers and start + * done before the topology starts accepting tuples. For ex: initialize window manager with any + * earlier stored tuples/triggers and start * WindowManager. */ void prepare(); diff --git a/storm-client/src/jvm/org/apache/storm/trident/windowing/InMemoryTridentWindowManager.java b/storm-client/src/jvm/org/apache/storm/trident/windowing/InMemoryTridentWindowManager.java index 6be4be4a10b..34ec1b9a0f9 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/windowing/InMemoryTridentWindowManager.java +++ b/storm-client/src/jvm/org/apache/storm/trident/windowing/InMemoryTridentWindowManager.java @@ -1,12 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -27,7 +32,8 @@ public class InMemoryTridentWindowManager extends AbstractTridentWindowManager { private static final Logger LOG = LoggerFactory.getLogger(InMemoryTridentWindowManager.class); - public InMemoryTridentWindowManager(WindowConfig windowConfig, String windowTaskId, WindowsStore windowStore, Aggregator aggregator, + public InMemoryTridentWindowManager(WindowConfig windowConfig, String windowTaskId, + WindowsStore windowStore, Aggregator aggregator, BatchOutputCollector delegateCollector) { super(windowConfig, windowTaskId, windowStore, aggregator, delegateCollector); } diff --git a/storm-client/src/jvm/org/apache/storm/trident/windowing/InMemoryWindowsStore.java b/storm-client/src/jvm/org/apache/storm/trident/windowing/InMemoryWindowsStore.java index a398fb01288..fb0c84bd74d 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/windowing/InMemoryWindowsStore.java +++ b/storm-client/src/jvm/org/apache/storm/trident/windowing/InMemoryWindowsStore.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

    - * http://www.apache.org/licenses/LICENSE-2.0 - *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

    http://www.apache.org/licenses/LICENSE-2.0 + * + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -37,6 +43,7 @@ public InMemoryWindowsStore() { /** * Creates a new in-memory window store. + * * @param maxSize maximum size of inmemory store * @param backingStore backing store containing the entries */ @@ -86,7 +93,8 @@ public String next() { @Override public void remove() { - throw new UnsupportedOperationException("remove operation is not supported as it is immutable."); + throw new UnsupportedOperationException("remove operation is not supported as it " + + "is immutable."); } }; diff --git a/storm-client/src/jvm/org/apache/storm/trident/windowing/InMemoryWindowsStoreFactory.java b/storm-client/src/jvm/org/apache/storm/trident/windowing/InMemoryWindowsStoreFactory.java index a8aef767961..c6da25eb5dc 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/windowing/InMemoryWindowsStoreFactory.java +++ b/storm-client/src/jvm/org/apache/storm/trident/windowing/InMemoryWindowsStoreFactory.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

    - * http://www.apache.org/licenses/LICENSE-2.0 - *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

    http://www.apache.org/licenses/LICENSE-2.0 + * + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -17,8 +23,10 @@ import org.apache.storm.trident.operation.TridentCollector; /** - * InMemoryWindowsStoreFactory contains a single instance of {@link InMemoryWindowsStore} which will be used for storing tuples and triggers - * of the window. The same InMemoryWindowsStoreFactory instance is passed to {@link WindowsStateUpdater}, which removes successfully emitted + * InMemoryWindowsStoreFactory contains a single instance of {@link InMemoryWindowsStore} which will + * be used for storing tuples and triggers + * of the window. The same InMemoryWindowsStoreFactory instance is passed to {@link + * WindowsStateUpdater}, which removes successfully emitted * triggers from the same {@code inMemoryWindowsStore} instance in {@link WindowsStateUpdater#updateState(WindowsState, List, * TridentCollector)}. */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/windowing/StoreBasedTridentWindowManager.java b/storm-client/src/jvm/org/apache/storm/trident/windowing/StoreBasedTridentWindowManager.java index f07d3f171d8..c0bd1dd6f43 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/windowing/StoreBasedTridentWindowManager.java +++ b/storm-client/src/jvm/org/apache/storm/trident/windowing/StoreBasedTridentWindowManager.java @@ -1,12 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -42,7 +47,8 @@ public class StoreBasedTridentWindowManager extends AbstractTridentWindowManager private Long maxCachedTuplesSize; private AtomicLong currentCachedTuplesSize = new AtomicLong(); - public StoreBasedTridentWindowManager(WindowConfig windowConfig, String windowTaskId, WindowsStore windowStore, Aggregator aggregator, + public StoreBasedTridentWindowManager(WindowConfig windowConfig, String windowTaskId, + WindowsStore windowStore, Aggregator aggregator, BatchOutputCollector delegateCollector, Long maxTuplesCacheSize, Fields inputFields) { super(windowConfig, windowTaskId, windowStore, aggregator, delegateCollector); @@ -55,9 +61,12 @@ public StoreBasedTridentWindowManager(WindowConfig windowConfig, String windowTa @Override protected void initialize() { - // get existing tuples and pending/unsuccessful triggers for this operator-component/task and add them to WindowManager - String windowTriggerInprocessId = WindowTridentProcessor.getWindowTriggerInprocessIdPrefix(windowTaskId); - String windowTriggerTaskId = WindowTridentProcessor.getWindowTriggerTaskPrefix(windowTaskId); + // get existing tuples and pending/unsuccessful triggers for this operator-component/task + // and add them to WindowManager + String windowTriggerInprocessId = WindowTridentProcessor + .getWindowTriggerInprocessIdPrefix(windowTaskId); + String windowTriggerTaskId = WindowTridentProcessor + .getWindowTriggerTaskPrefix(windowTaskId); List attemptedTriggerKeys = new ArrayList<>(); List triggerKeys = new ArrayList<>(); @@ -67,14 +76,17 @@ protected void initialize() { if (key.startsWith(windowTupleTaskId)) { int tupleIndexValue = lastPart(key); String batchId = secondLastPart(key); - LOG.debug("Received tuple with batch [{}] and tuple index [{}]", batchId, tupleIndexValue); - windowManager.add(new TridentBatchTuple(batchId, System.currentTimeMillis(), tupleIndexValue)); + LOG.debug("Received tuple with batch [{}] and tuple index [{}]", batchId, + tupleIndexValue); + windowManager.add(new TridentBatchTuple(batchId, System.currentTimeMillis(), + tupleIndexValue)); } else if (key.startsWith(windowTriggerTaskId)) { triggerKeys.add(key); LOG.debug("Received trigger with key [{}]", key); } else if (key.startsWith(windowTriggerInprocessId)) { attemptedTriggerKeys.add(key); - LOG.debug("Received earlier unsuccessful trigger [{}] from windows store [{}]", key); + LOG.debug("Received earlier unsuccessful trigger [{}] from windows store [{}]", + key); } } @@ -101,7 +113,8 @@ protected void initialize() { private int lastPart(String key) { int lastSepIndex = key.lastIndexOf(WindowsStore.KEY_SEPARATOR); if (lastSepIndex < 0) { - throw new IllegalArgumentException("primaryKey does not have key separator '" + WindowsStore.KEY_SEPARATOR + "'"); + throw new IllegalArgumentException("primaryKey does not have key separator '" + + WindowsStore.KEY_SEPARATOR + "'"); } return Integer.parseInt(key.substring(lastSepIndex + 1)); } @@ -109,12 +122,14 @@ private int lastPart(String key) { private String secondLastPart(String key) { int lastSepIndex = key.lastIndexOf(WindowsStore.KEY_SEPARATOR); if (lastSepIndex < 0) { - throw new IllegalArgumentException("key " + key + " does not have key separator '" + WindowsStore.KEY_SEPARATOR + "'"); + throw new IllegalArgumentException("key " + key + " does not have key separator '" + + WindowsStore.KEY_SEPARATOR + "'"); } String trimKey = key.substring(0, lastSepIndex); int secondLastSepIndex = trimKey.lastIndexOf(WindowsStore.KEY_SEPARATOR); if (secondLastSepIndex < 0) { - throw new IllegalArgumentException("key " + key + " does not have second key separator '" + WindowsStore.KEY_SEPARATOR + "'"); + throw new IllegalArgumentException("key " + key + + " does not have second key separator '" + WindowsStore.KEY_SEPARATOR + "'"); } return key.substring(secondLastSepIndex + 1, lastSepIndex); @@ -141,13 +156,15 @@ public void addTuplesBatch(Object batchId, List tuples) { } - private void addToWindowManager(int tupleIndex, String effectiveBatchId, TridentTuple tridentTuple) { + private void addToWindowManager(int tupleIndex, String effectiveBatchId, + TridentTuple tridentTuple) { TridentTuple actualTuple = null; if (maxCachedTuplesSize == null || currentCachedTuplesSize.get() < maxCachedTuplesSize) { actualTuple = tridentTuple; } currentCachedTuplesSize.incrementAndGet(); - windowManager.add(new TridentBatchTuple(effectiveBatchId, System.currentTimeMillis(), tupleIndex, actualTuple)); + windowManager.add(new TridentBatchTuple(effectiveBatchId, System.currentTimeMillis(), + tupleIndex, actualTuple)); } public String getBatchTxnId(Object batchId) { @@ -175,7 +192,8 @@ public List getTridentTuples(List tridentBatchT if (keys.size() > 0) { Iterable storedTupleValues = windowStore.get(keys); for (Object storedTupleValue : storedTupleValues) { - TridentTuple tridentTuple = freshOutputFactory.create((List) storedTupleValue); + TridentTuple tridentTuple = freshOutputFactory + .create((List) storedTupleValue); resultTuples.add(tridentTuple); } } @@ -183,7 +201,8 @@ public List getTridentTuples(List tridentBatchT return resultTuples; } - public TridentTuple collectTridentTupleOrKey(TridentBatchTuple tridentBatchTuple, List keys) { + public TridentTuple collectTridentTupleOrKey(TridentBatchTuple tridentBatchTuple, + List keys) { if (tridentBatchTuple.tridentTuple != null) { return tridentBatchTuple.tridentTuple; } diff --git a/storm-client/src/jvm/org/apache/storm/trident/windowing/TridentBatchTuple.java b/storm-client/src/jvm/org/apache/storm/trident/windowing/TridentBatchTuple.java index 7b429080f00..1d1b2ea297d 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/windowing/TridentBatchTuple.java +++ b/storm-client/src/jvm/org/apache/storm/trident/windowing/TridentBatchTuple.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

    - * http://www.apache.org/licenses/LICENSE-2.0 - *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

    http://www.apache.org/licenses/LICENSE-2.0 + * + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -24,7 +30,8 @@ public TridentBatchTuple(String effectiveBatchId, long timeStamp, int tupleIndex this(effectiveBatchId, timeStamp, tupleIndex, null); } - public TridentBatchTuple(String effectiveBatchId, long timeStamp, int tupleIndex, TridentTuple tridentTuple) { + public TridentBatchTuple(String effectiveBatchId, long timeStamp, int tupleIndex, + TridentTuple tridentTuple) { this.effectiveBatchId = effectiveBatchId; this.timeStamp = timeStamp; this.tupleIndex = tupleIndex; diff --git a/storm-client/src/jvm/org/apache/storm/trident/windowing/WindowKryoSerializer.java b/storm-client/src/jvm/org/apache/storm/trident/windowing/WindowKryoSerializer.java index bda83b38e30..3d932be4311 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/windowing/WindowKryoSerializer.java +++ b/storm-client/src/jvm/org/apache/storm/trident/windowing/WindowKryoSerializer.java @@ -1,12 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -20,7 +25,8 @@ import org.apache.storm.serialization.SerializationFactory; /** - * Kryo serializer/deserializer for values that are stored as part of windowing. This can be used in {@link WindowsStore}. This class is not + * Kryo serializer/deserializer for values that are stored as part of windowing. This can be used in + * {@link WindowsStore}. This class is not * thread safe. */ public class WindowKryoSerializer { @@ -47,7 +53,8 @@ public byte[] serialize(Object obj) { } /** - * Serializes the given object into a {@link ByteBuffer} backed by the byte array returned by Kryo serialization. + * Serializes the given object into a {@link ByteBuffer} backed by the byte array returned by + * Kryo serialization. * * @param obj Object to be serialized. */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/windowing/WindowTridentProcessor.java b/storm-client/src/jvm/org/apache/storm/trident/windowing/WindowTridentProcessor.java index c5e7f938897..214242b71e6 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/windowing/WindowTridentProcessor.java +++ b/storm-client/src/jvm/org/apache/storm/trident/windowing/WindowTridentProcessor.java @@ -1,12 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -61,7 +66,8 @@ public class WindowTridentProcessor implements TridentProcessor { private ITridentWindowManager tridentWindowManager; private String windowTaskId; - public WindowTridentProcessor(WindowConfig windowConfig, String uniqueWindowId, WindowsStoreFactory windowStoreFactory, + public WindowTridentProcessor(WindowConfig windowConfig, String uniqueWindowId, + WindowsStoreFactory windowStoreFactory, Fields inputFields, Aggregator aggregator, boolean storeTuplesInStore) { this.windowConfig = windowConfig; @@ -100,7 +106,8 @@ public static String generateWindowTriggerKey(String windowTaskId, int triggerId } @Override - public void prepare(Map topoConf, TopologyContext context, TridentContext tridentContext) { + public void prepare(Map topoConf, TopologyContext context, + TridentContext tridentContext) { this.topologyContext = context; List parents = tridentContext.getParentTupleFactories(); if (parents.size() != 1) { @@ -112,7 +119,8 @@ public void prepare(Map topoConf, TopologyContext context, Tride projection = new TridentTupleView.ProjectionFactory(parents.get(0), inputFields); windowStore = windowStoreFactory.create(topoConf); - windowTaskId = windowId + WindowsStore.KEY_SEPARATOR + topologyContext.getThisTaskId() + WindowsStore.KEY_SEPARATOR; + windowTaskId = windowId + WindowsStore.KEY_SEPARATOR + topologyContext.getThisTaskId() + + WindowsStore.KEY_SEPARATOR; windowTriggerInprocessId = getWindowTriggerInprocessIdPrefix(windowTaskId); Long maxTuplesCacheSize = getWindowTuplesCacheSize(topoConf); @@ -135,7 +143,8 @@ public void prepare(Map topoConf, TopologyContext context, Tride private Long getWindowTuplesCacheSize(Map conf) { if (conf.containsKey(Config.TOPOLOGY_TRIDENT_WINDOWING_INMEMORY_CACHE_LIMIT)) { - return ((Number) conf.get(Config.TOPOLOGY_TRIDENT_WINDOWING_INMEMORY_CACHE_LIMIT)).longValue(); + return ((Number) conf.get(Config.TOPOLOGY_TRIDENT_WINDOWING_INMEMORY_CACHE_LIMIT)) + .longValue(); } return DEFAULT_INMEMORY_TUPLE_CACHE_LIMIT; } @@ -177,7 +186,8 @@ public void finishBatch(ProcessorContext processorContext) { LOG.debug("Received finishBatch of : [{}] ", batchId); // get all the tuples in a batch and add it to trident-window-manager - List tuples = (List) processorContext.state[tridentContext.getStateIndex()]; + List tuples = (List) processorContext.state[tridentContext + .getStateIndex()]; tridentWindowManager.addTuplesBatch(batchId, tuples); List pendingTriggerIds = null; @@ -194,13 +204,17 @@ public void finishBatch(ProcessorContext processorContext) { } } - // if there are no trigger values in earlier attempts or this is a new batch, emit pending triggers. + // if there are no trigger values in earlier attempts or this is a new batch, emit pending + // triggers. if (triggerValues == null) { pendingTriggerIds = new ArrayList<>(); - Queue pendingTriggers = tridentWindowManager.getPendingTriggers(); - LOG.debug("pending triggers at batch: [{}] and triggers.size: [{}] ", batchId, pendingTriggers.size()); + Queue pendingTriggers = + tridentWindowManager.getPendingTriggers(); + LOG.debug("pending triggers at batch: [{}] and triggers.size: [{}] ", batchId, + pendingTriggers.size()); try { - Iterator pendingTriggersIter = pendingTriggers.iterator(); + Iterator pendingTriggersIter = + pendingTriggers.iterator(); List values = new ArrayList<>(); StoreBasedTridentWindowManager.TriggerResult triggerResult = null; while (pendingTriggersIter.hasNext()) { @@ -225,7 +239,8 @@ public void finishBatch(ProcessorContext processorContext) { collector.setContext(processorContext); int i = 0; for (Object resultValue : triggerValues) { - collector.emit(new ConsList(new TriggerInfo(windowTaskId, pendingTriggerIds.get(i++)), (List) resultValue)); + collector.emit(new ConsList(new TriggerInfo(windowTaskId, pendingTriggerIds.get(i++)), + (List) resultValue)); } collector.setContext(null); } diff --git a/storm-client/src/jvm/org/apache/storm/trident/windowing/WindowsState.java b/storm-client/src/jvm/org/apache/storm/trident/windowing/WindowsState.java index 1d99bc4f35b..26c5916e90f 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/windowing/WindowsState.java +++ b/storm-client/src/jvm/org/apache/storm/trident/windowing/WindowsState.java @@ -1,12 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/windowing/WindowsStateFactory.java b/storm-client/src/jvm/org/apache/storm/trident/windowing/WindowsStateFactory.java index d56212c66ba..9b685777f3a 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/windowing/WindowsStateFactory.java +++ b/storm-client/src/jvm/org/apache/storm/trident/windowing/WindowsStateFactory.java @@ -1,12 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -26,7 +31,8 @@ public WindowsStateFactory() { } @Override - public State makeState(Map conf, IMetricsContext metrics, int partitionIndex, int numPartitions) { + public State makeState(Map conf, IMetricsContext metrics, int partitionIndex, + int numPartitions) { return new WindowsState(); } } diff --git a/storm-client/src/jvm/org/apache/storm/trident/windowing/WindowsStateUpdater.java b/storm-client/src/jvm/org/apache/storm/trident/windowing/WindowsStateUpdater.java index d8a77eb5465..b67c56ebefe 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/windowing/WindowsStateUpdater.java +++ b/storm-client/src/jvm/org/apache/storm/trident/windowing/WindowsStateUpdater.java @@ -1,12 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -38,23 +43,30 @@ public WindowsStateUpdater(WindowsStoreFactory windowStoreFactory) { } @Override - public void updateState(WindowsState state, List tuples, TridentCollector collector) { + public void updateState(WindowsState state, List tuples, + TridentCollector collector) { Long currentTxId = state.getCurrentTxId(); LOG.debug("Removing triggers using WindowStateUpdater, txnId: [{}] ", currentTxId); for (TridentTuple tuple : tuples) { try { - Object fieldValue = tuple.getValueByField(WindowTridentProcessor.TRIGGER_FIELD_NAME); + Object fieldValue = tuple + .getValueByField(WindowTridentProcessor.TRIGGER_FIELD_NAME); if (!(fieldValue instanceof WindowTridentProcessor.TriggerInfo)) { - throw new ClassCastException("Expected WindowTridentProcessor.TriggerInfo but found " + fieldValue.getClass()); + throw new ClassCastException("Expected WindowTridentProcessor.TriggerInfo but " + + "found " + fieldValue.getClass()); } - WindowTridentProcessor.TriggerInfo triggerInfo = (WindowTridentProcessor.TriggerInfo) fieldValue; + WindowTridentProcessor.TriggerInfo triggerInfo = + (WindowTridentProcessor.TriggerInfo) fieldValue; String triggerCompletedKey = - WindowTridentProcessor.getWindowTriggerInprocessIdPrefix(triggerInfo.windowTaskId) + currentTxId; + WindowTridentProcessor + .getWindowTriggerInprocessIdPrefix(triggerInfo.windowTaskId) + currentTxId; - LOG.debug("Removing trigger key [{}] and trigger completed key [{}] from store: [{}]", triggerInfo, triggerCompletedKey, + LOG.debug("Removing trigger key [{}] and trigger completed key [{}] from store: " + + "[{}]", triggerInfo, triggerCompletedKey, windowsStore); - windowsStore.removeAll(Lists.newArrayList(triggerInfo.generateTriggerKey(), triggerCompletedKey)); + windowsStore.removeAll(Lists.newArrayList(triggerInfo.generateTriggerKey(), + triggerCompletedKey)); } catch (Exception ex) { LOG.warn(ex.getMessage()); collector.reportError(ex); diff --git a/storm-client/src/jvm/org/apache/storm/trident/windowing/WindowsStore.java b/storm-client/src/jvm/org/apache/storm/trident/windowing/WindowsStore.java index e8e5b3d765d..8662825b122 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/windowing/WindowsStore.java +++ b/storm-client/src/jvm/org/apache/storm/trident/windowing/WindowsStore.java @@ -1,12 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -18,7 +23,8 @@ import org.apache.storm.shade.com.google.common.base.Preconditions; /** - * Store for storing window related entities like windowed tuples, triggers etc. {@link WindowKryoSerializer} can be used for kryo + * Store for storing window related entities like windowed tuples, triggers etc. {@link + * WindowKryoSerializer} can be used for kryo * serialization/deserialization of keys and values. */ public interface WindowsStore extends Serializable { diff --git a/storm-client/src/jvm/org/apache/storm/trident/windowing/WindowsStoreFactory.java b/storm-client/src/jvm/org/apache/storm/trident/windowing/WindowsStoreFactory.java index 46621cd86d8..cbc7a88c575 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/windowing/WindowsStoreFactory.java +++ b/storm-client/src/jvm/org/apache/storm/trident/windowing/WindowsStoreFactory.java @@ -1,12 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -25,7 +30,8 @@ public interface WindowsStoreFactory extends Serializable { /** * Creates a window store. * - * @param topoConf storm topology configuration passed in {@link org.apache.storm.trident.planner.TridentProcessor#prepare(Map, + * @param topoConf storm topology configuration passed in {@link + * org.apache.storm.trident.planner.TridentProcessor#prepare(Map, * TopologyContext, TridentContext)} */ WindowsStore create(Map topoConf); diff --git a/storm-client/src/jvm/org/apache/storm/trident/windowing/config/BaseWindowConfig.java b/storm-client/src/jvm/org/apache/storm/trident/windowing/config/BaseWindowConfig.java index c1b94ad011a..1b70cab41a5 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/windowing/config/BaseWindowConfig.java +++ b/storm-client/src/jvm/org/apache/storm/trident/windowing/config/BaseWindowConfig.java @@ -1,12 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -35,7 +40,8 @@ public int getSlidingLength() { public void validate() { if (slideLength > windowLength) { throw new IllegalArgumentException( - "slideLength '" + slideLength + "' should always be less than windowLegth '" + windowLength + "'"); + "slideLength '" + slideLength + "' should always be less than windowLegth '" + + windowLength + "'"); } } } diff --git a/storm-client/src/jvm/org/apache/storm/trident/windowing/config/SlidingCountWindow.java b/storm-client/src/jvm/org/apache/storm/trident/windowing/config/SlidingCountWindow.java index c81d52b31fe..9d4bd34c1a4 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/windowing/config/SlidingCountWindow.java +++ b/storm-client/src/jvm/org/apache/storm/trident/windowing/config/SlidingCountWindow.java @@ -1,12 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/windowing/config/SlidingDurationWindow.java b/storm-client/src/jvm/org/apache/storm/trident/windowing/config/SlidingDurationWindow.java index 4a6e6536722..13c73b73f70 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/windowing/config/SlidingDurationWindow.java +++ b/storm-client/src/jvm/org/apache/storm/trident/windowing/config/SlidingDurationWindow.java @@ -1,12 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -26,7 +31,8 @@ private SlidingDurationWindow(int windowLength, int slideLength) { super(windowLength, slideLength); } - public static SlidingDurationWindow of(BaseWindowedBolt.Duration windowDuration, BaseWindowedBolt.Duration slidingDuration) { + public static SlidingDurationWindow of(BaseWindowedBolt.Duration windowDuration, + BaseWindowedBolt.Duration slidingDuration) { return new SlidingDurationWindow(windowDuration.value, slidingDuration.value); } diff --git a/storm-client/src/jvm/org/apache/storm/trident/windowing/config/TumblingCountWindow.java b/storm-client/src/jvm/org/apache/storm/trident/windowing/config/TumblingCountWindow.java index 1646fa52c8f..f2aa0ed5dfa 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/windowing/config/TumblingCountWindow.java +++ b/storm-client/src/jvm/org/apache/storm/trident/windowing/config/TumblingCountWindow.java @@ -1,12 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/windowing/config/TumblingDurationWindow.java b/storm-client/src/jvm/org/apache/storm/trident/windowing/config/TumblingDurationWindow.java index 75d1057c7cf..30bd5a05a99 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/windowing/config/TumblingDurationWindow.java +++ b/storm-client/src/jvm/org/apache/storm/trident/windowing/config/TumblingDurationWindow.java @@ -1,12 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/windowing/config/WindowConfig.java b/storm-client/src/jvm/org/apache/storm/trident/windowing/config/WindowConfig.java index 49b9d6a895e..fd26bf6097a 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/windowing/config/WindowConfig.java +++ b/storm-client/src/jvm/org/apache/storm/trident/windowing/config/WindowConfig.java @@ -1,12 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/windowing/strategy/BaseWindowStrategy.java b/storm-client/src/jvm/org/apache/storm/trident/windowing/strategy/BaseWindowStrategy.java index 0a14924468e..0054820a91b 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/windowing/strategy/BaseWindowStrategy.java +++ b/storm-client/src/jvm/org/apache/storm/trident/windowing/strategy/BaseWindowStrategy.java @@ -1,12 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/trident/windowing/strategy/SlidingCountWindowStrategy.java b/storm-client/src/jvm/org/apache/storm/trident/windowing/strategy/SlidingCountWindowStrategy.java index c4ff1900433..33fb6b9eda9 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/windowing/strategy/SlidingCountWindowStrategy.java +++ b/storm-client/src/jvm/org/apache/storm/trident/windowing/strategy/SlidingCountWindowStrategy.java @@ -1,12 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -33,8 +38,10 @@ public SlidingCountWindowStrategy(WindowConfig slidingCountWindow) { * Returns a {@code TriggerPolicy} which triggers for every count of given sliding window. */ @Override - public TriggerPolicy getTriggerPolicy(TriggerHandler triggerHandler, EvictionPolicy evictionPolicy) { - return new CountTriggerPolicy<>(windowConfig.getSlidingLength(), triggerHandler, evictionPolicy); + public TriggerPolicy getTriggerPolicy(TriggerHandler triggerHandler, EvictionPolicy evictionPolicy) { + return new CountTriggerPolicy<>(windowConfig.getSlidingLength(), triggerHandler, + evictionPolicy); } /** diff --git a/storm-client/src/jvm/org/apache/storm/trident/windowing/strategy/SlidingDurationWindowStrategy.java b/storm-client/src/jvm/org/apache/storm/trident/windowing/strategy/SlidingDurationWindowStrategy.java index d086fcbcd96..44ef8012f49 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/windowing/strategy/SlidingDurationWindowStrategy.java +++ b/storm-client/src/jvm/org/apache/storm/trident/windowing/strategy/SlidingDurationWindowStrategy.java @@ -1,12 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -33,8 +38,10 @@ public SlidingDurationWindowStrategy(WindowConfig slidingDurationWindow) { * Returns a {@code TriggerPolicy} which triggers for every configured sliding window duration. */ @Override - public TriggerPolicy getTriggerPolicy(TriggerHandler triggerHandler, EvictionPolicy evictionPolicy) { - return new TimeTriggerPolicy<>(windowConfig.getSlidingLength(), triggerHandler, evictionPolicy); + public TriggerPolicy getTriggerPolicy(TriggerHandler triggerHandler, EvictionPolicy evictionPolicy) { + return new TimeTriggerPolicy<>(windowConfig.getSlidingLength(), triggerHandler, + evictionPolicy); } /** diff --git a/storm-client/src/jvm/org/apache/storm/trident/windowing/strategy/TumblingCountWindowStrategy.java b/storm-client/src/jvm/org/apache/storm/trident/windowing/strategy/TumblingCountWindowStrategy.java index 85e2f6f4f96..62fc3a34443 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/windowing/strategy/TumblingCountWindowStrategy.java +++ b/storm-client/src/jvm/org/apache/storm/trident/windowing/strategy/TumblingCountWindowStrategy.java @@ -1,12 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -33,8 +38,10 @@ public TumblingCountWindowStrategy(WindowConfig tumblingCountWindow) { * Returns a {@code TriggerPolicy} which triggers for every count of given sliding window. */ @Override - public TriggerPolicy getTriggerPolicy(TriggerHandler triggerHandler, EvictionPolicy evictionPolicy) { - return new CountTriggerPolicy<>(windowConfig.getSlidingLength(), triggerHandler, evictionPolicy); + public TriggerPolicy getTriggerPolicy(TriggerHandler triggerHandler, EvictionPolicy evictionPolicy) { + return new CountTriggerPolicy<>(windowConfig.getSlidingLength(), triggerHandler, + evictionPolicy); } /** diff --git a/storm-client/src/jvm/org/apache/storm/trident/windowing/strategy/TumblingDurationWindowStrategy.java b/storm-client/src/jvm/org/apache/storm/trident/windowing/strategy/TumblingDurationWindowStrategy.java index 7e2932504fa..bc8a8211e13 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/windowing/strategy/TumblingDurationWindowStrategy.java +++ b/storm-client/src/jvm/org/apache/storm/trident/windowing/strategy/TumblingDurationWindowStrategy.java @@ -1,12 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -33,8 +38,10 @@ public TumblingDurationWindowStrategy(WindowConfig tumblingDurationWindow) { * Returns a {@code TriggerPolicy} which triggers for every given sliding duration. */ @Override - public TriggerPolicy getTriggerPolicy(TriggerHandler triggerHandler, EvictionPolicy evictionPolicy) { - return new TimeTriggerPolicy<>(windowConfig.getSlidingLength(), triggerHandler, evictionPolicy); + public TriggerPolicy getTriggerPolicy(TriggerHandler triggerHandler, EvictionPolicy evictionPolicy) { + return new TimeTriggerPolicy<>(windowConfig.getSlidingLength(), triggerHandler, + evictionPolicy); } /** diff --git a/storm-client/src/jvm/org/apache/storm/trident/windowing/strategy/WindowStrategy.java b/storm-client/src/jvm/org/apache/storm/trident/windowing/strategy/WindowStrategy.java index 860390e32a0..caf510a7495 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/windowing/strategy/WindowStrategy.java +++ b/storm-client/src/jvm/org/apache/storm/trident/windowing/strategy/WindowStrategy.java @@ -1,12 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -24,7 +29,8 @@ public interface WindowStrategy { /** * Returns a {@code TriggerPolicy} by creating with {@code triggerHandler} and {@code evictionPolicy} with the given configuration. */ - TriggerPolicy getTriggerPolicy(TriggerHandler triggerHandler, EvictionPolicy evictionPolicy); + TriggerPolicy getTriggerPolicy(TriggerHandler triggerHandler, EvictionPolicy evictionPolicy); /** * Returns an {@code EvictionPolicy} instance for this strategy with the given configuration. diff --git a/storm-client/src/jvm/org/apache/storm/tuple/AddressedTuple.java b/storm-client/src/jvm/org/apache/storm/tuple/AddressedTuple.java index 83f292a2593..399027ba2bd 100644 --- a/storm-client/src/jvm/org/apache/storm/tuple/AddressedTuple.java +++ b/storm-client/src/jvm/org/apache/storm/tuple/AddressedTuple.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -32,9 +38,11 @@ public AddressedTuple(int dest, Tuple tuple) { } public static AddressedTuple createFlushTuple(GeneralTopologyContext workerTopologyContext) { - TupleImpl tuple = new TupleImpl(workerTopologyContext, new Values(), Constants.SYSTEM_COMPONENT_ID, + TupleImpl tuple = new TupleImpl(workerTopologyContext, new Values(), + Constants.SYSTEM_COMPONENT_ID, (int) Constants.SYSTEM_TASK_ID, Constants.SYSTEM_FLUSH_STREAM_ID); - return new AddressedTuple(AddressedTuple.BROADCAST_DEST, tuple); // one instance per executor avoids false sharing of CPU cache + return new AddressedTuple(AddressedTuple.BROADCAST_DEST, + tuple); // one instance per executor avoids false sharing of CPU cache } public Tuple getTuple() { diff --git a/storm-client/src/jvm/org/apache/storm/tuple/DetachedTuple.java b/storm-client/src/jvm/org/apache/storm/tuple/DetachedTuple.java index c3635fbc4b3..e078170afdc 100644 --- a/storm-client/src/jvm/org/apache/storm/tuple/DetachedTuple.java +++ b/storm-client/src/jvm/org/apache/storm/tuple/DetachedTuple.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -20,16 +26,22 @@ import org.apache.storm.task.GeneralTopologyContext; /** - * A self-contained, serializable copy of a {@link Tuple} that is detached from the topology context. + * A self-contained, serializable copy of a {@link Tuple} that is detached from the topology + * context. * - *

    A regular {@link TupleImpl} holds a reference to the {@link GeneralTopologyContext} it was created in, which cannot be + *

    A regular {@link TupleImpl} holds a reference to the {@link GeneralTopologyContext} it was + * created in, which cannot be * serialized. A {@code DetachedTuple} instead snapshots the source component, task, stream, output fields and values of the - * original tuple, so it can be emitted as a value inside another tuple and cross worker boundaries (see STORM-4000). + * original tuple, so it can be emitted as a value inside another tuple and cross worker boundaries + * (see STORM-4000). * - *

    Detached tuples are unanchored: {@link #getMessageId()} always returns an unanchored message id, and - * {@link #getContext()} throws {@link UnsupportedOperationException} since no topology context is available. + *

    Detached tuples are unanchored: {@link #getMessageId()} always returns an unanchored message + * id, and + * {@link #getContext()} throws {@link UnsupportedOperationException} since no topology context is + * available. * - *

    Unlike {@link TupleImpl}, which uses identity-based equality, two detached tuples are equal if they snapshot + *

    Unlike {@link TupleImpl}, which uses identity-based equality, two detached tuples are equal if + * they snapshot * the same source metadata, fields and values, so a detached tuple keeps comparing equal after a * serialization round-trip. */ @@ -44,7 +56,8 @@ public class DetachedTuple implements Tuple, Serializable { private transient Fields fields; /** - * Creates a detached copy of the given tuple. The tuple must still be attached to its topology context, since the + * Creates a detached copy of the given tuple. The tuple must still be attached to its topology + * context, since the * output fields of the source component are resolved through it. * * @param tuple the tuple to detach diff --git a/storm-client/src/jvm/org/apache/storm/tuple/Fields.java b/storm-client/src/jvm/org/apache/storm/tuple/Fields.java index c76e36595e5..2bef15d8076 100644 --- a/storm-client/src/jvm/org/apache/storm/tuple/Fields.java +++ b/storm-client/src/jvm/org/apache/storm/tuple/Fields.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -46,7 +52,8 @@ public Fields(List fields) { } /** - * Select values out of tuple given a Fields selector Note that this function can throw a NullPointerException if the fields in selector + * Select values out of tuple given a Fields selector Note that this function can throw a + * NullPointerException if the fields in selector * are not found in the index. * * @param selector Fields to select @@ -75,7 +82,8 @@ public int size() { * Gets the field at position index in the collection. * * @param index index of the field to return - * @throws IndexOutOfBoundsException - if the index is out of range (index < 0 || index >= size()) + * @throws IndexOutOfBoundsException - if the index is out of range (index < 0 || index >= + * size()) */ public String get(int index) { return fields.get(index); @@ -102,6 +110,7 @@ public int fieldIndex(String field) { /** * Check contains. + * * @return true if this contains the specified name of the field. */ public boolean contains(String field) { diff --git a/storm-client/src/jvm/org/apache/storm/tuple/ITuple.java b/storm-client/src/jvm/org/apache/storm/tuple/ITuple.java index d176474fa9d..fdcd0354347 100644 --- a/storm-client/src/jvm/org/apache/storm/tuple/ITuple.java +++ b/storm-client/src/jvm/org/apache/storm/tuple/ITuple.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -46,7 +52,8 @@ public interface ITuple { /** * Gets the field at position i in the tuple. Returns object since tuples are dynamically typed. * - * @throws IndexOutOfBoundsException - if the index is out of range `(index < 0 || index >= size())` + * @throws IndexOutOfBoundsException - if the index is out of range `(index < 0 || index >= + * size())` */ Object getValue(int i); @@ -54,7 +61,8 @@ public interface ITuple { * Returns the String at position i in the tuple. * * @throws ClassCastException If that field is not a String - * @throws IndexOutOfBoundsException - if the index is out of range `(index < 0 || index >= size())` + * @throws IndexOutOfBoundsException - if the index is out of range `(index < 0 || index >= + * size())` */ String getString(int i); @@ -62,7 +70,8 @@ public interface ITuple { * Returns the Integer at position i in the tuple. * * @throws ClassCastException If that field is not a Integer - * @throws IndexOutOfBoundsException - if the index is out of range `(index < 0 || index >= size())` + * @throws IndexOutOfBoundsException - if the index is out of range `(index < 0 || index >= + * size())` */ Integer getInteger(int i); @@ -70,7 +79,8 @@ public interface ITuple { * Returns the Long at position i in the tuple. * * @throws ClassCastException If that field is not a Long - * @throws IndexOutOfBoundsException - if the index is out of range `(index < 0 || index >= size())` + * @throws IndexOutOfBoundsException - if the index is out of range `(index < 0 || index >= + * size())` */ Long getLong(int i); @@ -78,7 +88,8 @@ public interface ITuple { * Returns the Boolean at position i in the tuple. * * @throws ClassCastException If that field is not a Boolean - * @throws IndexOutOfBoundsException - if the index is out of range `(index < 0 || index >= size())` + * @throws IndexOutOfBoundsException - if the index is out of range `(index < 0 || index >= + * size())` */ Boolean getBoolean(int i); @@ -86,7 +97,8 @@ public interface ITuple { * Returns the Short at position i in the tuple. * * @throws ClassCastException If that field is not a Short - * @throws IndexOutOfBoundsException - if the index is out of range `(index < 0 || index >= size())` + * @throws IndexOutOfBoundsException - if the index is out of range `(index < 0 || index >= + * size())` */ Short getShort(int i); @@ -94,7 +106,8 @@ public interface ITuple { * Returns the Byte at position i in the tuple. * * @throws ClassCastException If that field is not a Byte - * @throws IndexOutOfBoundsException - if the index is out of range `(index < 0 || index >= size())` + * @throws IndexOutOfBoundsException - if the index is out of range `(index < 0 || index >= + * size())` */ Byte getByte(int i); @@ -102,7 +115,8 @@ public interface ITuple { * Returns the Double at position i in the tuple. * * @throws ClassCastException If that field is not a Double - * @throws IndexOutOfBoundsException - if the index is out of range `(index < 0 || index >= size())` + * @throws IndexOutOfBoundsException - if the index is out of range `(index < 0 || index >= + * size())` */ Double getDouble(int i); @@ -110,7 +124,8 @@ public interface ITuple { * Returns the Float at position i in the tuple. * * @throws ClassCastException If that field is not a Float - * @throws IndexOutOfBoundsException - if the index is out of range `(index < 0 || index >= size())` + * @throws IndexOutOfBoundsException - if the index is out of range `(index < 0 || index >= + * size())` */ Float getFloat(int i); @@ -118,7 +133,8 @@ public interface ITuple { * Returns the byte array at position i in the tuple. * * @throws ClassCastException If that field is not a byte array - * @throws IndexOutOfBoundsException - if the index is out of range `(index < 0 || index >= size())` + * @throws IndexOutOfBoundsException - if the index is out of range `(index < 0 || index >= + * size())` */ byte[] getBinary(int i); diff --git a/storm-client/src/jvm/org/apache/storm/tuple/MessageId.java b/storm-client/src/jvm/org/apache/storm/tuple/MessageId.java index cb10dd16177..356fa0e7ab4 100644 --- a/storm-client/src/jvm/org/apache/storm/tuple/MessageId.java +++ b/storm-client/src/jvm/org/apache/storm/tuple/MessageId.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -17,8 +23,8 @@ import java.io.IOException; import java.util.Collections; import java.util.HashMap; -import java.util.Map; import java.util.Map.Entry; +import java.util.Map; import java.util.Random; import java.util.Set; diff --git a/storm-client/src/jvm/org/apache/storm/tuple/Tuple.java b/storm-client/src/jvm/org/apache/storm/tuple/Tuple.java index 4dc51a2fc07..99e17f038a9 100644 --- a/storm-client/src/jvm/org/apache/storm/tuple/Tuple.java +++ b/storm-client/src/jvm/org/apache/storm/tuple/Tuple.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -16,12 +22,16 @@ import org.apache.storm.task.GeneralTopologyContext; /** - * The tuple is the main data structure in Storm. A tuple is a named list of values, where each value can be any type. Tuples are - * dynamically typed -- the types of the fields do not need to be declared. Tuples have helper methods like getInteger and getString to get + * The tuple is the main data structure in Storm. A tuple is a named list of values, where each + * value can be any type. Tuples are + * dynamically typed -- the types of the fields do not need to be declared. Tuples have helper + * methods like getInteger and getString to get * field values without having to cast the result. * - *

    Storm needs to know how to serialize all the values in a tuple. By default, Storm knows how to serialize the primitive types, strings, - * and byte arrays. If you want to use another type, you'll need to implement and register a serializer for that type. + *

    Storm needs to know how to serialize all the values in a tuple. By default, Storm knows how to + * serialize the primitive types, strings, + * and byte arrays. If you want to use another type, you'll need to implement and register a + * serializer for that type. * * @see Serialization */ diff --git a/storm-client/src/jvm/org/apache/storm/tuple/TupleImpl.java b/storm-client/src/jvm/org/apache/storm/tuple/TupleImpl.java index e0a2827eba5..00ce8b4efbd 100644 --- a/storm-client/src/jvm/org/apache/storm/tuple/TupleImpl.java +++ b/storm-client/src/jvm/org/apache/storm/tuple/TupleImpl.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -45,7 +51,8 @@ public TupleImpl(Tuple t) { } } - public TupleImpl(GeneralTopologyContext context, List values, String srcComponent, int taskId, String streamId, MessageId id) { + public TupleImpl(GeneralTopologyContext context, List values, String srcComponent, + int taskId, String streamId, MessageId id) { this.values = context.doSanityCheck() ? Collections.unmodifiableList(values) : values; this.taskId = taskId; this.streamId = streamId; @@ -57,13 +64,16 @@ public TupleImpl(GeneralTopologyContext context, List values, String src String componentId = context.getComponentId(taskId); Fields schema = context.getComponentOutputFields(componentId, streamId); if (values.size() != schema.size()) { - throw new IllegalArgumentException("Tuple created with wrong number of fields. Expected " + schema.size() - + " fields but got " + values.size() + " fields"); + throw new IllegalArgumentException("Tuple created with wrong number of fields. " + + "Expected " + schema.size() + + " fields but got " + values.size() + + " fields"); } } } - public TupleImpl(GeneralTopologyContext context, List values, String srcComponent, int taskId, String streamId) { + public TupleImpl(GeneralTopologyContext context, List values, String srcComponent, + int taskId, String streamId) { this(context, values, srcComponent, taskId, streamId, MessageId.makeUnanchored()); } diff --git a/storm-client/src/jvm/org/apache/storm/tuple/Values.java b/storm-client/src/jvm/org/apache/storm/tuple/Values.java index a663866ff4a..bb5730d83c7 100644 --- a/storm-client/src/jvm/org/apache/storm/tuple/Values.java +++ b/storm-client/src/jvm/org/apache/storm/tuple/Values.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/utils/BatchHelper.java b/storm-client/src/jvm/org/apache/storm/utils/BatchHelper.java index a75fb213ccc..361c60319fa 100644 --- a/storm-client/src/jvm/org/apache/storm/utils/BatchHelper.java +++ b/storm-client/src/jvm/org/apache/storm/utils/BatchHelper.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -23,7 +29,7 @@ public class BatchHelper { private static final Logger LOG = LoggerFactory.getLogger(BatchHelper.class); - private int batchSize = 15000; //default batch size 15000 + private int batchSize = 15000; // default batch size 15000 private List tupleBatch; private boolean forceFlush = false; private OutputCollector collector; diff --git a/storm-client/src/jvm/org/apache/storm/utils/BufferFileInputStream.java b/storm-client/src/jvm/org/apache/storm/utils/BufferFileInputStream.java index d533280d583..66da43e552f 100644 --- a/storm-client/src/jvm/org/apache/storm/utils/BufferFileInputStream.java +++ b/storm-client/src/jvm/org/apache/storm/utils/BufferFileInputStream.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -17,7 +23,6 @@ import java.io.IOException; import java.util.Arrays; - public class BufferFileInputStream { byte[] buffer; FileInputStream stream; diff --git a/storm-client/src/jvm/org/apache/storm/utils/CRC32OutputStream.java b/storm-client/src/jvm/org/apache/storm/utils/CRC32OutputStream.java index 7401ac5a57f..7817ba05fd1 100644 --- a/storm-client/src/jvm/org/apache/storm/utils/CRC32OutputStream.java +++ b/storm-client/src/jvm/org/apache/storm/utils/CRC32OutputStream.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/utils/ConfigUtils.java b/storm-client/src/jvm/org/apache/storm/utils/ConfigUtils.java index c78b38ff58d..15e2289ce81 100644 --- a/storm-client/src/jvm/org/apache/storm/utils/ConfigUtils.java +++ b/storm-client/src/jvm/org/apache/storm/utils/ConfigUtils.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -27,7 +33,6 @@ import java.util.function.BooleanSupplier; import java.util.regex.Pattern; import java.util.stream.Collectors; - import org.apache.storm.Config; import org.apache.storm.daemon.supervisor.AdvancedFSOps; import org.apache.storm.generated.StormTopology; @@ -43,7 +48,8 @@ public class ConfigUtils { public static final double RFC1889_ALPHA = 1.0 / 16.0; private static final Set passwordConfigKeys = new HashSet<>(); - private static final Pattern CREDENTIAL_KEY_NAME = Pattern.compile("(?i)(password|passwd|secret)"); + private static final Pattern CREDENTIAL_KEY_NAME = Pattern + .compile("(?i)(password|passwd|secret)"); static { for (Class clazz : ConfigValidation.getConfigClasses()) { @@ -63,13 +69,13 @@ public class ConfigUtils { } } - // A singleton instance allows us to mock delegated static methods in our // tests by subclassing. private static ConfigUtils instance = new ConfigUtils(); /** - * Provide an instance of this class for delegates to use. To mock out delegated methods, provide an instance of a subclass that + * Provide an instance of this class for delegates to use. To mock out delegated methods, + * provide an instance of a subclass that * overrides the implementation of the delegated method. * * @param u a ConfigUtils instance @@ -82,41 +88,49 @@ public static ConfigUtils setInstance(ConfigUtils u) { } public static Map maskPasswords(final Map conf) { - Maps.EntryTransformer maskPasswords = new Maps.EntryTransformer() { - @Override + Maps.EntryTransformer maskPasswords = + new Maps.EntryTransformer() { + @Override public Object transformEntry(String key, Object value) { - return passwordConfigKeys.contains(key) ? "*****" : value; - } - }; + return passwordConfigKeys.contains(key) ? "*****" : value; + } + }; return Maps.transformEntries(conf, maskPasswords); } /** * Mask credential values before a config map is served over an API. This covers what - * {@link #maskPasswords(Map)} covers, plus string values whose key name denotes a secret: plugins read their - * own keys straight out of the config map, so those keys are declared by no annotated field and the annotation - * scan cannot see them. Only string values are considered, so timeouts and class lists whose names merely + * {@link #maskPasswords(Map)} covers, plus string values whose key name denotes a secret: + * plugins read their + * own keys straight out of the config map, so those keys are declared by no annotated field and + * the annotation + * scan cannot see them. Only string values are considered, so timeouts and class lists whose + * names merely * mention credentials keep their value. * * @param conf the config to mask * @return a view of the config with credential values replaced */ public static Map maskCredentials(final Map conf) { - Maps.EntryTransformer maskCredentials = new Maps.EntryTransformer() { - @Override + Maps.EntryTransformer maskCredentials = + new Maps.EntryTransformer() { + @Override public Object transformEntry(String key, Object value) { - if (passwordConfigKeys.contains(key)) { - return "*****"; + if (passwordConfigKeys.contains(key)) { + return "*****"; + } + return value instanceof String && CREDENTIAL_KEY_NAME.matcher(key).find() + ? "*****" : value; } - return value instanceof String && CREDENTIAL_KEY_NAME.matcher(key).find() ? "*****" : value; - } - }; + }; return Maps.transformEntries(conf, maskCredentials); } /** - * Whether a config key holds a credential, and therefore whether {@link #maskCredentials(Map)} would replace its - * value. Callers that read a config back from a daemon use this to tell which entries carry no usable value and + * Whether a config key holds a credential, and therefore whether {@link #maskCredentials(Map)} + * would replace its + * value. Callers that read a config back from a daemon use this to tell which entries carry no + * usable value and * must be taken from their own configuration instead. * * @param key the config key @@ -244,7 +258,8 @@ public static boolean upstreamFeedbackEnable(Map conf) { } public static int upstreamFeedbackFreqSecs(Map conf) { - int freqSecs = ObjectReader.getInt(conf.get(Config.TOPOLOGY_UPSTREAM_FEEDBACK_FREQ_SECS), 10); + int freqSecs = ObjectReader.getInt(conf.get(Config.TOPOLOGY_UPSTREAM_FEEDBACK_FREQ_SECS), + 10); if (freqSecs > 0) { return freqSecs; } @@ -276,7 +291,8 @@ public boolean getAsBoolean() { }; } - public static StormTopology readSupervisorTopology(Map conf, String stormId, AdvancedFSOps ops) throws IOException { + public static StormTopology readSupervisorTopology(Map conf, String stormId, + AdvancedFSOps ops) throws IOException { return instance.readSupervisorTopologyImpl(conf, stormId, ops); } @@ -286,7 +302,8 @@ public static String supervisorStormCodePath(String stormRoot) { public static String concatIfNotNull(String dir) { String ret = ""; - // we do this since to concat a null String will actually concat a "null", which is not the expected: "" + // we do this since to concat a null String will actually concat a "null", which is not the + // expected: "" if (dir != null) { ret = dir; } @@ -299,11 +316,13 @@ public static String supervisorStormDistRoot(Map conf) throws IO } // we use this "weird" wrapper pattern temporarily for mocking in clojure test - public static String supervisorStormDistRoot(Map conf, String stormId) throws IOException { + public static String supervisorStormDistRoot(Map conf, + String stormId) throws IOException { return instance.supervisorStormDistRootImpl(conf, stormId); } - public static String sharedByTopologyDir(Map conf, String stormId) throws IOException { + public static String sharedByTopologyDir(Map conf, + String stormId) throws IOException { return supervisorStormDistRoot(conf, stormId) + FILE_SEPARATOR + "shared_by_topology"; } @@ -343,7 +362,8 @@ public static String absoluteStormBlobStoreDir(Map conf) { } } - public static StormTopology readSupervisorStormCodeGivenPath(String stormCodePath, AdvancedFSOps ops) throws IOException { + public static StormTopology readSupervisorStormCodeGivenPath(String stormCodePath, + AdvancedFSOps ops) throws IOException { return Utils.deserialize(ops.slurp(new File(stormCodePath)), StormTopology.class); } @@ -381,13 +401,16 @@ public static String workerArtifactsPidPath(Map conf, String id, } // we use this "weird" wrapper pattern temporarily for mocking in clojure test - public static Map readSupervisorStormConf(Map conf, String stormId) throws IOException { + public static Map readSupervisorStormConf(Map conf, + String stormId) throws IOException { return instance.readSupervisorStormConfImpl(conf, stormId); } - public static Map readSupervisorStormConfGivenPath(Map conf, String topoConfPath) throws IOException { + public static Map readSupervisorStormConfGivenPath(Map conf, + String topoConfPath) throws IOException { Map ret = new HashMap<>(conf); - ret.putAll(Utils.fromCompressedJsonConf(FileUtils.readFileToByteArray(new File(topoConfPath)))); + ret.putAll(Utils.fromCompressedJsonConf(FileUtils + .readFileToByteArray(new File(topoConfPath)))); return ret; } @@ -457,7 +480,8 @@ public static Map readYamlConfig(String name) { public static String stormDistPath(String stormRoot) { String ret = ""; - // we do this since to concat a null String will actually concat a "null", which is not the expected: "" + // we do this since to concat a null String will actually concat a "null", which is not the + // expected: "" if (stormRoot != null) { ret = stormRoot; } @@ -514,7 +538,8 @@ public static List getValueAsList(String name, Map conf) return listValue; } - public StormTopology readSupervisorTopologyImpl(Map conf, String stormId, AdvancedFSOps ops) throws IOException { + public StormTopology readSupervisorTopologyImpl(Map conf, String stormId, + AdvancedFSOps ops) throws IOException { String stormRoot = supervisorStormDistRoot(conf, stormId); String topologyPath = supervisorStormCodePath(stormRoot); return readSupervisorStormCodeGivenPath(topologyPath, ops); @@ -539,7 +564,8 @@ public String workerArtifactsRootImpl(Map conf) { } } - public String supervisorStormDistRootImpl(Map conf, String stormId) throws IOException { + public String supervisorStormDistRootImpl(Map conf, + String stormId) throws IOException { return supervisorStormDistRoot(conf) + FILE_SEPARATOR + Utils.urlEncodeUtf8(stormId); } @@ -551,7 +577,8 @@ public String workerRootImpl(Map conf) { return (absoluteStormLocalDir(conf) + FILE_SEPARATOR + "workers"); } - public Map readSupervisorStormConfImpl(Map conf, String stormId) throws IOException { + public Map readSupervisorStormConfImpl(Map conf, + String stormId) throws IOException { String stormRoot = supervisorStormDistRoot(conf, stormId); String confPath = supervisorStormConfPath(stormRoot); return readSupervisorStormConfGivenPath(conf, confPath); diff --git a/storm-client/src/jvm/org/apache/storm/utils/CuratorUtils.java b/storm-client/src/jvm/org/apache/storm/utils/CuratorUtils.java index d5e6f3f3e0d..a002ae13257 100644 --- a/storm-client/src/jvm/org/apache/storm/utils/CuratorUtils.java +++ b/storm-client/src/jvm/org/apache/storm/utils/CuratorUtils.java @@ -38,17 +38,20 @@ public class CuratorUtils { public static final String CLIENT_CNXN = org.apache.storm.shade.org.apache.zookeeper.ClientCnxnSocketNetty.class.getName(); - public static CuratorFramework newCurator(Map conf, List servers, Object port, String root, + public static CuratorFramework newCurator(Map conf, List servers, + Object port, String root, List defaultAcl) { return newCurator(conf, servers, port, root, null, defaultAcl); } - public static CuratorFramework newCurator(Map conf, List servers, Object port, ZookeeperAuthInfo auth, + public static CuratorFramework newCurator(Map conf, List servers, + Object port, ZookeeperAuthInfo auth, List defaultAcl) { return newCurator(conf, servers, port, "", auth, defaultAcl); } - public static CuratorFramework newCurator(Map conf, List servers, Object port, String root, + public static CuratorFramework newCurator(Map conf, List servers, + Object port, String root, ZookeeperAuthInfo auth, final List defaultAcl) { List serverPorts = new ArrayList<>(); for (String zkServer : servers) { @@ -75,15 +78,19 @@ public List getAclForPath(String s) { return builder.build(); } - protected static void setupBuilder(CuratorFrameworkFactory.Builder builder, final String zkStr, Map conf, + protected static void setupBuilder(CuratorFrameworkFactory.Builder builder, final String zkStr, + Map conf, ZookeeperAuthInfo auth) { builder.connectString(zkStr); builder - .connectionTimeoutMs(ObjectReader.getInt(conf.get(Config.STORM_ZOOKEEPER_CONNECTION_TIMEOUT))) - .sessionTimeoutMs(ObjectReader.getInt(conf.get(Config.STORM_ZOOKEEPER_SESSION_TIMEOUT))) + .connectionTimeoutMs(ObjectReader.getInt(conf + .get(Config.STORM_ZOOKEEPER_CONNECTION_TIMEOUT))) + .sessionTimeoutMs(ObjectReader.getInt(conf + .get(Config.STORM_ZOOKEEPER_SESSION_TIMEOUT))) .retryPolicy(new StormBoundedExponentialBackoffRetry( ObjectReader.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_INTERVAL)), - ObjectReader.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_INTERVAL_CEILING)), + ObjectReader.getInt(conf + .get(Config.STORM_ZOOKEEPER_RETRY_INTERVAL_CEILING)), ObjectReader.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_TIMES)))); if (auth != null && auth.scheme != null && auth.payload != null) { @@ -104,6 +111,7 @@ protected static void setupBuilder(CuratorFrameworkFactory.Builder builder, fina /** * Configure ZooKeeper Client with SSL/TLS connection. + * * @param zkClientConfig ZooKeeper Client configuration * @param x509Util The X509 utility * @param sslConf The truststore and keystore configs @@ -147,11 +155,13 @@ private static void validateSslConfiguration(SslConf sslConf) throws Configurati } if (StringUtils.isEmpty(sslConf.getTruststoreLocation())) { throw new ConfigurationException( - "The truststore location parameter is empty for the ZooKeeper client connection" + "."); + "The truststore location parameter is empty for the ZooKeeper client " + + "connection" + "."); } if (StringUtils.isEmpty(sslConf.getTruststorePassword())) { throw new ConfigurationException( - "The truststore password parameter is empty for the ZooKeeper client connection" + "."); + "The truststore password parameter is empty for the ZooKeeper client " + + "connection" + "."); } } @@ -176,11 +186,16 @@ static final class SslConf { * @param conf configuration map */ private SslConf(Map conf) { - keystoreLocation = ObjectReader.getString(conf.get(Config.STORM_ZOOKEEPER_SSL_KEYSTORE_PATH), ""); - keystorePassword = ObjectReader.getString(conf.get(Config.STORM_ZOOKEEPER_SSL_KEYSTORE_PASSWORD), ""); - truststoreLocation = ObjectReader.getString(conf.get(Config.STORM_ZOOKEEPER_SSL_TRUSTSTORE_PATH), ""); - truststorePassword = ObjectReader.getString(conf.get(Config.STORM_ZOOKEEPER_SSL_TRUSTSTORE_PASSWORD), ""); - hostnameVerification = ObjectReader.getBoolean(conf.get(Config.STORM_ZOOKEEPER_SSL_HOSTNAME_VERIFICATION), true); + keystoreLocation = ObjectReader.getString(conf + .get(Config.STORM_ZOOKEEPER_SSL_KEYSTORE_PATH), ""); + keystorePassword = ObjectReader.getString(conf + .get(Config.STORM_ZOOKEEPER_SSL_KEYSTORE_PASSWORD), ""); + truststoreLocation = ObjectReader.getString(conf + .get(Config.STORM_ZOOKEEPER_SSL_TRUSTSTORE_PATH), ""); + truststorePassword = ObjectReader.getString(conf + .get(Config.STORM_ZOOKEEPER_SSL_TRUSTSTORE_PASSWORD), ""); + hostnameVerification = ObjectReader.getBoolean(conf + .get(Config.STORM_ZOOKEEPER_SSL_HOSTNAME_VERIFICATION), true); } public String getKeystoreLocation() { @@ -209,7 +224,8 @@ public static void testSetupBuilder(CuratorFrameworkFactory.Builder setupBuilder(builder, zkStr, conf, auth); } - public static CuratorFramework newCuratorStarted(Map conf, List servers, Object port, + public static CuratorFramework newCuratorStarted(Map conf, List servers, + Object port, String root, ZookeeperAuthInfo auth, List defaultAcl) { CuratorFramework ret = newCurator(conf, servers, port, root, auth, defaultAcl); LOG.info("Starting Utils Curator..."); @@ -217,7 +233,8 @@ public static CuratorFramework newCuratorStarted(Map conf, List< return ret; } - public static CuratorFramework newCuratorStarted(Map conf, List servers, Object port, + public static CuratorFramework newCuratorStarted(Map conf, List servers, + Object port, ZookeeperAuthInfo auth, List defaultAcl) { CuratorFramework ret = newCurator(conf, servers, port, auth, defaultAcl); LOG.info("Starting Utils Curator (2)..."); diff --git a/storm-client/src/jvm/org/apache/storm/utils/DRPCClient.java b/storm-client/src/jvm/org/apache/storm/utils/DRPCClient.java index f5ba4b1ec09..9c9d464fb45 100644 --- a/storm-client/src/jvm/org/apache/storm/utils/DRPCClient.java +++ b/storm-client/src/jvm/org/apache/storm/utils/DRPCClient.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -50,8 +56,10 @@ public DRPCClient(Map conf, String host, int port) throws TTrans retryForever = true; } - public DRPCClient(Map conf, String host, int port, Integer timeout) throws TTransportException { - super(conf, localOverrideClient != null ? ThriftConnectionType.LOCAL_FAKE : ThriftConnectionType.DRPC, + public DRPCClient(Map conf, String host, int port, + Integer timeout) throws TTransportException { + super(conf, localOverrideClient != null + ? ThriftConnectionType.LOCAL_FAKE : ThriftConnectionType.DRPC, host, port, timeout, null); this.host = host; this.port = port; @@ -65,7 +73,9 @@ public DRPCClient(Map conf, String host, int port, Integer timeo /** * Check local override. - * @return true of new clients will be overridden to connect to a local cluster and not the configured remote cluster + * + * @return true of new clients will be overridden to connect to a local cluster and not the + * configured remote cluster */ public static boolean isLocalOverride() { return localOverrideClient != null; @@ -73,19 +83,21 @@ public static boolean isLocalOverride() { /** * Get override service ID. + * * @return the service ID of the local override DRPC instance */ public static String getOverrideServiceId() { return localOverrideClient.getServiceId(); } - public static DRPCClient getConfiguredClient(Map conf) throws TTransportException { + public static DRPCClient getConfiguredClient(Map conf) throws TTransportException { DistributedRPC.Iface override = localOverrideClient; if (override != null) { return new DRPCClient(override); } - //Extend the config with defaults and the command line + // Extend the config with defaults and the command line Map fullConf = Utils.readStormConfig(); fullConf.putAll(Utils.readCommandLineOpts()); fullConf.putAll(conf); @@ -93,7 +105,8 @@ public static DRPCClient getConfiguredClient(Map conf) throws TT int port = ObjectReader.getInt(fullConf.get(Config.DRPC_PORT), 3772); List servers = (List) fullConf.get(Config.DRPC_SERVERS); if (servers == null) { - throw new IllegalStateException(Config.DRPC_SERVERS + " is not set, could not find any DRPC servers to connect to."); + throw new IllegalStateException(Config.DRPC_SERVERS + + " is not set, could not find any DRPC servers to connect to."); } Collections.shuffle(servers); RuntimeException excpt = null; @@ -123,7 +136,8 @@ public int getPort() { } @Override - public String execute(String func, String args) throws TException, DRPCExecutionException, AuthorizationException { + public String execute(String func, + String args) throws TException, DRPCExecutionException, AuthorizationException { if (func == null) { throw new IllegalArgumentException("DRPC Function cannot be null"); } diff --git a/storm-client/src/jvm/org/apache/storm/utils/DefaultShellLogHandler.java b/storm-client/src/jvm/org/apache/storm/utils/DefaultShellLogHandler.java index 83ba501590e..525a2e397c9 100644 --- a/storm-client/src/jvm/org/apache/storm/utils/DefaultShellLogHandler.java +++ b/storm-client/src/jvm/org/apache/storm/utils/DefaultShellLogHandler.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -40,7 +46,8 @@ private Logger getLogger(final Class ownerCls) { } /** - * This default implementation saves the {@link ShellProcess} so it can output the process info string later. + * This default implementation saves the {@link ShellProcess} so it can output the process info + * string later. * * @param ownerCls - the class which instantiated this ShellLogHandler. * @param process - the current {@link ShellProcess}. diff --git a/storm-client/src/jvm/org/apache/storm/utils/DisallowedStrategyException.java b/storm-client/src/jvm/org/apache/storm/utils/DisallowedStrategyException.java index fd93bfd6bc8..aef70011ebc 100644 --- a/storm-client/src/jvm/org/apache/storm/utils/DisallowedStrategyException.java +++ b/storm-client/src/jvm/org/apache/storm/utils/DisallowedStrategyException.java @@ -25,9 +25,10 @@ public class DisallowedStrategyException extends RuntimeException { private String attemptedClass; private List allowedStrategies; - public DisallowedStrategyException(String attemptedClass, List allowedStrategies) { - super(attemptedClass + " is not an allowed scheduler strategy. Either pick one of the allowed strategies " + super(attemptedClass + + " is not an allowed scheduler strategy. Either pick one of the allowed " + + "strategies " + allowedStrategies + " or add " + attemptedClass + " to the nimbus config " + Config.NIMBUS_SCHEDULER_STRATEGY_CLASS_WHITELIST); this.attemptedClass = attemptedClass; diff --git a/storm-client/src/jvm/org/apache/storm/utils/ExtendedThreadPoolExecutor.java b/storm-client/src/jvm/org/apache/storm/utils/ExtendedThreadPoolExecutor.java index 86b196d677b..2785d31651b 100644 --- a/storm-client/src/jvm/org/apache/storm/utils/ExtendedThreadPoolExecutor.java +++ b/storm-client/src/jvm/org/apache/storm/utils/ExtendedThreadPoolExecutor.java @@ -29,24 +29,29 @@ public class ExtendedThreadPoolExecutor extends ThreadPoolExecutor { - public ExtendedThreadPoolExecutor(int corePoolSize, int maximumPoolSize, long keepAliveTime, TimeUnit unit, + public ExtendedThreadPoolExecutor(int corePoolSize, int maximumPoolSize, long keepAliveTime, + TimeUnit unit, BlockingQueue workQueue) { super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue); } - public ExtendedThreadPoolExecutor(int corePoolSize, int maximumPoolSize, long keepAliveTime, TimeUnit unit, + public ExtendedThreadPoolExecutor(int corePoolSize, int maximumPoolSize, long keepAliveTime, + TimeUnit unit, BlockingQueue workQueue, ThreadFactory threadFactory) { super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory); } - public ExtendedThreadPoolExecutor(int corePoolSize, int maximumPoolSize, long keepAliveTime, TimeUnit unit, + public ExtendedThreadPoolExecutor(int corePoolSize, int maximumPoolSize, long keepAliveTime, + TimeUnit unit, BlockingQueue workQueue, RejectedExecutionHandler handler) { super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, handler); } - public ExtendedThreadPoolExecutor(int corePoolSize, int maximumPoolSize, long keepAliveTime, TimeUnit unit, + public ExtendedThreadPoolExecutor(int corePoolSize, int maximumPoolSize, long keepAliveTime, + TimeUnit unit, BlockingQueue workQueue, ThreadFactory threadFactory, RejectedExecutionHandler handler) { - super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory, handler); + super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory, + handler); } @Override diff --git a/storm-client/src/jvm/org/apache/storm/utils/HadoopLoginUtil.java b/storm-client/src/jvm/org/apache/storm/utils/HadoopLoginUtil.java index 76d3a1a671d..12ff0e2d8e1 100644 --- a/storm-client/src/jvm/org/apache/storm/utils/HadoopLoginUtil.java +++ b/storm-client/src/jvm/org/apache/storm/utils/HadoopLoginUtil.java @@ -30,7 +30,8 @@ import org.slf4j.LoggerFactory; /** - * UserGroupInformation#loginUserFromKeytab(String, String) changes the static fields of UserGroupInformation, + * UserGroupInformation#loginUserFromKeytab(String, String) changes the static fields of + * UserGroupInformation, * especially the current logged-in user, and UserGroupInformation itself is not thread-safe. * So it could introduce bugs if it is called multiple times in a JVM process. * HadoopLoginUtil.loginHadoop guarantees at-most-once login in a JVM process. @@ -47,6 +48,7 @@ public class HadoopLoginUtil { * Login if a HDFS keytab/principal have been supplied; * otherwise, assume it's already logged in or running on insecure HDFS. * This also guarantees that login only happens at most once. + * * @param conf the daemon conf * @return the logged in subject or null */ @@ -65,10 +67,12 @@ public static Subject loginHadoop(Map conf) { loginFromKeytab(principal, keyTab); } else { if (principal == null && keyTab != null) { - throw new IllegalArgumentException("HDFS principal is null while keytab is present"); + throw new IllegalArgumentException("HDFS principal is null while " + + "keytab is present"); } else { if (principal != null && keyTab == null) { - throw new IllegalArgumentException("HDFS keytab is null while principal is present"); + throw new IllegalArgumentException("HDFS keytab is null while " + + "principal is present"); } } } @@ -86,8 +90,9 @@ public static Subject loginHadoop(Map conf) { return loginSubject; } - //The Hadoop UserGroupInformation class name - private static final String HADOOP_USER_GROUP_INFORMATION_CLASS = "org.apache.hadoop.security.UserGroupInformation"; + // The Hadoop UserGroupInformation class name + private static final String HADOOP_USER_GROUP_INFORMATION_CLASS = + "org.apache.hadoop.security.UserGroupInformation"; private static void loginFromKeytab(String principal, String keyTab) { Preconditions.checkNotNull(principal); diff --git a/storm-client/src/jvm/org/apache/storm/utils/IPredicate.java b/storm-client/src/jvm/org/apache/storm/utils/IPredicate.java index 0c27c0d35fb..df54177eda1 100644 --- a/storm-client/src/jvm/org/apache/storm/utils/IPredicate.java +++ b/storm-client/src/jvm/org/apache/storm/utils/IPredicate.java @@ -1,19 +1,26 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.utils; /** - * This interface is implemented by classes, instances of which can be passed into certain Util functions which test some collection for + * This interface is implemented by classes, instances of which can be passed into certain Util + * functions which test some collection for * elements matching the IPredicate. (IPredicate.test(...) == true) */ public interface IPredicate { diff --git a/storm-client/src/jvm/org/apache/storm/utils/IVersionInfo.java b/storm-client/src/jvm/org/apache/storm/utils/IVersionInfo.java index 35f274117a5..cf60832ca38 100644 --- a/storm-client/src/jvm/org/apache/storm/utils/IVersionInfo.java +++ b/storm-client/src/jvm/org/apache/storm/utils/IVersionInfo.java @@ -24,36 +24,42 @@ public interface IVersionInfo { /** * Get the version number of the build. + * * @return the version number of the build. */ String getVersion(); /** * Get the SCM revision number of the build. + * * @return the SCM revision number of the build. */ String getRevision(); /** * Get the SCM branch of the build. + * * @return the SCM branch of the build. */ String getBranch(); /** * Get the date/time the build happened. + * * @return the date/time of the build. */ String getDate(); /** * Get the checksum of the source. + * * @return the checksum of the source. */ String getSrcChecksum(); /** * Get a descriptive representation of the build meant for human consumption. + * * @return a descriptive representation of the build. */ String getBuildVersion(); diff --git a/storm-client/src/jvm/org/apache/storm/utils/InprocMessaging.java b/storm-client/src/jvm/org/apache/storm/utils/InprocMessaging.java index 9ecae4631a4..1d04d7aea3e 100644 --- a/storm-client/src/jvm/org/apache/storm/utils/InprocMessaging.java +++ b/storm-client/src/jvm/org/apache/storm/utils/InprocMessaging.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -23,7 +29,8 @@ public class InprocMessaging { private static final Logger LOG = LoggerFactory.getLogger(InprocMessaging.class); - private static Map> _queues = new HashMap>(); + private static Map> _queues = + new HashMap>(); private static ConcurrentMap _hasReader = new ConcurrentHashMap<>(); private static int port = 1; @@ -71,7 +78,7 @@ public static void waitForReader(int port) { try { Thread.sleep(10); } catch (InterruptedException e) { - //Ignored + // Ignored } if (Time.currentTimeMillis() - start > 20000) { LOG.error("DONE WAITING FOR READER AFTER {} ms", Time.currentTimeMillis() - start); diff --git a/storm-client/src/jvm/org/apache/storm/utils/JCQueue.java b/storm-client/src/jvm/org/apache/storm/utils/JCQueue.java index 3d6492e9633..8bbe8f788a8 100644 --- a/storm-client/src/jvm/org/apache/storm/utils/JCQueue.java +++ b/storm-client/src/jvm/org/apache/storm/utils/JCQueue.java @@ -40,38 +40,48 @@ public class JCQueue implements Closeable { private final MpscArrayQueue recvQueue; // only holds msgs from other workers (via WorkerTransfer), when recvQueue is full private final MpscUnboundedArrayQueue overflowQ; - // dedicated lane for low-volume control tuples (flush/tick/metrics tick/feedback), drained before recvQueue. - // Like recvQueue, this lane is not drained on close(): any tuples still buffered at shutdown are discarded. + // dedicated lane for low-volume control tuples (flush/tick/metrics tick/feedback), drained + // before recvQueue. + // Like recvQueue, this lane is not drained on close(): any tuples still buffered at shutdown + // are discarded. private final MpscArrayQueue controlQueue; private final int overflowLimit; // ensures... overflowCount <= overflowLimit. if set to 0, disables overflow limiting. private final int producerBatchSz; private final boolean dynamicBatch; private final DirectInserter directInserter = new DirectInserter(this); - private final ThreadLocal thdLocalBatcher = new ThreadLocal(); // ensure 1 instance per producer thd. + private final ThreadLocal thdLocalBatcher = + new ThreadLocal(); // ensure 1 instance per producer thd. // ensure 1 instance per producer thd. - private final ThreadLocal thdLocalDynamicBatcher = new ThreadLocal(); + private final ThreadLocal thdLocalDynamicBatcher = + new ThreadLocal(); private final IWaitStrategy backPressureWaitStrategy; private final String queueName; - // Throttle for the control-lane-full WARN so a stalled executor cannot flood the logs; log at most once per interval. + // Throttle for the control-lane-full WARN so a stalled executor cannot flood the logs; log at + // most once per interval. private static final long CONTROL_DROP_LOG_INTERVAL_NANOS = TimeUnit.SECONDS.toNanos(30); private final AtomicLong lastControlDropLogNanos = new AtomicLong(Long.MIN_VALUE); - public JCQueue(String queueName, String metricNamePrefix, int size, int overflowLimit, int producerBatchSz, + public JCQueue(String queueName, String metricNamePrefix, int size, int overflowLimit, + int producerBatchSz, IWaitStrategy backPressureWaitStrategy, String topologyId, String componentId, List taskIds, int port, StormMetricRegistry metricRegistry) { - this(queueName, metricNamePrefix, size, overflowLimit, producerBatchSz, backPressureWaitStrategy, topologyId, componentId, + this(queueName, metricNamePrefix, size, overflowLimit, producerBatchSz, + backPressureWaitStrategy, topologyId, componentId, taskIds, port, metricRegistry, false); } - public JCQueue(String queueName, String metricNamePrefix, int size, int overflowLimit, int producerBatchSz, + public JCQueue(String queueName, String metricNamePrefix, int size, int overflowLimit, + int producerBatchSz, IWaitStrategy backPressureWaitStrategy, String topologyId, String componentId, List taskIds, int port, StormMetricRegistry metricRegistry, boolean dynamicBatch) { - this(queueName, metricNamePrefix, size, overflowLimit, producerBatchSz, backPressureWaitStrategy, topologyId, componentId, + this(queueName, metricNamePrefix, size, overflowLimit, producerBatchSz, + backPressureWaitStrategy, topologyId, componentId, taskIds, port, metricRegistry, dynamicBatch, 0); } - public JCQueue(String queueName, String metricNamePrefix, int size, int overflowLimit, int producerBatchSz, + public JCQueue(String queueName, String metricNamePrefix, int size, int overflowLimit, + int producerBatchSz, IWaitStrategy backPressureWaitStrategy, String topologyId, String componentId, List taskIds, int port, StormMetricRegistry metricRegistry, boolean dynamicBatch, int controlQueueSize) { this.queueName = queueName; @@ -82,11 +92,13 @@ public JCQueue(String queueName, String metricNamePrefix, int size, int overflow this.controlQueue = (controlQueueSize > 0) ? new MpscArrayQueue<>(controlQueueSize) : null; for (Integer taskId : taskIds) { - this.jcqMetrics.add(new JCQueueMetrics(metricNamePrefix, topologyId, componentId, taskId, port, + this.jcqMetrics.add(new JCQueueMetrics(metricNamePrefix, topologyId, componentId, + taskId, port, metricRegistry, recvQueue, overflowQ, controlQueue)); } - //The batch size can be no larger than half the full recvQueue size, to avoid contention issues. + // The batch size can be no larger than half the full recvQueue size, to avoid contention + // issues. this.producerBatchSz = Math.max(1, Math.min(producerBatchSz, size / 2)); this.backPressureWaitStrategy = backPressureWaitStrategy; } @@ -110,7 +122,8 @@ public int consume(JCQueue.Consumer consumer) { } /** - * Non blocking. Returns immediately if Q is empty. Runs till Q is empty OR exitCond.keepRunning() return false. Returns number of + * Non blocking. Returns immediately if Q is empty. Runs till Q is empty OR + * exitCond.keepRunning() return false. Returns number of * elements consumed from Q. */ public int consume(JCQueue.Consumer consumer, ExitCondition exitCond) { @@ -127,7 +140,8 @@ public int size() { } /** - * Load of the data plane only. The control lane and overflow lane are deliberately excluded so that load-aware groupings + * Load of the data plane only. The control lane and overflow lane are deliberately excluded so + * that load-aware groupings * (e.g. LoadAwareShuffleGrouping) are unaffected by pending control tuples. */ public double getQueueLoad() { @@ -140,7 +154,8 @@ public double getQueueLoad() { private int consumeImpl(Consumer consumer, ExitCondition exitCond) throws InterruptedException { int controlDrainCount = 0; if (controlQueue != null) { - // drain the control lane first: it is small and bounded, so it cannot starve the data plane + // drain the control lane first: it is small and bounded, so it cannot starve the data + // plane while (exitCond.keepRunning()) { Object tuple = controlQueue.poll(); if (tuple == null) { @@ -163,7 +178,8 @@ private int consumeImpl(Consumer consumer, ExitCondition exitCond) throws Interr int overflowDrainCount = 0; int limit = overflowQ.size(); - while (exitCond.keepRunning() && (overflowDrainCount < limit)) { // 2nd cond prevents staying stuck with consuming overflow + while (exitCond.keepRunning() + && (overflowDrainCount < limit)) { // 2nd cond prevents staying stuck with consuming overflow Object tuple = overflowQ.poll(); ++overflowDrainCount; consumer.accept(tuple); @@ -229,7 +245,8 @@ private Inserter getInserter() { } /** - * Blocking call. Retries till it can successfully publish the obj. Can be interrupted via Thread.interrupt(). + * Blocking call. Retries till it can successfully publish the obj. Can be interrupted via + * Thread.interrupt(). */ public void publish(Object obj) throws InterruptedException { Inserter inserter = getInserter(); @@ -245,7 +262,8 @@ public boolean tryPublish(Object obj) { } /** - * Non-blocking call. Bypasses any batching that may be enabled on the recvQueue. Intended for sending flush/metrics tuples + * Non-blocking call. Bypasses any batching that may be enabled on the recvQueue. Intended for + * sending flush/metrics tuples */ public boolean tryPublishDirect(Object obj) { return tryPublishInternal(obj); @@ -259,10 +277,14 @@ public boolean isControlLaneEnabled() { } /** - * Non-blocking, un-batched write to the control lane, exempt from backpressure. Falls back to an un-batched - * write to the recvQueue (same as {@link #tryPublishDirect(Object)}) when the control lane is disabled. - * Returns false if the target queue is full; a failed control publish is dropped by the caller and counted - * via the control drop metric, which is safe because control signals are periodic and self-healing. + * Non-blocking, un-batched write to the control lane, exempt from backpressure. Falls back to + * an un-batched + * write to the recvQueue (same as {@link #tryPublishDirect(Object)}) when the control lane is + * disabled. + * Returns false if the target queue is full; a failed control publish is dropped by the caller + * and counted + * via the control drop metric, which is safe because control signals are periodic and + * self-healing. */ public boolean tryPublishControl(Object obj) { if (controlQueue == null) { @@ -278,24 +300,33 @@ public boolean tryPublishControl(Object obj) { return false; } - // A full control lane means the consuming executor thread is stalled; surface it in the logs (rate-limited) and + // A full control lane means the consuming executor thread is stalled; surface it in the logs + // (rate-limited) and // not only via the control_dropped_messages metric, which an operator may not be charting. private void maybeWarnControlDrop() { - // Reads the clock on every invocation, but this is not a hot path: it runs only when a control tuple is - // dropped, and control tuples reach tryPublishControl only via the isControlStreamId guard (timer-driven - // tick/flush/metrics-tick/feedback signals). So drops are bounded by those signal periods (order of a few - // per second per queue even under a full stall), not by the data rate. A time-based throttle also has to - // read the clock to know the interval elapsed; on Linux nanoTime() is a vDSO read (no syscall/context switch). + // Reads the clock on every invocation, but this is not a hot path: it runs only when a + // control tuple is + // dropped, and control tuples reach tryPublishControl only via the isControlStreamId guard + // (timer-driven + // tick/flush/metrics-tick/feedback signals). So drops are bounded by those signal periods + // (order of a few + // per second per queue even under a full stall), not by the data rate. A time-based + // throttle also has to + // read the clock to know the interval elapsed; on Linux nanoTime() is a vDSO read (no + // syscall/context switch). long now = System.nanoTime(); long last = lastControlDropLogNanos.get(); - if (now - last >= CONTROL_DROP_LOG_INTERVAL_NANOS && lastControlDropLogNanos.compareAndSet(last, now)) { - LOG.warn("Control lane full on queue '{}'; dropping control tuple. The consuming executor thread is likely " + if (now - last >= CONTROL_DROP_LOG_INTERVAL_NANOS && lastControlDropLogNanos + .compareAndSet(last, now)) { + LOG.warn("Control lane full on queue '{}'; dropping control tuple. The consuming " + + "executor thread is likely " + "stalled. See the receive-queue-control_dropped_messages metric.", queueName); } } /** - * Un-batched write to overflowQ. Should only be called by WorkerTransfer returns false if overflowLimit has reached + * Un-batched write to overflowQ. Should only be called by WorkerTransfer returns false if + * overflowLimit has reached */ public boolean tryPublishToOverflow(Object obj) { if (overflowLimit > 0 && overflowQ.size() >= overflowLimit) { @@ -324,7 +355,8 @@ public int getQueuedCount() { } /** - * if(batchSz>1) : Blocking call. Does not return until at least 1 element is drained or Thread.interrupt() is received if(batchSz==1) + * if(batchSz>1) : Blocking call. Does not return until at least 1 element is drained or + * Thread.interrupt() is received if(batchSz==1) * : NO-OP. Returns immediately. doesnt throw. */ public void flush() throws InterruptedException { @@ -333,7 +365,8 @@ public void flush() throws InterruptedException { } /** - * if(batchSz>1) : Non-Blocking call. Tries to flush as many as it can. Returns true if flushed at least 1. if(batchSz==1) : This is a + * if(batchSz>1) : Non-Blocking call. Tries to flush as many as it can. Returns true if flushed + * at least 1. if(batchSz==1) : This is a * NO-OP. Returns true immediately. */ public boolean tryFlush() { @@ -372,7 +405,7 @@ private static class DirectInserter implements Inserter { } /** - * Blocking call, that can be interrupted via Thread.interrupt + * Blocking call, that can be interrupted via Thread.interrupt. */ @Override public void publish(Object obj) throws InterruptedException { @@ -383,7 +416,8 @@ public void publish(Object obj) throws InterruptedException { jcQueueMetric.notifyInsertFailure(); } if (idleCount == 0) { // check avoids multiple log msgs when in a idle loop - LOG.debug("Experiencing Back Pressure on recvQueue: '{}'. Entering BackPressure Wait", + LOG.debug("Experiencing Back Pressure on recvQueue: '{}'. Entering " + + "BackPressure Wait", queue.getQueueName()); } @@ -424,7 +458,8 @@ public boolean tryFlush() { /* Not thread safe. Have one instance per producer thread or synchronize externally */ private static class BatchInserter implements Inserter { private final int batchSz; - // WeakReference breaks the ThreadLocal retention cycle: thdLocalBatcher is an instance field + // WeakReference breaks the ThreadLocal retention cycle: thdLocalBatcher is an instance + // field // of JCQueue, so the ThreadLocalMap key (the ThreadLocal object) is kept strongly reachable // via value(BatchInserter) -> queue(JCQueue) -> field. A WeakReference here cuts that path, // allowing the key to become weakly-reachable and the entry to be expunged once the JCQueue @@ -439,14 +474,16 @@ private static class BatchInserter implements Inserter { } /** - * Number of buffered elements that triggers a flush. Constant here; subclasses may vary it at runtime. + * Number of buffered elements that triggers a flush. Constant here; subclasses may vary it + * at runtime. */ int batchSize() { return batchSz; } /** - * Hook invoked after every non-empty flush, passing whether the batch had reached {@link #batchSize()} when the flush started. + * Hook invoked after every non-empty flush, passing whether the batch had reached {@link + * #batchSize()} when the flush started. * No-op here; subclasses may use it to adapt {@link #batchSize()}. */ void afterFlush(boolean wasFull) { @@ -478,7 +515,8 @@ public boolean tryPublish(Object obj) { } /** - * Blocking call - Does not return until at least 1 element is drained or Thread.interrupt() is received. Uses backpressure wait + * Blocking call - Does not return until at least 1 element is drained or Thread.interrupt() + * is received. Uses backpressure wait * strategy. */ @Override @@ -488,7 +526,8 @@ public void flush() throws InterruptedException { } JCQueue queue = queueRef.get(); if (queue == null) { - // The JCQueue was GC'd (topology stopped on a long-lived thread, e.g. LocalCluster). + // The JCQueue was GC'd (topology stopped on a long-lived thread, e.g. + // LocalCluster). // Nothing to flush; discard the buffered batch and return cleanly. currentBatch.clear(); return; @@ -501,7 +540,8 @@ public void flush() throws InterruptedException { jcQueueMetric.notifyInsertFailure(); } if (retryCount == 0) { // check avoids multiple log msgs when in a idle loop - LOG.debug("Experiencing Back Pressure when flushing batch to Q: '{}'. Entering BackPressure Wait.", + LOG.debug("Experiencing Back Pressure when flushing batch to Q: '{}'. " + + "Entering BackPressure Wait.", queue.getQueueName()); } retryCount = queue.backPressureWaitStrategy.idle(retryCount); @@ -515,7 +555,8 @@ public void flush() throws InterruptedException { } /** - * Non blocking call. tries to flush as many as possible. Returns true if at least one from non-empty currentBatch was flushed or if + * Non blocking call. tries to flush as many as possible. Returns true if at least one from + * non-empty currentBatch was flushed or if * currentBatch is empty. Returns false otherwise */ @Override @@ -525,7 +566,8 @@ public boolean tryFlush() { } JCQueue queue = queueRef.get(); if (queue == null) { - // The JCQueue was GC'd (topology stopped on a long-lived thread, e.g. LocalCluster). + // The JCQueue was GC'd (topology stopped on a long-lived thread, e.g. + // LocalCluster). // Nothing to flush; discard the buffered batch and report success. currentBatch.clear(); return true; @@ -536,10 +578,14 @@ public boolean tryFlush() { for (JCQueueMetrics jcQueueMetric : queue.jcqMetrics) { jcQueueMetric.notifyInsertFailure(); } - // afterFlush is invoked intentionally even though nothing was published: a full batch that the recvQueue - // could not accept is a heavy-load signal, so subclasses grow the effective batch size (see - // DynamicBatchInserter#afterFlush). This matches the blocking flush(), which also grows on wasFull after its - // retry loop drains the queue. Do not move this out of the failure branch without revisiting that symmetry. + // afterFlush is invoked intentionally even though nothing was published: a full + // batch that the recvQueue + // could not accept is a heavy-load signal, so subclasses grow the effective batch + // size (see + // DynamicBatchInserter#afterFlush). This matches the blocking flush(), which also + // grows on wasFull after its + // retry loop drains the queue. Do not move this out of the failure branch without + // revisiting that symmetry. afterFlush(wasFull); return false; } else { @@ -551,11 +597,16 @@ public boolean tryFlush() { } // class BatchInserter /** - * A {@link BatchInserter} that adapts its batch size between 1 and a configured maximum using AIMD, to favor low latency under - * light load while preserving throughput under heavy load. It reuses the parent's publish/flush logic and only customizes the - * flush threshold ({@link #batchSize()}) and the post-flush adaptation ({@link #afterFlush(boolean)}): a flush of a full batch - * is read as heavy load and additively grows the effective size; a flush of a partially-filled batch (e.g. driven by the - * flush-tuple timer) is read as light load and multiplicatively shrinks it toward 1. Not thread safe. Have one instance per + * A {@link BatchInserter} that adapts its batch size between 1 and a configured maximum using + * AIMD, to favor low latency under + * light load while preserving throughput under heavy load. It reuses the parent's publish/flush + * logic and only customizes the + * flush threshold ({@link #batchSize()}) and the post-flush adaptation ({@link + * #afterFlush(boolean)}): a flush of a full batch + * is read as heavy load and additively grows the effective size; a flush of a partially-filled + * batch (e.g. driven by the + * flush-tuple timer) is read as light load and multiplicatively shrinks it toward 1. Not thread + * safe. Have one instance per * producer thread or synchronize externally. */ static class DynamicBatchInserter extends BatchInserter { @@ -563,7 +614,8 @@ static class DynamicBatchInserter extends BatchInserter { private int effectiveBatchSz; DynamicBatchInserter(JCQueue queue, int maxBatchSz) { - super(queue, maxBatchSz); // sizes the buffer to the max; the flush threshold comes from batchSize() + super(queue, + maxBatchSz); // sizes the buffer to the max; the flush threshold comes from batchSize() this.maxBatchSz = maxBatchSz; this.effectiveBatchSz = 1; // start small to favor latency; grows under sustained load } diff --git a/storm-client/src/jvm/org/apache/storm/utils/JCQueueMetrics.java b/storm-client/src/jvm/org/apache/storm/utils/JCQueueMetrics.java index 771e13a1b54..02483da700f 100644 --- a/storm-client/src/jvm/org/apache/storm/utils/JCQueueMetrics.java +++ b/storm-client/src/jvm/org/apache/storm/utils/JCQueueMetrics.java @@ -37,7 +37,8 @@ public class JCQueueMetrics implements Closeable { * Creates and registers the queue gauges. {@code controlQ} may be null when the queue has no control lane; * the control-lane gauges are then not registered. */ - public JCQueueMetrics(String metricNamePrefix, String topologyId, String componentId, int taskId, int port, + public JCQueueMetrics(String metricNamePrefix, String topologyId, String componentId, + int taskId, int port, StormMetricRegistry metricRegistry, MpscArrayQueue receiveQ, MpscUnboundedArrayQueue overflowQ, MpscArrayQueue controlQ) { @@ -72,8 +73,10 @@ public Double getValue() { Gauge sojourn = new Gauge() { @Override public Double getValue() { - // Assume the recvQueue is stable, in which the arrival rate is equal to the consumption rate. - // If this assumption does not hold, the calculation of sojourn time should also consider + // Assume the recvQueue is stable, in which the arrival rate is equal to the + // consumption rate. + // If this assumption does not hold, the calculation of sojourn time should also + // consider // departure rate according to Queuing Theory. return receiveQ.size() / Math.max(arrivalsTracker.reportRate(), 0.00001) * 1000.0; } @@ -100,21 +103,31 @@ public Integer getValue() { } }; - metricRegistry.gauge(metricNamePrefix + "-capacity", cap, topologyId, componentId, taskId, port); - metricRegistry.gauge(metricNamePrefix + "-pct_full", pctFull, topologyId, componentId, taskId, port); - metricRegistry.gauge(metricNamePrefix + "-population", pop, topologyId, componentId, taskId, port); - metricRegistry.gauge(metricNamePrefix + "-arrival_rate_secs", arrivalRate, topologyId, componentId, taskId, port); - metricRegistry.gauge(metricNamePrefix + "-sojourn_time_ms", sojourn, topologyId, componentId, taskId, port); - metricRegistry.gauge(metricNamePrefix + "-insert_failures", insertFailures, topologyId, componentId, taskId, port); - metricRegistry.gauge(metricNamePrefix + "-dropped_messages", dropped, topologyId, componentId, taskId, port); - metricRegistry.gauge(metricNamePrefix + "-overflow", overflow, topologyId, componentId, taskId, port); + metricRegistry.gauge(metricNamePrefix + "-capacity", cap, topologyId, componentId, taskId, + port); + metricRegistry.gauge(metricNamePrefix + "-pct_full", pctFull, topologyId, componentId, + taskId, port); + metricRegistry.gauge(metricNamePrefix + "-population", pop, topologyId, componentId, taskId, + port); + metricRegistry.gauge(metricNamePrefix + "-arrival_rate_secs", arrivalRate, topologyId, + componentId, taskId, port); + metricRegistry.gauge(metricNamePrefix + "-sojourn_time_ms", sojourn, topologyId, + componentId, taskId, port); + metricRegistry.gauge(metricNamePrefix + "-insert_failures", insertFailures, topologyId, + componentId, taskId, port); + metricRegistry.gauge(metricNamePrefix + "-dropped_messages", dropped, topologyId, + componentId, taskId, port); + metricRegistry.gauge(metricNamePrefix + "-overflow", overflow, topologyId, componentId, + taskId, port); if (controlQ != null) { Gauge controlPop = controlQ::size; Gauge controlDropped = droppedControlMessages::get; - metricRegistry.gauge(metricNamePrefix + "-control_population", controlPop, topologyId, componentId, taskId, port); - metricRegistry.gauge(metricNamePrefix + "-control_dropped_messages", controlDropped, topologyId, componentId, taskId, port); + metricRegistry.gauge(metricNamePrefix + "-control_population", controlPop, topologyId, + componentId, taskId, port); + metricRegistry.gauge(metricNamePrefix + "-control_dropped_messages", controlDropped, + topologyId, componentId, taskId, port); } } diff --git a/storm-client/src/jvm/org/apache/storm/utils/KeyStreamRandom.java b/storm-client/src/jvm/org/apache/storm/utils/KeyStreamRandom.java index a8c7903f5ff..dfbc71ff43b 100644 --- a/storm-client/src/jvm/org/apache/storm/utils/KeyStreamRandom.java +++ b/storm-client/src/jvm/org/apache/storm/utils/KeyStreamRandom.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -22,14 +28,20 @@ import javax.crypto.spec.SecretKeySpec; /** - * A {@link Random} that returns slices of an AES counter mode key stream, keyed from {@link SecureRandom} when the instance is - * created and buffered a block at a time. It is meant for values that other parties must not be able to guess, such as the tuple - * tree ids handed out by {@link org.apache.storm.tuple.MessageId#generateId(Random)}: unlike {@link Random}, - * {@link java.util.SplittableRandom} and {@link java.util.concurrent.ThreadLocalRandom}, whose internal state is recoverable from + * A {@link Random} that returns slices of an AES counter mode key stream, keyed from {@link + * SecureRandom} when the instance is + * created and buffered a block at a time. It is meant for values that other parties must not be + * able to guess, such as the tuple + * tree ids handed out by {@link org.apache.storm.tuple.MessageId#generateId(Random)}: unlike {@link + * Random}, + * {@link java.util.SplittableRandom} and {@link java.util.concurrent.ThreadLocalRandom}, whose + * internal state is recoverable from * a couple of returned values, the values returned here say nothing about the values returned next. * - *

    The buffering keeps it cheap enough for the emit path; it is measurably faster than {@link Random}, whose seed update is a - * contended compare and set per value. It is not a drop in replacement for a seeded {@link Random} though, because it cannot be + *

    The buffering keeps it cheap enough for the emit path; it is measurably faster than {@link + * Random}, whose seed update is a + * contended compare and set per value. It is not a drop in replacement for a seeded {@link Random} + * though, because it cannot be * reseeded and so cannot produce a repeatable sequence.

    */ public class KeyStreamRandom extends Random { @@ -65,7 +77,8 @@ private static Cipher newCipher(SecureRandom source) { source.nextBytes(iv); try { Cipher cipher = Cipher.getInstance(TRANSFORMATION); - cipher.init(Cipher.ENCRYPT_MODE, new SecretKeySpec(key, "AES"), new IvParameterSpec(iv)); + cipher.init(Cipher.ENCRYPT_MODE, new SecretKeySpec(key, "AES"), + new IvParameterSpec(iv)); return cipher; } catch (GeneralSecurityException e) { // no AES here, fall back to taking the values from the SecureRandom itself @@ -88,7 +101,8 @@ protected int next(int bits) { @Override public void setSeed(long seed) { - // there is nothing to seed, and Random's constructor calls this before this class' fields exist + // there is nothing to seed, and Random's constructor calls this before this class' fields + // exist } private void fill() { diff --git a/storm-client/src/jvm/org/apache/storm/utils/KeyedRoundRobinQueue.java b/storm-client/src/jvm/org/apache/storm/utils/KeyedRoundRobinQueue.java index 9836a921b33..fc4755810d8 100644 --- a/storm-client/src/jvm/org/apache/storm/utils/KeyedRoundRobinQueue.java +++ b/storm-client/src/jvm/org/apache/storm/utils/KeyedRoundRobinQueue.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/utils/ListDelegate.java b/storm-client/src/jvm/org/apache/storm/utils/ListDelegate.java index dea4559788a..bc5f7f311d8 100644 --- a/storm-client/src/jvm/org/apache/storm/utils/ListDelegate.java +++ b/storm-client/src/jvm/org/apache/storm/utils/ListDelegate.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/utils/LocalState.java b/storm-client/src/jvm/org/apache/storm/utils/LocalState.java index 019cf7534e1..0eacce120c1 100644 --- a/storm-client/src/jvm/org/apache/storm/utils/LocalState.java +++ b/storm-client/src/jvm/org/apache/storm/utils/LocalState.java @@ -1,12 +1,17 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -38,7 +43,8 @@ import org.slf4j.LoggerFactory; /** - * A simple, durable, atomic K/V database. *Very inefficient*, should only be used for occasional reads/writes. Every read/write hits disk. + * A simple, durable, atomic K/V database. *Very inefficient*, should only be used for occasional + * reads/writes. Every read/write hits disk. */ public class LocalState { public static final Logger LOG = LoggerFactory.getLogger(LocalState.class); @@ -72,7 +78,8 @@ private Map deserializeLatestVersion() throws IOException { try { Map result = new HashMap<>(); TDeserializer td = new TDeserializer(); - for (Map.Entry ent : partialDeserializeLatestVersion(td).entrySet()) { + for (Map.Entry ent : partialDeserializeLatestVersion(td) + .entrySet()) { result.put(ent.getKey(), deserialize(ent.getValue(), td)); } return result; @@ -87,8 +94,9 @@ private TBase deserialize(ThriftSerializedObject obj, TDeserializer td) { try { clazz = Class.forName(obj.get_name()); } catch (ClassNotFoundException ex) { - //Try to maintain rolling upgrade compatible with 0.10 releases - clazz = Class.forName(obj.get_name().replaceAll("^backtype\\.storm\\.", "org.apache.storm.")); + // Try to maintain rolling upgrade compatible with 0.10 releases + clazz = Class.forName(obj.get_name().replaceAll("^backtype\\.storm\\.", + "org.apache.storm.")); } TBase instance = (TBase) clazz.newInstance(); td.deserialize(instance, obj.get_bits()); @@ -261,7 +269,8 @@ public void setLocalAssignmentsMap(Map localAssignment put(LS_LOCAL_ASSIGNMENTS, new LSSupervisorAssignments(localAssignmentMap)); } - private void persistInternal(Map serialized, TSerializer ser, boolean cleanup) { + private void persistInternal(Map serialized, TSerializer ser, + boolean cleanup) { try { if (ser == null) { ser = new TSerializer(); @@ -287,7 +296,8 @@ private void persistInternal(Map serialized, TSe private ThriftSerializedObject serialize(TBase o, TSerializer ser) { try { - return new ThriftSerializedObject(o.getClass().getName(), ByteBuffer.wrap(ser.serialize(o))); + return new ThriftSerializedObject(o.getClass().getName(), ByteBuffer.wrap(ser + .serialize(o))); } catch (Exception e) { throw new RuntimeException(e); } diff --git a/storm-client/src/jvm/org/apache/storm/utils/MutableInt.java b/storm-client/src/jvm/org/apache/storm/utils/MutableInt.java index 681e7d92d07..61443b1c8f2 100644 --- a/storm-client/src/jvm/org/apache/storm/utils/MutableInt.java +++ b/storm-client/src/jvm/org/apache/storm/utils/MutableInt.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/utils/MutableLong.java b/storm-client/src/jvm/org/apache/storm/utils/MutableLong.java index e2dd977af81..d4dfa0ce2d6 100644 --- a/storm-client/src/jvm/org/apache/storm/utils/MutableLong.java +++ b/storm-client/src/jvm/org/apache/storm/utils/MutableLong.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/utils/MutableObject.java b/storm-client/src/jvm/org/apache/storm/utils/MutableObject.java index 05be6e69853..bd820567e30 100644 --- a/storm-client/src/jvm/org/apache/storm/utils/MutableObject.java +++ b/storm-client/src/jvm/org/apache/storm/utils/MutableObject.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/utils/NimbusClient.java b/storm-client/src/jvm/org/apache/storm/utils/NimbusClient.java index 2b5e77628a0..1edaf1eed34 100644 --- a/storm-client/src/jvm/org/apache/storm/utils/NimbusClient.java +++ b/storm-client/src/jvm/org/apache/storm/utils/NimbusClient.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -70,7 +76,8 @@ public Builder forDaemon() { return this; } - public NimbusClient buildWithNimbusHostPort(String host, Integer port) throws TTransportException { + public NimbusClient buildWithNimbusHostPort(String host, + Integer port) throws TTransportException { return new NimbusClient(conf, host, port, timeout, asUser, false); } @@ -81,6 +88,7 @@ public NimbusClient build() { /** * Constructor. + * * @param conf the conf for the client. * @param host the host the client is to talk to. * @param port the port the client is to talk to. @@ -89,35 +97,42 @@ public NimbusClient build() { * @deprecated use {@link Builder #buildWithNimbusHostPort()} instead. */ @Deprecated - public NimbusClient(Map conf, String host, int port, Integer timeout) throws TTransportException { + public NimbusClient(Map conf, String host, int port, + Integer timeout) throws TTransportException { this(conf, host, port, timeout, null); } /** * Constructor. + * * @param conf the conf for the client. * @param host the host the client is to talk to. * @param port the port the client is to talk to. * @param timeout the timeout to use when connecting. - * @param asUser the name of the user you want to impersonate (use with caution as it is not always supported). + * @param asUser the name of the user you want to impersonate (use with caution as it is not + * always supported). * @throws TTransportException on any error. * @deprecated use {@link Builder #buildWithNimbusHostPort()} instead. */ @Deprecated - public NimbusClient(Map conf, String host, Integer port, Integer timeout, String asUser) throws TTransportException { + public NimbusClient(Map conf, String host, Integer port, Integer timeout, + String asUser) throws TTransportException { this(conf, host, port, timeout, asUser, false); } @Deprecated - public NimbusClient(Map conf, String host, Integer port, Integer timeout, String asUser, boolean useTls) + public NimbusClient(Map conf, String host, Integer port, Integer timeout, + String asUser, boolean useTls) throws TTransportException { - super(conf, useTls ? ThriftConnectionType.NIMBUS_TLS : ThriftConnectionType.NIMBUS, host, port, timeout, asUser); + super(conf, useTls ? ThriftConnectionType.NIMBUS_TLS : ThriftConnectionType.NIMBUS, host, + port, timeout, asUser); client = new Nimbus.Client(protocol); isLocal = false; } /** * Constructor. + * * @param conf the conf for the client. * @param host the host the client is to talk to. * @throws TTransportException on any error. @@ -136,7 +151,9 @@ private NimbusClient(Nimbus.Iface client) { /** * Is the local override set or not. - * @return true of new clients will be overridden to connect to a local cluster and not the configured remote cluster. + * + * @return true of new clients will be overridden to connect to a local cluster and not the + * configured remote cluster. */ public static boolean isLocalOverride() { return _localOverrideClient != null; @@ -144,6 +161,7 @@ public static boolean isLocalOverride() { /** * Execute cb with a configured nimbus client that will be closed once cb returns. + * * @param cb the callback to send to nimbus. * @throws Exception on any kind of error. */ @@ -153,11 +171,13 @@ public static void withConfiguredClient(WithNimbus cb) throws Exception { /** * Execute cb with a configured nimbus client that will be closed once cb returns. + * * @param cb the callback to send to nimbus. * @param conf the conf to use instead of reading the global storm conf. * @throws Exception on any kind of error. */ - public static void withConfiguredClient(WithNimbus cb, Map conf) throws Exception { + public static void withConfiguredClient(WithNimbus cb, Map conf) throws Exception { try (NimbusClient client = NimbusClient.Builder.withConf(conf).build()) { cb.run(client.getClient()); } @@ -165,6 +185,7 @@ public static void withConfiguredClient(WithNimbus cb, Map conf) /** * Get a nimbus client as configured by conf. + * * @param conf the configuration to use. * @return the client, don't forget to close it when done. * @deprecated use {@link Builder #build()} instead. @@ -176,6 +197,7 @@ public static NimbusClient getConfiguredClient(Map conf) { /** * Get a nimbus client as configured by conf. + * * @param conf the configuration to use. * @param timeout the timeout to use when connecting. * @return the client, don't forget to close it when done. @@ -187,8 +209,10 @@ public static NimbusClient getConfiguredClient(Map conf, Integer } /** - * Check to see if we should log the leader we are connecting to or not. This typically happens when the leader changes or if debug - * logging is enabled. The code remembers the last leader it was called with, but it should be transparent to the caller. + * Check to see if we should log the leader we are connecting to or not. This typically happens + * when the leader changes or if debug + * logging is enabled. The code remembers the last leader it was called with, but it should be + * transparent to the caller. * * @param leader the leader we are trying to connect to. * @return true if it should be logged else false. @@ -196,10 +220,10 @@ public static NimbusClient getConfiguredClient(Map conf, Integer private static synchronized boolean shouldLogLeader(String leader) { assert leader != null; if (LOG.isDebugEnabled()) { - //If debug logging is turned on we should just log the leader all the time.... + // If debug logging is turned on we should just log the leader all the time.... return true; } - //Only log if the leader has changed. It is not interesting otherwise. + // Only log if the leader has changed. It is not interesting otherwise. if (oldLeader.equals(leader)) { return false; } @@ -209,6 +233,7 @@ private static synchronized boolean shouldLogLeader(String leader) { /** * Get a nimbus client as configured by conf. + * * @param conf the configuration to use. * @param asUser the user to impersonate (this does not always work). * @return the client, don't forget to close it when done. @@ -221,6 +246,7 @@ public static NimbusClient getConfiguredClientAs(Map conf, Strin /** * Get a nimbus client as configured by conf. + * * @param conf the configuration to use. * @param asUser the user to impersonate (this does not always work). * @param timeout the timeout to use when connecting. @@ -228,11 +254,13 @@ public static NimbusClient getConfiguredClientAs(Map conf, Strin * @deprecated use {@link Builder #build()} instead. */ @Deprecated - public static NimbusClient getConfiguredClientAs(Map conf, String asUser, Integer timeout) { + public static NimbusClient getConfiguredClientAs(Map conf, String asUser, + Integer timeout) { return createNimbusClient(conf, asUser, timeout); } - private static NimbusClient createNimbusClient(Map conf, String asUser, Integer timeout) { + private static NimbusClient createNimbusClient(Map conf, String asUser, + Integer timeout) { Nimbus.Iface override = _localOverrideClient; if (override != null) { return new NimbusClient(override); @@ -243,14 +271,15 @@ private static NimbusClient createNimbusClient(Map conf, String conf = fullConf; if (conf.containsKey(Config.STORM_DO_AS_USER)) { if (asUser != null && !asUser.isEmpty()) { - LOG.warn("You have specified a doAsUser as param {} and a doAsParam as config, config will take precedence.", + LOG.warn("You have specified a doAsUser as param {} and a doAsParam as config, " + + "config will take precedence.", asUser, conf.get(Config.STORM_DO_AS_USER)); } asUser = (String) conf.get(Config.STORM_DO_AS_USER); } if (asUser == null || asUser.isEmpty()) { - //The user is not set so lets see what the request context is. + // The user is not set so lets see what the request context is. ReqContext context = ReqContext.context(); Principal principal = context.principal(); asUser = principal == null ? null : principal.getName(); @@ -259,8 +288,10 @@ private static NimbusClient createNimbusClient(Map conf, String List seeds = (List) conf.get(Config.NIMBUS_SEEDS); - boolean useTls = ObjectReader.getBoolean(conf.get(Config.NIMBUS_THRIFT_CLIENT_USE_TLS), false); - if (useTls && null == ObjectReader.getString(conf.get(Config.NIMBUS_THRIFT_TLS_TRANSPORT_PLUGIN))) { + boolean useTls = ObjectReader.getBoolean(conf.get(Config.NIMBUS_THRIFT_CLIENT_USE_TLS), + false); + if (useTls && null == ObjectReader.getString(conf + .get(Config.NIMBUS_THRIFT_TLS_TRANSPORT_PLUGIN))) { throw new RuntimeException(Config.NIMBUS_THRIFT_TLS_TRANSPORT_PLUGIN + " must be set to use a transport plugin that supports tls"); } @@ -276,21 +307,25 @@ private static NimbusClient createNimbusClient(Map conf, String client = new NimbusClient(conf, host, configuredPortToUse, timeout, asUser, useTls); nimbusSummary = client.getClient().getLeader(); if (nimbusSummary != null) { - String leaderNimbus = nimbusSummary.get_host() + ":" + nimbusSummary.get_port() + ":" + nimbusSummary.get_tlsPort(); + String leaderNimbus = nimbusSummary.get_host() + ":" + nimbusSummary.get_port() + + ":" + nimbusSummary.get_tlsPort(); if (shouldLogLeader(leaderNimbus)) { LOG.info("Found leader nimbus : {}", leaderNimbus); } - int nimbusPortFromSummary = useTls ? nimbusSummary.get_tlsPort() : nimbusSummary.get_port(); + int nimbusPortFromSummary = useTls ? nimbusSummary.get_tlsPort() : nimbusSummary + .get_port(); if (nimbusSummary.get_host().equals(host) && nimbusPortFromSummary == port) { NimbusClient ret = client; client = null; return ret; } try { - return new NimbusClient(conf, nimbusSummary.get_host(), nimbusPortFromSummary, timeout, asUser, useTls); + return new NimbusClient(conf, nimbusSummary.get_host(), + nimbusPortFromSummary, timeout, asUser, useTls); } catch (TTransportException e) { - throw new RuntimeException("Failed to create a nimbus client for the leader " + leaderNimbus, e); + throw new RuntimeException("Failed to create a nimbus client for the " + + "leader " + leaderNimbus, e); } } } catch (Exception e) { @@ -302,7 +337,8 @@ private static NimbusClient createNimbusClient(Map conf, String client.close(); } } - throw new NimbusLeaderNotFoundException("Could not find a nimbus leader, please try again after some time."); + throw new NimbusLeaderNotFoundException("Could not find a nimbus leader, please try " + + "again after some time."); } throw new NimbusLeaderNotFoundException( "Could not find leader nimbus from seed hosts " + seeds + ". " @@ -312,6 +348,7 @@ private static NimbusClient createNimbusClient(Map conf, String /** * Get the underlying thrift client. + * * @return the underlying thrift client. */ public Nimbus.Iface getClient() { @@ -324,6 +361,7 @@ public Nimbus.Iface getClient() { public interface WithNimbus { /** * Run what you need with the nimbus client. + * * @param client the client. * @throws Exception on any error. */ diff --git a/storm-client/src/jvm/org/apache/storm/utils/NimbusLeaderNotFoundException.java b/storm-client/src/jvm/org/apache/storm/utils/NimbusLeaderNotFoundException.java index e41d56d6c89..4a30cb88a53 100644 --- a/storm-client/src/jvm/org/apache/storm/utils/NimbusLeaderNotFoundException.java +++ b/storm-client/src/jvm/org/apache/storm/utils/NimbusLeaderNotFoundException.java @@ -19,7 +19,8 @@ package org.apache.storm.utils; /** - * This exception class is used to signify a problem with nimbus leader identification. It should not be used when connection failures + * This exception class is used to signify a problem with nimbus leader identification. It should + * not be used when connection failures * happen, but only when successful operations result in the absence of an identified leader. */ public class NimbusLeaderNotFoundException extends RuntimeException { diff --git a/storm-client/src/jvm/org/apache/storm/utils/ObjectReader.java b/storm-client/src/jvm/org/apache/storm/utils/ObjectReader.java index ac28be8cf30..eb3db12157e 100644 --- a/storm-client/src/jvm/org/apache/storm/utils/ObjectReader.java +++ b/storm-client/src/jvm/org/apache/storm/utils/ObjectReader.java @@ -26,6 +26,7 @@ public class ObjectReader { /** * Convert the input into a list of string; ignore null members. + * * @param o the input object * @return a list of string */ diff --git a/storm-client/src/jvm/org/apache/storm/utils/ReflectionUtils.java b/storm-client/src/jvm/org/apache/storm/utils/ReflectionUtils.java index 4e41dda5552..9d6c28593c4 100644 --- a/storm-client/src/jvm/org/apache/storm/utils/ReflectionUtils.java +++ b/storm-client/src/jvm/org/apache/storm/utils/ReflectionUtils.java @@ -26,11 +26,15 @@ public class ReflectionUtils { /** - * The scheduler strategies shipped with Storm. They are used when {@link Config#NIMBUS_SCHEDULER_STRATEGY_CLASS_WHITELIST} is not - * set at all, so that an unset config does not mean that any class on the nimbus classpath may be instantiated. The names are - * spelled out because the strategies live in storm-server. Keep this list in sync with conf/defaults.yaml. + * The scheduler strategies shipped with Storm. They are used when {@link + * Config#NIMBUS_SCHEDULER_STRATEGY_CLASS_WHITELIST} is not + * set at all, so that an unset config does not mean that any class on the nimbus classpath may + * be instantiated. The names are + * spelled out because the strategies live in storm-server. Keep this list in sync with + * conf/defaults.yaml. */ - public static final List DEFAULT_SCHEDULER_STRATEGIES = Collections.unmodifiableList(Arrays.asList( + public static final List DEFAULT_SCHEDULER_STRATEGIES = Collections + .unmodifiableList(Arrays.asList( "org.apache.storm.scheduler.resource.strategies.scheduling.DefaultResourceAwareStrategy", "org.apache.storm.scheduler.resource.strategies.scheduling.DefaultResourceAwareStrategyOld", "org.apache.storm.scheduler.resource.strategies.scheduling.GenericResourceAwareStrategy", @@ -43,7 +47,8 @@ public class ReflectionUtils { private static ReflectionUtils _instance = new ReflectionUtils(); /** - * Provide an instance of this class for delegates to use. To mock out delegated methods, provide an instance of a subclass that + * Provide an instance of this class for delegates to use. To mock out delegated methods, + * provide an instance of a subclass that * overrides the implementation of the delegated method. * * @param u a Utils instance @@ -90,7 +95,8 @@ public static T newInstance(Class klass, Map conf) { } public static T newSchedulerStrategyInstance(String klass, Map conf) { - List allowedSchedulerStrategies = (List) conf.get(Config.NIMBUS_SCHEDULER_STRATEGY_CLASS_WHITELIST); + List allowedSchedulerStrategies = (List) conf + .get(Config.NIMBUS_SCHEDULER_STRATEGY_CLASS_WHITELIST); if (allowedSchedulerStrategies == null) { allowedSchedulerStrategies = DEFAULT_SCHEDULER_STRATEGIES; } diff --git a/storm-client/src/jvm/org/apache/storm/utils/RegisteredGlobalState.java b/storm-client/src/jvm/org/apache/storm/utils/RegisteredGlobalState.java index 19a48d973f7..45f2e2b9144 100644 --- a/storm-client/src/jvm/org/apache/storm/utils/RegisteredGlobalState.java +++ b/storm-client/src/jvm/org/apache/storm/utils/RegisteredGlobalState.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -16,8 +22,10 @@ import java.util.UUID; /** - * This class is used as part of testing Storm. It is used to keep track of "global metrics" in an atomic way. For example, it is used for - * doing fine-grained detection of when a local Storm cluster is idle by tracking the number of transferred tuples vs the number of + * This class is used as part of testing Storm. It is used to keep track of "global metrics" in an + * atomic way. For example, it is used for + * doing fine-grained detection of when a local Storm cluster is idle by tracking the number of + * transferred tuples vs the number of * processed tuples. */ public class RegisteredGlobalState { diff --git a/storm-client/src/jvm/org/apache/storm/utils/RotatingMap.java b/storm-client/src/jvm/org/apache/storm/utils/RotatingMap.java index 6283b508bfe..b126dcacd37 100644 --- a/storm-client/src/jvm/org/apache/storm/utils/RotatingMap.java +++ b/storm-client/src/jvm/org/apache/storm/utils/RotatingMap.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -15,22 +21,25 @@ import java.util.HashMap; import java.util.Iterator; import java.util.LinkedList; -import java.util.Map; import java.util.Map.Entry; +import java.util.Map; /** - * Expires keys that have not been updated in the configured number of seconds. The algorithm used will take between expirationSecs and + * Expires keys that have not been updated in the configured number of seconds. The algorithm used + * will take between expirationSecs and * expirationSecs * (1 + 1 / (numBuckets-1)) to actually expire the message. * *

    get, put, remove, containsKey, and size take O(numBuckets) time to run. * - *

    The advantage of this design is that the expiration thread only locks the object for O(1) time, meaning the object + *

    The advantage of this design is that the expiration thread only locks the object for O(1) + * time, meaning the object * is essentially always available for gets/puts. * - *

    Note: This class is not thread-safe since it does not protect against changes to buckets while it is being read + *

    Note: This class is not thread-safe since it does not protect against changes to buckets while + * it is being read */ public class RotatingMap { - //this default ensures things expire at most 50% past the expiration time + // this default ensures things expire at most 50% past the expiration time private static final int DEFAULT_NUM_BUCKETS = 3; private final LinkedList> buckets; private final ExpiredCallback callback; diff --git a/storm-client/src/jvm/org/apache/storm/utils/SecurityUtils.java b/storm-client/src/jvm/org/apache/storm/utils/SecurityUtils.java index b7e8628257a..352527127dd 100644 --- a/storm-client/src/jvm/org/apache/storm/utils/SecurityUtils.java +++ b/storm-client/src/jvm/org/apache/storm/utils/SecurityUtils.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -16,7 +22,8 @@ public class SecurityUtils { /** * Check if the suffix ends in pkcs12/p12/jks (case insensitive) and return PKCS12 ot JKS. - * If not, then return null. The path can be url to a resource since only the ending is compared. + * If not, then return null. The path can be url to a resource since only the ending is + * compared. * * @param path to the key resource file - can be embedded resource in a jar file. * @return PKCS12 or JKS or null diff --git a/storm-client/src/jvm/org/apache/storm/utils/ServiceRegistry.java b/storm-client/src/jvm/org/apache/storm/utils/ServiceRegistry.java index 01516974a9d..d79106a009a 100644 --- a/storm-client/src/jvm/org/apache/storm/utils/ServiceRegistry.java +++ b/storm-client/src/jvm/org/apache/storm/utils/ServiceRegistry.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/utils/ShellBoltMessageQueue.java b/storm-client/src/jvm/org/apache/storm/utils/ShellBoltMessageQueue.java index 071ef4e933b..df6f5724411 100644 --- a/storm-client/src/jvm/org/apache/storm/utils/ShellBoltMessageQueue.java +++ b/storm-client/src/jvm/org/apache/storm/utils/ShellBoltMessageQueue.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -22,7 +28,8 @@ import org.apache.storm.multilang.BoltMsg; /** - * A data structure for ShellBolt which includes two queues (FIFO), which one is for task ids (unbounded), another one is for bolt msg + * A data structure for ShellBolt which includes two queues (FIFO), which one is for task ids + * (unbounded), another one is for bolt msg * (bounded). */ public class ShellBoltMessageQueue implements Serializable { @@ -44,7 +51,7 @@ public ShellBoltMessageQueue() { } /** - * put list of task id to its queue. + * Put list of task id to its queue. * * @param taskIds task ids that received the tuples */ @@ -59,7 +66,7 @@ public void putTaskIds(List taskIds) { } /** - * put bolt message to its queue. + * Put bolt message to its queue. * * @param boltMsg BoltMsg to pass to subprocess */ @@ -74,8 +81,10 @@ public void putBoltMsg(BoltMsg boltMsg) throws InterruptedException { } /** - * poll() is a core feature of ShellBoltMessageQueue. It retrieves and removes the head of one queues, waiting up to the specified wait - * time if necessary for an element to become available. There's priority that what queue it retrieves first, taskIds is higher than + * poll() is a core feature of ShellBoltMessageQueue. It retrieves and removes the head of one + * queues, waiting up to the specified wait + * time if necessary for an element to become available. There's priority that what queue it + * retrieves first, taskIds is higher than * boltMsgQueue. * * @param timeout how long to wait before giving up, in units of unit diff --git a/storm-client/src/jvm/org/apache/storm/utils/ShellCommandRunner.java b/storm-client/src/jvm/org/apache/storm/utils/ShellCommandRunner.java index e1a58786cb0..5259d8f7fb0 100644 --- a/storm-client/src/jvm/org/apache/storm/utils/ShellCommandRunner.java +++ b/storm-client/src/jvm/org/apache/storm/utils/ShellCommandRunner.java @@ -20,13 +20,15 @@ import java.util.Map; /** - * Contains convenience functions for running shell commands for cases that are too simple to need a full {@link ShellUtils} + * Contains convenience functions for running shell commands for cases that are too simple to need a + * full {@link ShellUtils} * implementation. */ public interface ShellCommandRunner { /** - * Method to execute a shell command. Covers most of the simple cases without requiring the user to implement the {@link ShellUtils} + * Method to execute a shell command. Covers most of the simple cases without requiring the user + * to implement the {@link ShellUtils} * interface. * * @param cmd shell command to execute. @@ -35,7 +37,8 @@ public interface ShellCommandRunner { String execCommand(String... cmd) throws IOException; /** - * Method to execute a shell command. Covers most of the simple cases without requiring the user to implement the {@link ShellUtils} + * Method to execute a shell command. Covers most of the simple cases without requiring the user + * to implement the {@link ShellUtils} * interface. * * @param env the map of environment key=value @@ -48,7 +51,8 @@ String execCommand(Map env, String[] cmd, long timeout) throws IOException; /** - * Method to execute a shell command. Covers most of the simple cases without requiring the user to implement the {@link ShellUtils} + * Method to execute a shell command. Covers most of the simple cases without requiring the user + * to implement the {@link ShellUtils} * interface. * * @param env the map of environment key=value diff --git a/storm-client/src/jvm/org/apache/storm/utils/ShellCommandRunnerImpl.java b/storm-client/src/jvm/org/apache/storm/utils/ShellCommandRunnerImpl.java index e01a66d4767..070d36ffdb7 100644 --- a/storm-client/src/jvm/org/apache/storm/utils/ShellCommandRunnerImpl.java +++ b/storm-client/src/jvm/org/apache/storm/utils/ShellCommandRunnerImpl.java @@ -27,7 +27,8 @@ public String execCommand(String... cmd) throws IOException { } @Override - public String execCommand(Map env, String[] cmd, long timeout) throws IOException { + public String execCommand(Map env, String[] cmd, + long timeout) throws IOException { ShellUtils.ShellCommandExecutor exec = new ShellUtils.ShellCommandExecutor(cmd, null, env, timeout); exec.execute(); diff --git a/storm-client/src/jvm/org/apache/storm/utils/ShellLogHandler.java b/storm-client/src/jvm/org/apache/storm/utils/ShellLogHandler.java index 1440c4c072a..0c31dedcb7b 100644 --- a/storm-client/src/jvm/org/apache/storm/utils/ShellLogHandler.java +++ b/storm-client/src/jvm/org/apache/storm/utils/ShellLogHandler.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -21,7 +27,8 @@ public interface ShellLogHandler { /** - * Called at least once before {@link ShellLogHandler#log} for each spout and bolt. Allows implementing classes to save information + * Called at least once before {@link ShellLogHandler#log} for each spout and bolt. Allows + * implementing classes to save information * about the current running context e.g. pid, thread, task. * * @param ownerCls - the class which instantiated this ShellLogHandler. diff --git a/storm-client/src/jvm/org/apache/storm/utils/ShellProcess.java b/storm-client/src/jvm/org/apache/storm/utils/ShellProcess.java index 8cb98bf05d2..4583c4c1a54 100644 --- a/storm-client/src/jvm/org/apache/storm/utils/ShellProcess.java +++ b/storm-client/src/jvm/org/apache/storm/utils/ShellProcess.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -51,7 +57,8 @@ public void setEnv(Map env) { private void modifyEnvironment(Map buildEnv) { for (Map.Entry entry : env.entrySet()) { if ("PATH".equals(entry.getKey())) { - buildEnv.put("PATH", buildEnv.get("PATH") + File.pathSeparatorChar + env.get("PATH")); + buildEnv.put("PATH", buildEnv.get("PATH") + File.pathSeparatorChar + env + .get("PATH")); } else { buildEnv.put(entry.getKey(), entry.getValue()); } @@ -62,7 +69,8 @@ public Number launch(Map conf, TopologyContext context) { return launch(conf, context, true); } - public Number launch(Map conf, TopologyContext context, boolean changeDirectory) { + public Number launch(Map conf, TopologyContext context, + boolean changeDirectory) { ProcessBuilder builder = new ProcessBuilder(command); if (!env.isEmpty()) { Map buildEnv = builder.environment(); @@ -93,19 +101,20 @@ public Number launch(Map conf, TopologyContext context, boolean } private ISerializer getSerializer(Map conf) { - //get factory class name + // get factory class name String serializerClassName = (String) conf.get(Config.TOPOLOGY_MULTILANG_SERIALIZER); LOG.info("Storm multilang serializer: " + serializerClassName); ISerializer serializer; try { - //create a factory class + // create a factory class Class klass = Class.forName(serializerClassName); - //obtain a serializer object + // obtain a serializer object Object obj = klass.newInstance(); serializer = (ISerializer) obj; } catch (Exception e) { - throw new RuntimeException("Failed to construct multilang serializer from serializer " + serializerClassName, e); + throw new RuntimeException("Failed to construct multilang serializer from serializer " + + serializerClassName, e); } return serializer; } @@ -149,7 +158,7 @@ public void logErrorStream() { ShellLogger.info(new String(errorReadingBuffer)); } } catch (Exception e) { - //ignore + // ignore } } @@ -174,6 +183,7 @@ public String getErrorsString() { /** * Get PID. + * * @return pid, if the process has been launched, null otherwise. */ public Number getPid() { @@ -186,7 +196,9 @@ public String getComponentName() { /** * Get exit code. - * @return exit code of the process if process is terminated, -1 if process is not started or terminated. + * + * @return exit code of the process if process is terminated, -1 if process is not started or + * terminated. */ public int getExitCode() { try { diff --git a/storm-client/src/jvm/org/apache/storm/utils/ShellUtils.java b/storm-client/src/jvm/org/apache/storm/utils/ShellUtils.java index c1308d4b435..9884ba9b6a3 100644 --- a/storm-client/src/jvm/org/apache/storm/utils/ShellUtils.java +++ b/storm-client/src/jvm/org/apache/storm/utils/ShellUtils.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -38,7 +44,7 @@ public abstract class ShellUtils { public static final boolean LINUX = (osType == OSType.OS_TYPE_LINUX); public static final boolean OTHER = (osType == OSType.OS_TYPE_OTHER); - //Meter declared here can be registered by any daemon, and is currently used by Supervisor + // Meter declared here can be registered by any daemon, and is currently used by Supervisor public static final Meter numShellExceptions = new Meter(); /** @@ -76,6 +82,7 @@ public ShellUtils(long interval) { /** * Creates a new shell utils instance. + * * @param interval the minimum duration to wait before re-executing the command */ public ShellUtils(long interval, boolean redirectErrorStream) { @@ -104,13 +111,14 @@ private static OSType getOSType() { } /** - * a Unix command to get a given user's groups list. Windows is not supported. + * A Unix command to get a given user's groups list. Windows is not supported. */ public static String[] getGroupsForUserCommand(final String user) { if (WINDOWS) { - throw new UnsupportedOperationException("Getting user groups is not supported on Windows"); + throw new UnsupportedOperationException("Getting user groups is not supported on " + + "Windows"); } - //'groups username' command return is non-consistent across different unixes + // 'groups username' command return is non-consistent across different unixes return new String[]{"id", "-Gn", user}; } @@ -135,17 +143,19 @@ public static ShellLogHandler getLogHandler(Map topoConf) { String logHandlerClassName = null; if (topoConf.containsKey(Config.TOPOLOGY_MULTILANG_LOG_HANDLER)) { try { - logHandlerClassName = topoConf.get(Config.TOPOLOGY_MULTILANG_LOG_HANDLER).toString(); + logHandlerClassName = topoConf.get(Config.TOPOLOGY_MULTILANG_LOG_HANDLER) + .toString(); return (ShellLogHandler) Class.forName(logHandlerClassName).newInstance(); } catch (ClassCastException | InstantiationException | IllegalAccessException | ClassNotFoundException e) { - throw new RuntimeException("Error loading ShellLogHandler " + logHandlerClassName, e); + throw new RuntimeException("Error loading ShellLogHandler " + logHandlerClassName, + e); } } return new DefaultShellLogHandler(); } /** - * get the exit code. + * Get the exit code. * * @return the exit code of the process */ @@ -154,7 +164,7 @@ public int getExitCode() { } /** - * set the environment for the command. + * Set the environment for the command. * * @param env Mapping of environment variables */ @@ -163,7 +173,7 @@ protected void setEnvironment(Map env) { } /** - * set the working directory. + * Set the working directory. * * @param dir The directory where the command would be executed */ @@ -172,7 +182,7 @@ protected void setWorkingDirectory(File dir) { } /** - * check to see if a command needs to be executed and execute if needed. + * Check to see if a command needs to be executed and execute if needed. */ protected void run() throws IOException { if (lastTime + interval > System.currentTimeMillis()) { @@ -211,7 +221,7 @@ private void runCommand() throws IOException { if (timeOutInterval > 0) { timeOutTimer = new Timer("Shell command timeout"); timeoutTimerTask = new ShellTimeoutTimerTask(this); - //One time scheduling. + // One time scheduling. timeOutTimer.schedule(timeoutTimerTask, timeOutInterval); } final BufferedReader errReader = @@ -244,7 +254,7 @@ public void run() { try { errThread.start(); } catch (IllegalStateException ise) { - //ignore + // ignore } try { parseExecResult(inReader); // parse the output @@ -258,8 +268,8 @@ public void run() { // make sure that the error thread exits joinThread(errThread); completed.set(true); - //the timeout thread handling - //taken care in finally block + // the timeout thread handling + // taken care in finally block if (exitCode != 0) { throw new ExitCodeException(exitCode, errMsg.toString()); } @@ -303,7 +313,7 @@ public void run() { } /** - * return an array containing the command name & its parameters. + * Return an array containing the command name & its parameters. */ protected abstract String[] getExecString(); @@ -314,7 +324,7 @@ protected abstract void parseExecResult(BufferedReader lines) throws IOException; /** - * get the current sub-process executing the given command. + * Get the current sub-process executing the given command. * * @return process executing the command */ @@ -371,7 +381,8 @@ public int getExitCode() { * A simple shell command executor. * * ShellCommandExecutorshould be used in cases where the output - * of the command needs no explicit parsing and where the command, working directory and the environment remains unchanged. The output + * of the command needs no explicit parsing and where the command, working directory and the + * environment remains unchanged. The output * of the command is stored as-is and is expected to be small. */ public static class ShellCommandExecutor extends ShellUtils { @@ -379,7 +390,6 @@ public static class ShellCommandExecutor extends ShellUtils { private String[] command; private StringBuffer output; - public ShellCommandExecutor(String[] execString) { this(execString, null); } @@ -397,11 +407,14 @@ public ShellCommandExecutor(String[] execString, File dir, * Create a new instance of the ShellCommandExecutor to execute a command. * * @param execString The command to execute with arguments - * @param dir If not-null, specifies the directory which should be set as the current working directory for the command. If + * @param dir If not-null, specifies the directory which should be set as the current + * working directory for the command. If * null, the current working directory is not modified. - * @param env If not-null, environment of the command will include the key-value pairs specified in the map. If null, the + * @param env If not-null, environment of the command will include the key-value pairs + * specified in the map. If null, the * current environment is not modified. - * @param timeout Specifies the time in milliseconds, after which the command will be killed and the status marked as timedout. + * @param timeout Specifies the time in milliseconds, after which the command will be killed + * and the status marked as timedout. * If 0, the command will not be timed out. */ public ShellCommandExecutor(String[] execString, File dir, @@ -416,7 +429,6 @@ public ShellCommandExecutor(String[] execString, File dir, timeOutInterval = timeout; } - /** * Execute the shell command. */ @@ -447,7 +459,8 @@ public String getOutput() { } /** - * Returns the commands of this instance. Arguments with spaces in are presented with quotes round; other arguments are presented + * Returns the commands of this instance. Arguments with spaces in are presented with quotes + * round; other arguments are presented * raw * * @return a string representation of the object. @@ -485,9 +498,9 @@ public void run() { try { p.exitValue(); } catch (Exception e) { - //Process has not terminated. - //So check if it has completed - //if not just destroy it. + // Process has not terminated. + // So check if it has completed + // if not just destroy it. if (p != null && !shell.completed.get()) { shell.setTimedOut(); p.destroy(); diff --git a/storm-client/src/jvm/org/apache/storm/utils/SimpleVersion.java b/storm-client/src/jvm/org/apache/storm/utils/SimpleVersion.java index 3fe2729c315..03ca8000859 100644 --- a/storm-client/src/jvm/org/apache/storm/utils/SimpleVersion.java +++ b/storm-client/src/jvm/org/apache/storm/utils/SimpleVersion.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -16,7 +22,7 @@ import java.util.regex.Pattern; /** - * Take a version string and parse out a Major.Minor version + * Take a version string and parse out a Major.Minor version. */ public class SimpleVersion implements Comparable { private static final Pattern VERSION_PATTERN = Pattern.compile("^(\\d+)[\\.\\-\\_]+(\\d+).*$"); @@ -28,7 +34,7 @@ public SimpleVersion(String version) { int maj = -1; int min = -1; if (!m.matches()) { - //Unknown should only happen during compilation or some unit tests. + // Unknown should only happen during compilation or some unit tests. if (!"Unknown".equals(version)) { throw new IllegalArgumentException("Cannot parse '" + version + "'"); } diff --git a/storm-client/src/jvm/org/apache/storm/utils/StormBoundedExponentialBackoffRetry.java b/storm-client/src/jvm/org/apache/storm/utils/StormBoundedExponentialBackoffRetry.java index 1cb40c90a97..41eb9d62ccb 100644 --- a/storm-client/src/jvm/org/apache/storm/utils/StormBoundedExponentialBackoffRetry.java +++ b/storm-client/src/jvm/org/apache/storm/utils/StormBoundedExponentialBackoffRetry.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

    http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

    Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -18,20 +24,24 @@ import org.slf4j.LoggerFactory; public class StormBoundedExponentialBackoffRetry extends BoundedExponentialBackoffRetry { - private static final Logger LOG = LoggerFactory.getLogger(StormBoundedExponentialBackoffRetry.class); + private static final Logger LOG = LoggerFactory + .getLogger(StormBoundedExponentialBackoffRetry.class); private final Random random = new Random(); private final int linearBaseSleepMs; private final int stepSize; private int expRetriesThreshold; /** - * The class provides generic exponential-linear backoff retry strategy for storm. It calculates threshold for exponentially increasing + * The class provides generic exponential-linear backoff retry strategy for storm. It calculates + * threshold for exponentially increasing * sleeptime for retries. Beyond this threshold, the sleeptime increase is linear. * - *

    Also adds jitter for exponential/linear retry. It guarantees `currSleepTimeMs >= prevSleepTimeMs` and `baseSleepTimeMs <= + *

    Also adds jitter for exponential/linear retry. It guarantees `currSleepTimeMs >= + * prevSleepTimeMs` and `baseSleepTimeMs <= * currSleepTimeMs <= maxSleepTimeMs` */ - public StormBoundedExponentialBackoffRetry(int baseSleepTimeMs, int maxSleepTimeMs, int maxRetries) { + public StormBoundedExponentialBackoffRetry(int baseSleepTimeMs, int maxSleepTimeMs, + int maxRetries) { super(baseSleepTimeMs, maxSleepTimeMs, maxRetries); expRetriesThreshold = 1; while ((1 << (expRetriesThreshold + 1)) < ((maxSleepTimeMs - baseSleepTimeMs) / 2)) { @@ -40,11 +50,13 @@ public StormBoundedExponentialBackoffRetry(int baseSleepTimeMs, int maxSleepTime LOG.debug("The baseSleepTimeMs [{}] the maxSleepTimeMs [{}] the maxRetries [{}]", baseSleepTimeMs, maxSleepTimeMs, maxRetries); if (baseSleepTimeMs > maxSleepTimeMs) { - LOG.warn("Misconfiguration: the baseSleepTimeMs [" + baseSleepTimeMs + "] can't be greater than " + LOG.warn("Misconfiguration: the baseSleepTimeMs [" + baseSleepTimeMs + + "] can't be greater than " + "the maxSleepTimeMs [" + maxSleepTimeMs + "]."); } if (maxRetries > 0 && maxRetries > expRetriesThreshold) { - this.stepSize = Math.max(1, (maxSleepTimeMs - (1 << expRetriesThreshold)) / (maxRetries - expRetriesThreshold)); + this.stepSize = Math.max(1, + (maxSleepTimeMs - (1 << expRetriesThreshold)) / (maxRetries - expRetriesThreshold)); } else { this.stepSize = 1; } diff --git a/storm-client/src/jvm/org/apache/storm/utils/SupervisorClient.java b/storm-client/src/jvm/org/apache/storm/utils/SupervisorClient.java index e52286a08ee..6596c3c2ec4 100644 --- a/storm-client/src/jvm/org/apache/storm/utils/SupervisorClient.java +++ b/storm-client/src/jvm/org/apache/storm/utils/SupervisorClient.java @@ -1,12 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

    * http://www.apache.org/licenses/LICENSE-2.0 *

    - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -22,7 +27,8 @@ import org.slf4j.LoggerFactory; /** - * Client for interacting with Supervisor server, now we use supervisor server mainly for cases below. + * Client for interacting with Supervisor server, now we use supervisor server mainly for cases + * below. *

      *
    • worker <- supervisor: get worker local assignment for a storm.
    • *
    • nimbus -> supervisor: assign assignments for a node.
    • @@ -57,8 +63,10 @@ public Builder withPort(Integer port) { } public SupervisorClient build() throws TTransportException { - boolean useTls = ObjectReader.getBoolean(conf.get(Config.SUPERVISOR_THRIFT_CLIENT_USE_TLS), false); - ThriftConnectionType type = useTls ? ThriftConnectionType.SUPERVISOR_TLS : ThriftConnectionType.SUPERVISOR; + boolean useTls = ObjectReader.getBoolean(conf + .get(Config.SUPERVISOR_THRIFT_CLIENT_USE_TLS), false); + ThriftConnectionType type = useTls + ? ThriftConnectionType.SUPERVISOR_TLS : ThriftConnectionType.SUPERVISOR; return new SupervisorClient(this.conf, type, this.hostname, this.port, this.asUser); } @@ -66,7 +74,8 @@ public SupervisorClient createSupervisorClient() { try { if (conf.containsKey(Config.STORM_DO_AS_USER)) { if (asUser != null && !asUser.isEmpty()) { - LOG.warn("You have specified a doAsUser as param {} and a doAsParam as config, config will take precedence.", + LOG.warn("You have specified a doAsUser as param {} and a doAsParam as " + + "config, config will take precedence.", asUser, conf.get(Config.STORM_DO_AS_USER)); } @@ -74,18 +83,21 @@ public SupervisorClient createSupervisorClient() { } return this.build(); } catch (TTransportException e) { - throw new RuntimeException("Failed to create a supervisor client for host " + this.hostname); + throw new RuntimeException("Failed to create a supervisor client for host " + + this.hostname); } } } - private SupervisorClient(Map conf, ThriftConnectionType type, String host, Integer port, String asUser) throws TTransportException { + private SupervisorClient(Map conf, ThriftConnectionType type, String host, Integer port, + String asUser) throws TTransportException { super(conf, type, host, port, null, asUser); client = new Supervisor.Client(protocol); } /** * Constructor. + * * @param conf the conf for the client. * @param host the host the client is to talk to. * @param port the port for the client. @@ -99,6 +111,7 @@ public SupervisorClient(Map conf, String host, int port) throws TTransportExcept /** * Constructor. + * * @param conf the conf for the client. * @param host the host the client is to talk to. * @param port the port for the client. @@ -107,13 +120,15 @@ public SupervisorClient(Map conf, String host, int port) throws TTransportExcept * @deprecated use {@link SupervisorClient.Builder #build()} instead. */ @Deprecated - public SupervisorClient(Map conf, String host, int port, Integer timeout) throws TTransportException { + public SupervisorClient(Map conf, String host, int port, + Integer timeout) throws TTransportException { super(conf, ThriftConnectionType.SUPERVISOR, host, port, timeout, null); client = new Supervisor.Client(protocol); } /** * Constructor. + * * @param conf the conf for the client. * @param host the host the client is to talk to. * @param port the port for the client. @@ -123,13 +138,15 @@ public SupervisorClient(Map conf, String host, int port, Integer timeout) throws * @deprecated use {@link SupervisorClient.Builder #build()} instead. */ @Deprecated - public SupervisorClient(Map conf, String host, Integer port, Integer timeout, String asUser) throws TTransportException { + public SupervisorClient(Map conf, String host, Integer port, Integer timeout, + String asUser) throws TTransportException { super(conf, ThriftConnectionType.SUPERVISOR, host, port, timeout, asUser); client = new Supervisor.Client(protocol); } /** * Constructor. + * * @param conf the conf for the client. * @param host the host the client is to talk to. * @throws TTransportException on any error. @@ -143,6 +160,7 @@ public SupervisorClient(Map conf, String host) throws TTransportException { /** * Get a supervisor client as configured by conf. + * * @param conf the configuration to use. * @param host the host to use. * @return the client, don't forget to close it when done. @@ -150,13 +168,14 @@ public SupervisorClient(Map conf, String host) throws TTransportException { */ @Deprecated public static SupervisorClient getConfiguredClient(Map conf, String host) { - //use the default server port. + // use the default server port. int port = Integer.parseInt(conf.get(Config.SUPERVISOR_THRIFT_PORT).toString()); return getConfiguredClientAs(conf, host, port, null); } /** * Get a supervisor client as configured by conf. + * * @param conf the configuration to use. * @param host the host to use. * @param port the port to use. @@ -170,6 +189,7 @@ public static SupervisorClient getConfiguredClient(Map conf, String host, int po /** * Get a supervisor client as configured by conf. + * * @param conf the configuration to use. * @param host the host to use. * @param port the port to use. @@ -178,10 +198,12 @@ public static SupervisorClient getConfiguredClient(Map conf, String host, int po * @deprecated use {@link SupervisorClient.Builder #createSupervisorClient()} instead. */ @Deprecated - public static SupervisorClient getConfiguredClientAs(Map conf, String host, int port, String asUser) { + public static SupervisorClient getConfiguredClientAs(Map conf, String host, int port, + String asUser) { if (conf.containsKey(Config.STORM_DO_AS_USER)) { if (asUser != null && !asUser.isEmpty()) { - LOG.warn("You have specified a doAsUser as param {} and a doAsParam as config, config will take precedence.", + LOG.warn("You have specified a doAsUser as param {} and a doAsParam as config, " + + "config will take precedence.", asUser, conf.get(Config.STORM_DO_AS_USER)); } diff --git a/storm-client/src/jvm/org/apache/storm/utils/ThriftTopologyUtils.java b/storm-client/src/jvm/org/apache/storm/utils/ThriftTopologyUtils.java index a1e010dcacc..3034f6ea49d 100644 --- a/storm-client/src/jvm/org/apache/storm/utils/ThriftTopologyUtils.java +++ b/storm-client/src/jvm/org/apache/storm/utils/ThriftTopologyUtils.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -26,7 +32,8 @@ public static boolean isWorkerHook(StormTopology._Fields f) { } public static boolean isDependencies(StormTopology._Fields f) { - return f.equals(StormTopology._Fields.DEPENDENCY_JARS) || f.equals(StormTopology._Fields.DEPENDENCY_ARTIFACTS); + return f.equals(StormTopology._Fields.DEPENDENCY_JARS) || f + .equals(StormTopology._Fields.DEPENDENCY_ARTIFACTS); } public static Set getComponentIds(StormTopology topology) { diff --git a/storm-client/src/jvm/org/apache/storm/utils/Time.java b/storm-client/src/jvm/org/apache/storm/utils/Time.java index 9946b46e8c9..ea05f7d7f92 100644 --- a/storm-client/src/jvm/org/apache/storm/utils/Time.java +++ b/storm-client/src/jvm/org/apache/storm/utils/Time.java @@ -1,20 +1,26 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.utils; import java.util.Iterator; -import java.util.Map; import java.util.Map.Entry; +import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; @@ -23,15 +29,19 @@ import org.slf4j.LoggerFactory; /** - * This class implements time simulation support. When time simulation is enabled, methods on this class will use fixed time. When time - * simulation is disabled, methods will pass through to relevant java.lang.System/java.lang.Thread calls. Methods using units higher than - * nanoseconds will pass through to System.currentTimeMillis(). Methods supporting nanoseconds will pass through to System.nanoTime(). + * This class implements time simulation support. When time simulation is enabled, methods on this + * class will use fixed time. When time + * simulation is disabled, methods will pass through to relevant java.lang.System/java.lang.Thread + * calls. Methods using units higher than + * nanoseconds will pass through to System.currentTimeMillis(). Methods supporting nanoseconds will + * pass through to System.nanoTime(). */ public class Time { private static final Logger LOG = LoggerFactory.getLogger(Time.class); private static final AtomicBoolean SIMULATING = new AtomicBoolean(false); private static final AtomicLong AUTO_ADVANCE_NANOS_ON_SLEEP = new AtomicLong(0); - private static final Map THREAD_SLEEP_TIMES_NANOS = new ConcurrentHashMap<>(); + private static final Map THREAD_SLEEP_TIMES_NANOS = + new ConcurrentHashMap<>(); private static final Object SLEEP_TIMES_LOCK = new Object(); private static final AtomicLong SIMULATED_CURR_TIME_NANOS = new AtomicLong(0); @@ -67,16 +77,19 @@ private static void simulatedSleepUntilNanos(long targetTimeNanos) throws Interr try { synchronized (SLEEP_TIMES_LOCK) { if (!SIMULATING.get()) { - LOG.debug("{} is still sleeping after simulated time disabled.", Thread.currentThread(), + LOG.debug("{} is still sleeping after simulated time disabled.", Thread + .currentThread(), new RuntimeException("STACK TRACE")); throw new InterruptedException(); } - THREAD_SLEEP_TIMES_NANOS.put(Thread.currentThread(), new AtomicLong(targetTimeNanos)); + THREAD_SLEEP_TIMES_NANOS.put(Thread.currentThread(), + new AtomicLong(targetTimeNanos)); } while (SIMULATED_CURR_TIME_NANOS.get() < targetTimeNanos) { synchronized (SLEEP_TIMES_LOCK) { if (!SIMULATING.get()) { - LOG.debug("{} is still sleeping after simulated time disabled.", Thread.currentThread(), + LOG.debug("{} is still sleeping after simulated time disabled.", Thread + .currentThread(), new RuntimeException("STACK TRACE")); throw new InterruptedException(); } @@ -202,7 +215,8 @@ public static void advanceTimeNanos(long nanos) { throw new IllegalStateException("Cannot simulate time unless in simulation mode"); } if (nanos < 0) { - throw new IllegalArgumentException("advanceTime only accepts positive time as an argument"); + throw new IllegalArgumentException("advanceTime only accepts positive time as an " + + "argument"); } synchronized (SLEEP_TIMES_LOCK) { long newTime = SIMULATED_CURR_TIME_NANOS.addAndGet(nanos); diff --git a/storm-client/src/jvm/org/apache/storm/utils/TimeCacheMap.java b/storm-client/src/jvm/org/apache/storm/utils/TimeCacheMap.java index 7d8521969f0..b4f99167edf 100644 --- a/storm-client/src/jvm/org/apache/storm/utils/TimeCacheMap.java +++ b/storm-client/src/jvm/org/apache/storm/utils/TimeCacheMap.java @@ -1,33 +1,41 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.utils; -import java.util.Map; import java.util.Map.Entry; +import java.util.Map; /** - * Expires keys that have not been updated in the configured number of seconds. The algorithm used will take between expirationSecs and + * Expires keys that have not been updated in the configured number of seconds. The algorithm used + * will take between expirationSecs and * expirationSecs * (1 + 1 / (numBuckets-1)) to actually expire the message. * *

      get, put, remove, containsKey, and size take O(numBuckets) time to run. * - *

      The advantage of this design is that the expiration thread only locks the object for O(1) time, meaning the object + *

      The advantage of this design is that the expiration thread only locks the object for O(1) + * time, meaning the object * is essentially always available for gets/puts. */ -//deprecated in favor of non-threaded RotatingMap +// deprecated in favor of non-threaded RotatingMap @Deprecated public class TimeCacheMap { - //this default ensures things expire at most 50% past the expiration time + // this default ensures things expire at most 50% past the expiration time private static final int DEFAULT_NUM_BUCKETS = 3; private final RotatingMap rotatingMap; private final Object lock = new Object(); @@ -58,7 +66,7 @@ public void run() { } } } catch (InterruptedException ex) { - //ignore + // ignore } } }); diff --git a/storm-client/src/jvm/org/apache/storm/utils/TransferDrainer.java b/storm-client/src/jvm/org/apache/storm/utils/TransferDrainer.java index 540d691ec1e..54217469e9e 100644 --- a/storm-client/src/jvm/org/apache/storm/utils/TransferDrainer.java +++ b/storm-client/src/jvm/org/apache/storm/utils/TransferDrainer.java @@ -1,13 +1,19 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

      - * http://www.apache.org/licenses/LICENSE-2.0 - *

      - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

      http://www.apache.org/licenses/LICENSE-2.0 + * + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -16,8 +22,8 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; -import java.util.Map; import java.util.Map.Entry; +import java.util.Map; import java.util.stream.Stream; import org.apache.storm.generated.NodeInfo; import org.apache.storm.messaging.IConnection; @@ -42,7 +48,8 @@ public void add(TaskMessage taskMsg) { } public void send(Map taskToNode, Map connections) { - HashMap> bundleMapByDestination = groupBundleByDestination(taskToNode); + HashMap> bundleMapByDestination = + groupBundleByDestination(taskToNode); for (Map.Entry> entry : bundleMapByDestination.entrySet()) { NodeInfo node = entry.getKey(); @@ -58,7 +65,8 @@ public void send(Map taskToNode, Map c } } - private HashMap> groupBundleByDestination(Map taskToNode) { + private HashMap> groupBundleByDestination(Map taskToNode) { HashMap> result = new HashMap<>(); for (Entry> entry : bundles.entrySet()) { diff --git a/storm-client/src/jvm/org/apache/storm/utils/TupleUtils.java b/storm-client/src/jvm/org/apache/storm/utils/TupleUtils.java index fd4397cf489..60512f23c09 100644 --- a/storm-client/src/jvm/org/apache/storm/utils/TupleUtils.java +++ b/storm-client/src/jvm/org/apache/storm/utils/TupleUtils.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -46,7 +52,8 @@ private static int listHashCode(List alist) { } } - public static Map putTickFrequencyIntoComponentConfig(Map conf, int tickFreqSecs) { + public static Map putTickFrequencyIntoComponentConfig(Map conf, + int tickFreqSecs) { if (conf == null) { conf = new Config(); } diff --git a/storm-client/src/jvm/org/apache/storm/utils/Utils.java b/storm-client/src/jvm/org/apache/storm/utils/Utils.java index feebf0fabaf..678484e0ae7 100644 --- a/storm-client/src/jvm/org/apache/storm/utils/Utils.java +++ b/storm-client/src/jvm/org/apache/storm/utils/Utils.java @@ -58,8 +58,8 @@ import java.util.HashSet; import java.util.Iterator; import java.util.List; -import java.util.Map; import java.util.Map.Entry; +import java.util.Map; import java.util.NavigableMap; import java.util.Objects; import java.util.Set; @@ -123,7 +123,8 @@ public class Utils { public static final Logger LOG = LoggerFactory.getLogger(Utils.class); public static final String DEFAULT_STREAM_ID = "default"; private static final Set> defaultAllowedExceptions = Collections.emptySet(); - private static final List LOCALHOST_ADDRESSES = Lists.newArrayList("localhost", "127.0.0.1", "0:0:0:0:0:0:0:1"); + private static final List LOCALHOST_ADDRESSES = Lists.newArrayList("localhost", + "127.0.0.1", "0:0:0:0:0:0:0:1"); static SerializationDelegate serializationDelegate; private static ThreadLocal threadSer = new ThreadLocal(); private static ThreadLocal threadDes = new ThreadLocal(); @@ -143,7 +144,8 @@ public class Utils { } /** - * Provide an instance of this class for delegates to use. To mock out delegated methods, provide an instance of a subclass that + * Provide an instance of this class for delegates to use. To mock out delegated methods, + * provide an instance of a subclass that * overrides the implementation of the delegated method. * * @param u a Utils instance @@ -167,7 +169,8 @@ public static void resetClassLoaderForJavaDeSerialize() { public static List findResources(String name) { try { - Enumeration resources = Thread.currentThread().getContextClassLoader().getResources(name); + Enumeration resources = Thread.currentThread().getContextClassLoader() + .getResources(name); List ret = new ArrayList(); while (resources.hasMoreElements()) { ret.add(resources.nextElement()); @@ -186,7 +189,8 @@ public static Map findAndReadConfigFile(String name, boolean mus if (null != in) { Yaml yaml = new Yaml(new SafeConstructor(new LoaderOptions())); @SuppressWarnings("unchecked") - Map ret = (Map) yaml.load(new InputStreamReader(in)); + Map ret = (Map) yaml + .load(new InputStreamReader(in)); if (null != ret) { return new HashMap<>(ret); } else { @@ -196,7 +200,8 @@ public static Map findAndReadConfigFile(String name, boolean mus if (mustExist) { if (confFileEmpty) { - throw new RuntimeException("Config file " + name + " doesn't have any valid storm configs"); + throw new RuntimeException("Config file " + name + + " doesn't have any valid storm configs"); } else { throw new RuntimeException("Could not find config file on classpath " + name); } @@ -251,28 +256,31 @@ public static Map readDefaultConfig() { } /** - * URL encode the given string using the UTF-8 charset. Once Storm is baselined to Java 11, we can use URLEncoder.encode(String, + * URL encode the given string using the UTF-8 charset. Once Storm is baselined to Java 11, we + * can use URLEncoder.encode(String, * Charset) instead, which obsoletes this method. */ public static String urlEncodeUtf8(String s) { try { return URLEncoder.encode(s, StandardCharsets.UTF_8.name()); } catch (UnsupportedEncodingException e) { - //This cannot happen since we're using a standard charset + // This cannot happen since we're using a standard charset throw Utils.wrapInRuntime(e); } } /** - * URL decode the given string using the UTF-8 charset. Once Storm is baselined to Java 11, we can use URLDecoder.decode(String, + * URL decode the given string using the UTF-8 charset. Once Storm is baselined to Java 11, we + * can use URLDecoder.decode(String, * Charset) instead, which obsoletes this method. */ public static String urlDecodeUtf8(String s) { try { - //Once Storm is baselined to Java 11, we can use URLDecoder.decode(String, Charset) instead, which obsoletes this method. + // Once Storm is baselined to Java 11, we can use URLDecoder.decode(String, Charset) + // instead, which obsoletes this method. return URLDecoder.decode(s, StandardCharsets.UTF_8.name()); } catch (UnsupportedEncodingException e) { - //This cannot happen since we're using a standard charset + // This cannot happen since we're using a standard charset throw Utils.wrapInRuntime(e); } } @@ -300,7 +308,7 @@ replace below code with split(",") once 'storm.cmd' is fixed to send url-encoded try { val = JSONValue.parseWithException(options[1]); } catch (ParseException ignored) { - //fall back to string, which is already set + // fall back to string, which is already set } ret.put(options[0], val); } @@ -336,7 +344,8 @@ public static long bitXor(Long a, Long b) { } /** - * Adds the user supplied function as a shutdown hook for cleanup. Also adds a function that sleeps for a second and then halts the + * Adds the user supplied function as a shutdown hook for cleanup. Also adds a function that + * sleeps for a second and then halts the * runtime to avoid any zombie process in case cleanup function hangs. */ public static void addShutdownHookWithForceKillIn1Sec(Runnable func) { @@ -344,7 +353,8 @@ public static void addShutdownHookWithForceKillIn1Sec(Runnable func) { } /** - * Adds the user supplied function as a shutdown hook for cleanup. Also adds a function that sleeps for numSecs and then halts the + * Adds the user supplied function as a shutdown hook for cleanup. Also adds a function that + * sleeps for numSecs and then halts the * runtime to avoid any zombie process in case cleanup function hangs. */ public static void addShutdownHookWithDelayedForceKill(Runnable func, int numSecs) { @@ -355,7 +365,7 @@ public static void addShutdownHookWithDelayedForceKill(Runnable func, int numSec LOG.warn("Forcing Halt... {}", Utils.threadDump()); Runtime.getRuntime().halt(20); } catch (InterruptedException ie) { - //Ignored/expected... + // Ignored/expected... } catch (Exception e) { LOG.warn("Exception in the ShutDownHook", e); } @@ -374,10 +384,12 @@ public static boolean isSystemId(String id) { } /** - * Creates a thread that calls the given code repeatedly, sleeping for an interval of seconds equal to the return value of the previous + * Creates a thread that calls the given code repeatedly, sleeping for an interval of seconds + * equal to the return value of the previous * call. * - *

      The given afn may be a callable that returns the number of seconds to sleep, or it may be a Callable that returns another Callable + *

      The given afn may be a callable that returns the number of seconds to sleep, or it may be + * a Callable that returns another Callable * that in turn returns the number of seconds to sleep. In the latter case isFactory. * * @param afn the code to call on each iteration @@ -391,7 +403,8 @@ public static boolean isSystemId(String id) { * * @see Thread */ - public static SmartThread asyncLoop(final Callable afn, boolean isDaemon, final Thread.UncaughtExceptionHandler eh, + public static SmartThread asyncLoop(final Callable afn, boolean isDaemon, + final Thread.UncaughtExceptionHandler eh, int priority, final boolean isFactory, boolean startImmediately, String threadName) { SmartThread thread = new SmartThread(new Runnable() { @@ -453,7 +466,8 @@ public void uncaughtException(Thread t, Throwable e) { * * @see Thread */ - public static SmartThread asyncLoop(final Callable afn, String threadName, final Thread.UncaughtExceptionHandler eh) { + public static SmartThread asyncLoop(final Callable afn, String threadName, + final Thread.UncaughtExceptionHandler eh) { return asyncLoop(afn, false, eh, Thread.NORM_PRIORITY, false, true, threadName); } @@ -510,7 +524,8 @@ public static long secureRandomLong() { } /** - * Gets the storm.local.hostname value, or tries to figure out the local hostname if it is not set in the config. + * Gets the storm.local.hostname value, or tries to figure out the local hostname if it is not + * set in the config. * * @return a string representation of the hostname. */ @@ -546,7 +561,8 @@ public static byte[] javaSerialize(Object obj) { public static T javaDeserialize(byte[] serialized, Class clazz) { if ("true".equalsIgnoreCase(System.getProperty("java.deserialization.disabled"))) { - throw new AssertionError("java deserialization has been disabled and is only safe from within a worker process"); + throw new AssertionError("java deserialization has been disabled and is only safe " + + "from within a worker process"); } try { @@ -595,7 +611,8 @@ public static String join(Iterable coll, String sep) { public static Id parseZkId(String id, String configName) { String[] split = id.split(":", 2); if (split.length != 2) { - throw new IllegalArgumentException(configName + " does not appear to be in the form scheme:acl, i.e. sasl:storm-user"); + throw new IllegalArgumentException(configName + + " does not appear to be in the form scheme:acl, i.e. sasl:storm-user"); } return new Id(split[0], split[1]); } @@ -610,7 +627,8 @@ public static Id parseZkId(String id, String configName) { public static ACL getSuperUserAcl(Map conf) { String stormZKUser = (String) conf.get(Config.STORM_ZOOKEEPER_SUPERACL); if (stormZKUser == null) { - throw new IllegalArgumentException("Authentication is enabled but " + Config.STORM_ZOOKEEPER_SUPERACL + " is not set"); + throw new IllegalArgumentException("Authentication is enabled but " + + Config.STORM_ZOOKEEPER_SUPERACL + " is not set"); } return new ACL(ZooDefs.Perms.ALL, parseZkId(stormZKUser, Config.STORM_ZOOKEEPER_SUPERACL)); } @@ -648,13 +666,15 @@ public static boolean isZkAuthenticationConfiguredTopology(Map c * * @param worker true if this is for handling worker exceptions */ - public static void handleUncaughtException(Throwable t, Set> allowedExceptions, boolean worker) { + public static void handleUncaughtException(Throwable t, Set> allowedExceptions, + boolean worker) { if (t != null) { if (t instanceof OutOfMemoryError) { try { - System.err.println("Halting due to Out Of Memory Error..." + Thread.currentThread().getName()); + System.err.println("Halting due to Out Of Memory Error..." + Thread + .currentThread().getName()); } catch (Throwable err) { - //Again we don't want to exit because of logging issues. + // Again we don't want to exit because of logging issues. } Runtime.getRuntime().halt(-1); } @@ -672,7 +692,7 @@ public static void handleUncaughtException(Throwable t, Set> allowedExc return; } - //Running in daemon mode, we would pass Error to calling thread. + // Running in daemon mode, we would pass Error to calling thread. throw new Error(t); } @@ -684,13 +704,15 @@ public static void handleWorkerUncaughtException(Throwable t) { handleUncaughtException(t, defaultAllowedExceptions, true); } - // Hadoop UserGroupInformation can launch an autorenewal thread that can cause a NullPointerException + // Hadoop UserGroupInformation can launch an autorenewal thread that can cause a + // NullPointerException // for workers. See STORM-3606 for an explanation. private static boolean isAllowedWorkerException(Throwable t) { if (t instanceof NullPointerException) { StackTraceElement[] stackTrace = t.getStackTrace(); for (StackTraceElement trace : stackTrace) { - if (trace.getClassName().startsWith("org.apache.hadoop.security.UserGroupInformation") + if (trace.getClassName() + .startsWith("org.apache.hadoop.security.UserGroupInformation") && trace.getMethodName().equals("run")) { return true; } @@ -771,10 +793,13 @@ public static UptimeComputer makeUptimeComputer() { * "{:a 1 :b 1 :c 2} -> {1 [:a :b] 2 :c}". * *

      Example usage in java: - * Map<Integer, String> tasks; Map<String, List<Integer>> componentTasks = Utils.reverse_map(tasks); + * Map<Integer, String> tasks; Map<String, List<Integer>> componentTasks + * = Utils.reverse_map(tasks); * - *

      The order of he resulting list values depends on the ordering properties of the Map passed in. The caller is - * responsible for passing an ordered map if they expect the result to be consistently ordered as well. + *

      The order of he resulting list values depends on the ordering properties of the Map passed + * in. The caller is + * responsible for passing an ordered map if they expect the result to be consistently ordered + * as well. * * @param map to reverse * @return a reversed map @@ -798,7 +823,8 @@ public static HashMap> reverseMap(Map map) { } /** - * "[[:a 1] [:b 1] [:c 2]} -> {1 [:a :b] 2 :c}" Reverses an assoc-list style Map like reverseMap(Map...) + * "[[:a 1] [:b 1] [:c 2]} -> {1 [:a :b] 2 :c}" Reverses an assoc-list style Map like + * reverseMap(Map...). * * @param listSeq to reverse * @return a reversed map @@ -833,7 +859,8 @@ public static boolean checkFileExists(String path) { } /** - * Deletes a file or directory and its contents if it exists. Does not complain if the input is null or does not exist. + * Deletes a file or directory and its contents if it exists. Does not complain if the input is + * null or does not exist. * * @param path the path to the file or directory */ @@ -850,7 +877,8 @@ public static T deserialize(byte[] serialized, Class clazz) { } /** - * Serialize an object using the configured serialization and then base64 encode it into a string. + * Serialize an object using the configured serialization and then base64 encode it into a + * string. * * @param obj the object to encode * @return a string with the encoded object in it. @@ -860,7 +888,8 @@ public static String serializeToString(Object obj) { } /** - * Deserialize an object stored in a string. The String is assumed to be a base64 encoded string containing the bytes to actually + * Deserialize an object stored in a string. The String is assumed to be a base64 encoded string + * containing the bytes to actually * deserialize. * * @param str the encoded string. @@ -899,7 +928,8 @@ public static void readAndLogStream(String prefix, InputStream in) { } /** - * Creates an instance of the pluggable SerializationDelegate or falls back to DefaultSerializationDelegate if something goes wrong. + * Creates an instance of the pluggable SerializationDelegate or falls back to + * DefaultSerializationDelegate if something goes wrong. * * @param topoConf The config from which to pull the name of the pluggable class. * @return an instance of the class specified by storm.meta.serialization.delegate @@ -911,7 +941,8 @@ private static SerializationDelegate getSerializationDelegate(Map fromCompressedJsonConf(byte[] serialized) { } /** - * Creates a new map with a string value in the map replaced with an equivalently-lengthed string of '#'. (If the object is not a + * Creates a new map with a string value in the map replaced with an equivalently-lengthed + * string of '#'. (If the object is not a * string to string will be called on it and replaced) * * @param m The map that a value will be redacted from @@ -1239,7 +1280,8 @@ public static UncaughtExceptionHandler createDefaultUncaughtExceptionHandler() { try { handleUncaughtException(thrown); } catch (Error err) { - LOG.error("Received error in thread {}.. terminating server...", thread.getName(), err); + LOG.error("Received error in thread {}.. terminating server...", thread.getName(), + err); Runtime.getRuntime().exit(-2); } }; @@ -1250,7 +1292,8 @@ public static UncaughtExceptionHandler createWorkerUncaughtExceptionHandler() { try { handleWorkerUncaughtException(thrown); } catch (Error err) { - LOG.error("Received error in thread {}.. terminating worker...", thread.getName(), err); + LOG.error("Received error in thread {}.. terminating worker...", thread.getName(), + err); Runtime.getRuntime().exit(-2); } }; @@ -1265,7 +1308,7 @@ public static void setupWorkerUncaughtExceptionHandler() { } /** - * parses the arguments to extract jvm heap memory size in MB. + * Parses the arguments to extract jvm heap memory size in MB. * * @return the value of the JVM heap memory setting (in MB) in a java command. */ @@ -1305,7 +1348,8 @@ public static Double parseJvmHeapMemByChildOpts(List options, Double def } public static ClientBlobStore getClientBlobStore(Map conf) { - ClientBlobStore store = (ClientBlobStore) ReflectionUtils.newInstance((String) conf.get(Config.CLIENT_BLOBSTORE)); + ClientBlobStore store = (ClientBlobStore) ReflectionUtils.newInstance((String) conf + .get(Config.CLIENT_BLOBSTORE)); store.prepare(conf); return store; } @@ -1346,7 +1390,8 @@ public static boolean isValidConf(Map topoConfIn) { Map origTopoConf = normalizeConf(topoConfIn); try { Map deserTopoConf = normalizeConf( - (Map) JSONValue.parseWithException(JSONValue.toJSONString(topoConfIn))); + (Map) JSONValue.parseWithException(JSONValue + .toJSONString(topoConfIn))); return isValidConf(origTopoConf, deserTopoConf); } catch (ParseException e) { LOG.error("Json serialized config could not be deserialized", e); @@ -1372,16 +1417,19 @@ static boolean isValidConf(Map orig, Map deser) + "serialization. Name: {} - Value: {}", entryOnRight.getKey(), entryOnRight.getKey(), entryOnRight.getValue()); } - for (Map.Entry> entryDiffers : diff.entriesDiffering().entrySet()) { + for (Map.Entry> entryDiffers : diff + .entriesDiffering().entrySet()) { Object leftValue = entryDiffers.getValue().leftValue(); Object rightValue = entryDiffers.getValue().rightValue(); - LOG.warn("Config value differs after json serialization. Name: {} - Original Value: {} - DeSer. Value: {}", + LOG.warn("Config value differs after json serialization. Name: {} - Original Value: " + + "{} - DeSer. Value: {}", entryDiffers.getKey(), leftValue, rightValue); } return false; } - public static TopologyInfo getTopologyInfo(String name, String asUser, Map topoConf) { + public static TopologyInfo getTopologyInfo(String name, String asUser, Map topoConf) { NimbusClient.Builder builder = NimbusClient.Builder.withConf(topoConf).asUser(asUser); try (NimbusClient client = builder.build()) { return client.getClient().getTopologyInfoByName(name); @@ -1411,7 +1459,8 @@ public static String getTopologyId(String name, Nimbus.Iface client) { * * @param topoConf Topology configuration */ - public static void validateTopologyBlobStoreMap(Map topoConf) throws InvalidTopologyException, AuthorizationException { + public static void validateTopologyBlobStoreMap(Map topoConf) throws InvalidTopologyException, AuthorizationException { try (NimbusBlobStore client = new NimbusBlobStore()) { client.prepare(topoConf); validateTopologyBlobStoreMap(topoConf, client); @@ -1424,9 +1473,11 @@ public static void validateTopologyBlobStoreMap(Map topoConf) th * @param topoConf Topology configuration * @param client The NimbusBlobStore client. It must call prepare() before being used here. */ - public static void validateTopologyBlobStoreMap(Map topoConf, NimbusBlobStore client) + public static void validateTopologyBlobStoreMap(Map topoConf, + NimbusBlobStore client) throws InvalidTopologyException, AuthorizationException { - Map> blobStoreMap = (Map>) topoConf.get(Config.TOPOLOGY_BLOBSTORE_MAP); + Map> blobStoreMap = (Map>) topoConf + .get(Config.TOPOLOGY_BLOBSTORE_MAP); if (blobStoreMap != null) { for (String key : blobStoreMap.keySet()) { @@ -1435,7 +1486,8 @@ public static void validateTopologyBlobStoreMap(Map topoConf, Ni ObjectReader.getBoolean(blobConf.get("uncompress"), false); ObjectReader.getBoolean(blobConf.get("workerRestart"), false); } catch (IllegalArgumentException e) { - throw new WrappedInvalidTopologyException("Invalid blob conf option: " + e.getMessage()); + throw new WrappedInvalidTopologyException("Invalid blob conf option: " + e + .getMessage()); } // try to get BlobMeta @@ -1444,7 +1496,8 @@ public static void validateTopologyBlobStoreMap(Map topoConf, Ni client.getBlobMeta(key); } catch (KeyNotFoundException keyNotFound) { // wrap KeyNotFoundException in an InvalidTopologyException - throw new WrappedInvalidTopologyException("Key not found: " + keyNotFound.get_msg()); + throw new WrappedInvalidTopologyException("Key not found: " + keyNotFound + .get_msg()); } } } @@ -1453,9 +1506,11 @@ public static void validateTopologyBlobStoreMap(Map topoConf, Ni /** * Validate topology blobstore map. */ - public static void validateTopologyBlobStoreMap(Map topoConf, BlobStore blobStore) + public static void validateTopologyBlobStoreMap(Map topoConf, + BlobStore blobStore) throws InvalidTopologyException, AuthorizationException { - Map blobStoreMap = (Map) topoConf.get(Config.TOPOLOGY_BLOBSTORE_MAP); + Map blobStoreMap = (Map) topoConf + .get(Config.TOPOLOGY_BLOBSTORE_MAP); if (blobStoreMap != null) { Subject subject = ReqContext.context().subject(); for (String key : blobStoreMap.keySet()) { @@ -1463,7 +1518,8 @@ public static void validateTopologyBlobStoreMap(Map topoConf, Bl blobStore.getBlobMeta(key, subject); } catch (KeyNotFoundException keyNotFound) { // wrap KeyNotFoundException in an InvalidTopologyException - throw new WrappedInvalidTopologyException("Key not found: " + keyNotFound.get_msg()); + throw new WrappedInvalidTopologyException("Key not found: " + keyNotFound + .get_msg()); } } } @@ -1477,12 +1533,13 @@ public static void validateTopologyBlobStoreMap(Map topoConf, Bl public static String threadDump() { final StringBuilder dump = new StringBuilder(); final java.lang.management.ThreadMXBean threadMxBean = ManagementFactory.getThreadMXBean(); - final java.lang.management.ThreadInfo[] threadInfos = threadMxBean.getThreadInfo(threadMxBean.getAllThreadIds(), 100); + final java.lang.management.ThreadInfo[] threadInfos = threadMxBean + .getThreadInfo(threadMxBean.getAllThreadIds(), 100); for (Entry entry : Thread.getAllStackTraces().entrySet()) { Thread t = entry.getKey(); ThreadInfo threadInfo = threadMxBean.getThreadInfo(t.getId()); if (threadInfo == null) { - //Thread died before we could get the info, skip + // Thread died before we could get the info, skip continue; } dump.append('"'); @@ -1527,7 +1584,8 @@ public static Object getConfiguredClass(Map conf, Object configK } /** - * Is the cluster configured to interact with ZooKeeper in a secure way? This only works when called from within Nimbus or a Supervisor + * Is the cluster configured to interact with ZooKeeper in a secure way? This only works when + * called from within Nimbus or a Supervisor * process. * * @param conf the storm configuration, not the topology configuration @@ -1557,7 +1615,7 @@ public static double nullToZero(Double v) { } /** - * a or b the first one that is not null. + * A or b the first one that is not null. * * @param a something * @param b something else @@ -1583,7 +1641,8 @@ public static TreeMap integerDivided(int sum, int numPieces) { /** * Fills up chunks out of a collection (given a maximum amount of chunks). * - *

      i.e. partitionFixed(5, [1,2,3]) -> [[1,2,3]] partitionFixed(5, [1..9]) -> [[1,2], [3,4], [5,6], [7,8], [9]] partitionFixed(3, + *

      i.e. partitionFixed(5, [1,2,3]) -> [[1,2,3]] partitionFixed(5, [1..9]) -> [[1,2], [3,4], + * [5,6], [7,8], [9]] partitionFixed(3, * [1..10]) -> [[1,2,3,4], [5,6,7], [8,9,10]] * * @param maxNumChunks the maximum number of chunks to return @@ -1638,7 +1697,8 @@ public static Object readYamlFile(String yamlFile) { } /** - * Gets an available port. Consider if it is possible to pass port 0 to the server instead of using this method, since there is no + * Gets an available port. Consider if it is possible to pass port 0 to the server instead of + * using this method, since there is no * guarantee that the port returned by this method will remain free. * * @return The preferred port if available, or a random available port @@ -1731,19 +1791,23 @@ public static StormTopology addVersions(StormTopology topology) { } /** - * Get a map of version to classpath from the conf Config.SUPERVISOR_WORKER_VERSION_CLASSPATH_MAP + * Get a map of version to classpath from the conf + * Config.SUPERVISOR_WORKER_VERSION_CLASSPATH_MAP. * * @param conf what to read it out of - * @param currentClassPath the current classpath for this version of storm (not included in the conf, but returned by this) + * @param currentClassPath the current classpath for this version of storm (not included in the + * conf, but returned by this) * @return the map */ public static NavigableMap> getConfiguredClasspathVersions(Map conf, List currentClassPath) { TreeMap> ret = new TreeMap<>(); Map fromConf = - (Map) conf.getOrDefault(Config.SUPERVISOR_WORKER_VERSION_CLASSPATH_MAP, Collections.emptyMap()); + (Map) conf.getOrDefault(Config.SUPERVISOR_WORKER_VERSION_CLASSPATH_MAP, + Collections.emptyMap()); for (Map.Entry entry : fromConf.entrySet()) { - ret.put(new SimpleVersion(entry.getKey()), Arrays.asList(entry.getValue().split(File.pathSeparator))); + ret.put(new SimpleVersion(entry.getKey()), Arrays.asList(entry.getValue() + .split(File.pathSeparator))); } ret.put(VersionInfo.OUR_VERSION, currentClassPath); return ret; @@ -1751,19 +1815,23 @@ public static NavigableMap> getConfiguredClasspathVe /** * Get a mapping of the configured supported versions of storm to their actual versions. + * * @param conf what to read the configuration out of. * @return the map. */ - public static NavigableMap getAlternativeVersionsMap(Map conf) { + public static NavigableMap getAlternativeVersionsMap(Map conf) { TreeMap ret = new TreeMap<>(); Map fromConf = - (Map) conf.getOrDefault(Config.SUPERVISOR_WORKER_VERSION_CLASSPATH_MAP, Collections.emptyMap()); + (Map) conf.getOrDefault(Config.SUPERVISOR_WORKER_VERSION_CLASSPATH_MAP, + Collections.emptyMap()); for (Map.Entry entry : fromConf.entrySet()) { IVersionInfo version = VersionInfo.getFromClasspath(entry.getValue()); if (version != null) { ret.put(entry.getKey(), version); } else { - LOG.error("Could not find the real version of {} from CP {}", entry.getKey(), entry.getValue()); + LOG.error("Could not find the real version of {} from CP {}", entry.getKey(), entry + .getValue()); ret.put(entry.getKey(), new IVersionInfo() { @Override public String getVersion() { @@ -1801,15 +1869,17 @@ public String getBuildVersion() { } /** - * Get a map of version to worker main from the conf Config.SUPERVISOR_WORKER_VERSION_MAIN_MAP + * Get a map of version to worker main from the conf Config.SUPERVISOR_WORKER_VERSION_MAIN_MAP. * * @param conf what to read it out of * @return the map */ - public static NavigableMap getConfiguredWorkerMainVersions(Map conf) { + public static NavigableMap getConfiguredWorkerMainVersions(Map conf) { TreeMap ret = new TreeMap<>(); Map fromConf = - (Map) conf.getOrDefault(Config.SUPERVISOR_WORKER_VERSION_MAIN_MAP, Collections.emptyMap()); + (Map) conf.getOrDefault(Config.SUPERVISOR_WORKER_VERSION_MAIN_MAP, + Collections.emptyMap()); for (Map.Entry entry : fromConf.entrySet()) { ret.put(new SimpleVersion(entry.getKey()), entry.getValue()); } @@ -1819,7 +1889,8 @@ public static NavigableMap getConfiguredWorkerMainVersion } /** - * Get a map of version to worker log writer from the conf Config.SUPERVISOR_WORKER_VERSION_LOGWRITER_MAP + * Get a map of version to worker log writer from the conf + * Config.SUPERVISOR_WORKER_VERSION_LOGWRITER_MAP. * * @param conf what to read it out of * @return the map @@ -1827,7 +1898,8 @@ public static NavigableMap getConfiguredWorkerMainVersion public static NavigableMap getConfiguredWorkerLogWriterVersions(Map conf) { TreeMap ret = new TreeMap<>(); Map fromConf = - (Map) conf.getOrDefault(Config.SUPERVISOR_WORKER_VERSION_LOGWRITER_MAP, Collections.emptyMap()); + (Map) conf.getOrDefault(Config.SUPERVISOR_WORKER_VERSION_LOGWRITER_MAP, + Collections.emptyMap()); for (Map.Entry entry : fromConf.entrySet()) { ret.put(new SimpleVersion(entry.getKey()), entry.getValue()); } @@ -1836,25 +1908,30 @@ public static NavigableMap getConfiguredWorkerLogWriterVe return ret; } - public static T getCompatibleVersion(NavigableMap versionedMap, SimpleVersion desiredVersion, String what, + public static T getCompatibleVersion(NavigableMap versionedMap, + SimpleVersion desiredVersion, String what, T defaultValue) { Entry ret = versionedMap.ceilingEntry(desiredVersion); if (ret == null || ret.getKey().getMajor() != desiredVersion.getMajor()) { - //Could not find a "fully" compatible version. Look to see if there is a possibly compatible version right below it + // Could not find a "fully" compatible version. Look to see if there is a possibly + // compatible version right below it ret = versionedMap.floorEntry(desiredVersion); if (ret == null || ret.getKey().getMajor() != desiredVersion.getMajor()) { if (defaultValue != null) { - LOG.warn("Could not find any compatible {} falling back to using {}", what, defaultValue); + LOG.warn("Could not find any compatible {} falling back to using {}", what, + defaultValue); } return defaultValue; } - LOG.warn("Could not find a higer compatible version for {} {}, using {} instead", what, desiredVersion, ret.getKey()); + LOG.warn("Could not find a higer compatible version for {} {}, using {} instead", what, + desiredVersion, ret.getKey()); } return ret.getValue(); } @SuppressWarnings("unchecked") - private static Map readConfIgnoreNotFound(Yaml yaml, File f) throws IOException { + private static Map readConfIgnoreNotFound(Yaml yaml, + File f) throws IOException { Map ret = null; if (f.exists()) { try (FileReader fr = new FileReader(f)) { @@ -1864,7 +1941,8 @@ private static Map readConfIgnoreNotFound(Yaml yaml, File f) thr return ret; } - public static Map getConfigFromClasspath(List cp, Map conf) throws IOException { + public static Map getConfigFromClasspath(List cp, Map conf) throws IOException { if (cp == null || cp.isEmpty()) { return conf; } @@ -1886,17 +1964,20 @@ public static Map getConfigFromClasspath(List cp, Map name.endsWith(".jar") || name.endsWith(".JAR")); + File[] jarFiles = dir.listFiles((dir1, name) -> name.endsWith(".jar") || name + .endsWith(".JAR")); // Quoting Javadoc in File.listFiles(FilenameFilter filter): - // Returns {@code null} if this abstract pathname does not denote a directory, or if an I/O error occurs. + // Returns {@code null} if this abstract pathname does not denote a directory, or if + // an I/O error occurs. // Both things are not expected and should not happen. if (jarFiles == null) { throw new IOException("Fail to list jar files in directory: " + dir); } for (File jarFile : jarFiles) { - JarConfigReader jarConfigReader = new JarConfigReader(yaml, defaultsConf, stormConf, jarFile).readJar(); + JarConfigReader jarConfigReader = new JarConfigReader(yaml, defaultsConf, + stormConf, jarFile).readJar(); defaultsConf = jarConfigReader.getDefaultsConf(); stormConf = jarConfigReader.getStormConf(); } @@ -1913,11 +1994,13 @@ public static Map getConfigFromClasspath(List cp, Map Map merge(Map first, Map other) { + public static Map merge(Map first, Map other) { Map ret = new HashMap<>(first); if (other != null) { ret.putAll(other); @@ -1947,7 +2031,8 @@ public static ArrayList convertToArray(Map srcMap, int start) Set ids = srcMap.keySet(); Integer largestId = ids.stream().max(Integer::compareTo).get(); int end = largestId - start; - ArrayList result = new ArrayList<>(Collections.nCopies(end + 1, null)); // creates array[largestId+1] filled with nulls + ArrayList result = new ArrayList<>(Collections.nCopies(end + 1, + null)); // creates array[largestId+1] filled with nulls for (Map.Entry entry : srcMap.entrySet()) { int id = entry.getKey(); if (id < start) { @@ -1966,7 +2051,7 @@ protected void forceDeleteImpl(String path) throws IOException { try { FileUtils.forceDelete(new File(path)); } catch (FileNotFoundException ignored) { - //ignore + // ignore } } } @@ -1999,8 +2084,10 @@ protected String hostnameImpl() throws UnknownHostException { * @param key Key for the blob. */ public static boolean isValidKey(String key) { - if (StringUtils.isEmpty(key) || "..".equals(key) || ".".equals(key) || !BLOB_KEY_PATTERN.matcher(key).matches()) { - LOG.error("'{}' does not appear to be valid. It must match {}. And it can't be \".\", \"..\", null or empty string.", key, + if (StringUtils.isEmpty(key) || "..".equals(key) || ".".equals(key) || !BLOB_KEY_PATTERN + .matcher(key).matches()) { + LOG.error("'{}' does not appear to be valid. It must match {}. And it can't be \".\", " + + "\"..\", null or empty string.", key, BLOB_KEY_PATTERN); return false; } @@ -2009,18 +2096,21 @@ public static boolean isValidKey(String key) { /** * Validates topology name. + * * @param name the topology name * @throws IllegalArgumentException if the topology name is not valid */ public static void validateTopologyName(String name) throws IllegalArgumentException { if (name == null || !TOPOLOGY_NAME_REGEX.matcher(name).matches()) { - String message = "Topology name '" + name + "' is not valid. It can't be null and it must match " + TOPOLOGY_NAME_REGEX; + String message = "Topology name '" + name + + "' is not valid. It can't be null and it must match " + TOPOLOGY_NAME_REGEX; throw new IllegalArgumentException(message); } } /** - * A thread that can answer if it is sleeping in the case of simulated time. This class is not useful when simulated time is not being + * A thread that can answer if it is sleeping in the case of simulated time. This class is not + * useful when simulated time is not being * used. */ public static class SmartThread extends Thread { @@ -2051,7 +2141,8 @@ private static class JarConfigReader { private Map stormConf; private File file; - JarConfigReader(Yaml yaml, Map defaultsConf, Map stormConf, File file) { + JarConfigReader(Yaml yaml, Map defaultsConf, Map stormConf, + File file) { this.yaml = yaml; this.defaultsConf = defaultsConf; this.stormConf = stormConf; @@ -2086,13 +2177,15 @@ private void readArchive(ZipFile zipFile) throws IOException { ZipEntry entry = zipEnums.nextElement(); if (!entry.isDirectory()) { if (defaultsConf == null && entry.getName().equals("defaults.yaml")) { - try (InputStreamReader isr = new InputStreamReader(zipFile.getInputStream(entry))) { + try (InputStreamReader isr = new InputStreamReader(zipFile + .getInputStream(entry))) { defaultsConf = (Map) yaml.load(isr); } } if (stormConf == null && entry.getName().equals("storm.yaml")) { - try (InputStreamReader isr = new InputStreamReader(zipFile.getInputStream(entry))) { + try (InputStreamReader isr = new InputStreamReader(zipFile + .getInputStream(entry))) { stormConf = (Map) yaml.load(isr); } } @@ -2102,7 +2195,8 @@ private void readArchive(ZipFile zipFile) throws IOException { } /** - * Create a map of forward edges for bolts in a topology. Note that spouts can be source but not a target in + * Create a map of forward edges for bolts in a topology. Note that spouts can be source but not + * a target in * the edge. The mapping contains ids of spouts and bolts. * * @param topology StormTopology to examine. @@ -2115,7 +2209,8 @@ private static Map> getStormTopologyForwardGraph(StormTopolo topology.get_bolts().entrySet().forEach(entry -> { if (!Utils.isSystemId(entry.getKey())) { entry.getValue().get_common().get_inputs().forEach((k, v) -> { - edgesOut.computeIfAbsent(k.get_componentId(), x -> new HashSet<>()).add(entry.getKey()); + edgesOut.computeIfAbsent(k.get_componentId(), x -> new HashSet<>()) + .add(entry.getKey()); }); } }); @@ -2124,7 +2219,8 @@ private static Map> getStormTopologyForwardGraph(StormTopolo } /** - * Use recursive descent to detect cycles. This is a Depth First recursion. Component Cycle is recorded when encountered. + * Use recursive descent to detect cycles. This is a Depth First recursion. Component Cycle is + * recorded when encountered. * In addition, the last link in the cycle is removed to avoid re-detecting same cycle/subcycle. * * @param stack used for recursion. @@ -2149,7 +2245,8 @@ private static void findComponentCyclesRecursion( List possibleCycle = new ArrayList<>(); if (compId1.equals(compId2)) { possibleCycle.add(compId2); - } else if (edgesOut.get(compId2) != null && edgesOut.get(compId2).contains(compId1)) { + } else if (edgesOut.get(compId2) != null && edgesOut.get(compId2) + .contains(compId1)) { possibleCycle.addAll(Arrays.asList(compId1, compId2)); } else { List tmp = Collections.list(stack.elements()); @@ -2193,7 +2290,8 @@ public static List> findComponentCycles(StormTopology topology, Str }); if (topology.get_spouts_size() == 0) { - LOG.error("Topology {} does not contain any spouts, cannot traverse graph to determine cycles", topoId); + LOG.error("Topology {} does not contain any spouts, cannot traverse graph to " + + "determine cycles", topoId); return ret; } @@ -2209,23 +2307,28 @@ public static List> findComponentCycles(StormTopology topology, Str // warning about unreachable components if (!unreachable.isEmpty()) { - LOG.warn("Topology {} contains unreachable components \"{}\"", topoId, String.join(",", unreachable)); + LOG.warn("Topology {} contains unreachable components \"{}\"", topoId, String.join(",", + unreachable)); } return ret; } /** - * Validate that the topology is cycle free. If not, then throw an InvalidTopologyException describing the cycle(s). + * Validate that the topology is cycle free. If not, then throw an InvalidTopologyException + * describing the cycle(s). * * @param topology StormTopology instance to examine. * @param name Name of the topology, used in exception error message. - * @throws InvalidTopologyException if there are cycles, with message describing the cycles encountered. + * @throws InvalidTopologyException if there are cycles, with message describing the cycles + * encountered. */ - public static void validateCycleFree(StormTopology topology, String name) throws InvalidTopologyException { + public static void validateCycleFree(StormTopology topology, + String name) throws InvalidTopologyException { List> cycles = Utils.findComponentCycles(topology, name); if (!cycles.isEmpty()) { String err = String.format("Topology %s contains cycles in components \"%s\"", name, - cycles.stream().map(x -> String.join(",", x)).collect(Collectors.joining(" ; "))); + cycles.stream().map(x -> String.join(",", x)).collect(Collectors + .joining(" ; "))); throw new WrappedInvalidTopologyException(err); } } diff --git a/storm-client/src/jvm/org/apache/storm/utils/VersionInfo.java b/storm-client/src/jvm/org/apache/storm/utils/VersionInfo.java index 4c9a4478738..5dac383cfbe 100644 --- a/storm-client/src/jvm/org/apache/storm/utils/VersionInfo.java +++ b/storm-client/src/jvm/org/apache/storm/utils/VersionInfo.java @@ -40,9 +40,11 @@ public final class VersionInfo { private static final Logger LOG = LoggerFactory.getLogger(VersionInfo.class); private static final String STORM_CORE_PROPERTIES_NAME = "storm-core-version-info.properties"; - private static final String STORM_CLIENT_PROPERTIES_NAME = "storm-client-version-info.properties"; + private static final String STORM_CLIENT_PROPERTIES_NAME = + "storm-client-version-info.properties"; public static final IVersionInfo OUR_FULL_VERSION = new VersionInfoImpl("storm-client"); - public static final SimpleVersion OUR_VERSION = new SimpleVersion(OUR_FULL_VERSION.getVersion()); + public static final SimpleVersion OUR_VERSION = new SimpleVersion(OUR_FULL_VERSION + .getVersion()); private static class VersionInfoImpl implements IVersionInfo { private Properties info; @@ -100,6 +102,7 @@ public String getBuildVersion() { /** * Look for the version of storm defined by the given classpath. + * * @param cp the classpath as a string to be parsed. * @return the IVersionInfo or null. */ @@ -110,19 +113,21 @@ public static IVersionInfo getFromClasspath(String cp) { /** * Look for the version of storm defined by the given classpath. + * * @param classpath the classpath as list of files/directories. * @return the IVersionInfo or null. */ public static IVersionInfo getFromClasspath(List classpath) { IVersionInfo ret = getFromClasspath(classpath, STORM_CLIENT_PROPERTIES_NAME); if (ret == null) { - //storm-core is needed here for backwards compatibility. + // storm-core is needed here for backwards compatibility. ret = getFromClasspath(classpath, STORM_CORE_PROPERTIES_NAME); } return ret; } - private static IVersionInfo getFromClasspath(List classpath, final String propFileName) { + private static IVersionInfo getFromClasspath(List classpath, + final String propFileName) { IVersionInfo ret = null; for (String part : classpath) { Path p = Paths.get(part); @@ -135,18 +140,20 @@ private static IVersionInfo getFromClasspath(List classpath, final Strin ret = new VersionInfoImpl(info); break; } catch (IOException e) { - LOG.error("Skipping {}; got an error while trying to parse the file.", part, e); + LOG.error("Skipping {}; got an error while trying to parse the file.", part, + e); } } } else if (part.toLowerCase().endsWith(".jar") || part.toLowerCase().endsWith(".zip")) { - //Treat it like a jar + // Treat it like a jar try (JarFile jf = new JarFile(p.toFile())) { Enumeration zipEnums = jf.entries(); while (zipEnums.hasMoreElements()) { ZipEntry entry = zipEnums.nextElement(); if (!entry.isDirectory() && entry.getName().equals(propFileName)) { - try (InputStreamReader reader = new InputStreamReader(jf.getInputStream(entry))) { + try (InputStreamReader reader = new InputStreamReader(jf + .getInputStream(entry))) { Properties info = new Properties(); info.load(reader); ret = new VersionInfoImpl(info); @@ -155,14 +162,15 @@ private static IVersionInfo getFromClasspath(List classpath, final Strin } } } catch (IOException e) { - LOG.error("Skipping {}; got an error while trying to parse the jar file.", part, e); + LOG.error("Skipping {}; got an error while trying to parse the jar file.", part, + e); } } else if (p.endsWith("*")) { - //for a path like //* + // for a path like //* Path parent = p.getParent(); List children = new ArrayList<>(); try (Stream stream = Files.list(parent)) { - //avoid infinite recursion + // avoid infinite recursion stream.filter(path -> !path.endsWith("*")) .forEach(path -> children.add(path.toString())); IVersionInfo resFromChildren = getFromClasspath(children, propFileName); @@ -182,6 +190,7 @@ private static IVersionInfo getFromClasspath(List classpath, final Strin /** * Get the version number of the build. + * * @return the version number of the build. */ public static String getVersion() { @@ -190,6 +199,7 @@ public static String getVersion() { /** * Get the SCM revision number of the build. + * * @return the SCM revision number of the build. */ public static String getRevision() { @@ -198,6 +208,7 @@ public static String getRevision() { /** * Get the SCM branch of the build. + * * @return the SCM branch of the build. */ public static String getBranch() { @@ -206,6 +217,7 @@ public static String getBranch() { /** * Get the date/time the build happened. + * * @return the date/time of the build. */ public static String getDate() { @@ -214,6 +226,7 @@ public static String getDate() { /** * Get the checksum of the source. + * * @return the checksum of the source. */ public static String getSrcChecksum() { @@ -222,6 +235,7 @@ public static String getSrcChecksum() { /** * Get a descriptive representation of the build meant for human consumption. + * * @return a descriptive representation of the build. */ public static String getBuildVersion() { diff --git a/storm-client/src/jvm/org/apache/storm/utils/VersionedStore.java b/storm-client/src/jvm/org/apache/storm/utils/VersionedStore.java index 68582289bd3..597f3813ab5 100644 --- a/storm-client/src/jvm/org/apache/storm/utils/VersionedStore.java +++ b/storm-client/src/jvm/org/apache/storm/utils/VersionedStore.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -143,7 +149,8 @@ public List getAllVersions() throws IOException { List ret = new ArrayList(); for (String s : listDir(root)) { - if (s.endsWith(FINISHED_VERSION_SUFFIX) && new File(s.substring(0, s.length() - FINISHED_VERSION_SUFFIX.length())).exists()) { + if (s.endsWith(FINISHED_VERSION_SUFFIX) && new File(s.substring(0, s + .length() - FINISHED_VERSION_SUFFIX.length())).exists()) { ret.add(validateAndGetVersion(s)); } } diff --git a/storm-client/src/jvm/org/apache/storm/utils/WindowedTimeThrottler.java b/storm-client/src/jvm/org/apache/storm/utils/WindowedTimeThrottler.java index 5b6d9b5eb8c..570505a5c52 100644 --- a/storm-client/src/jvm/org/apache/storm/utils/WindowedTimeThrottler.java +++ b/storm-client/src/jvm/org/apache/storm/utils/WindowedTimeThrottler.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -29,7 +35,7 @@ public boolean isThrottled() { return windowEvents >= maxAmt; } - //returns void if the event should continue, false if the event should not be done + // returns void if the event should continue, false if the event should not be done public void markEvent() { resetIfNecessary(); windowEvents++; diff --git a/storm-client/src/jvm/org/apache/storm/utils/WritableUtils.java b/storm-client/src/jvm/org/apache/storm/utils/WritableUtils.java index 2d964390c93..f98bdf052ca 100644 --- a/storm-client/src/jvm/org/apache/storm/utils/WritableUtils.java +++ b/storm-client/src/jvm/org/apache/storm/utils/WritableUtils.java @@ -40,7 +40,8 @@ public static byte[] readCompressedByteArray(DataInput in) throws IOException { } byte[] buffer = new byte[length]; in.readFully(buffer); // could/should use readFully(buffer,0,length)? - GZIPInputStream gzi = new GZIPInputStream(new ByteArrayInputStream(buffer, 0, buffer.length)); + GZIPInputStream gzi = new GZIPInputStream(new ByteArrayInputStream(buffer, 0, + buffer.length)); byte[] outbuf = new byte[length]; ByteArrayOutputStream bos = new ByteArrayOutputStream(); int len; @@ -89,7 +90,6 @@ public static String readCompressedString(DataInput in) throws IOException { return new String(bytes, "UTF-8"); } - public static int writeCompressedString(DataOutput out, String s) throws IOException { return writeCompressedByteArray(out, (s != null) ? s.getBytes("UTF-8") : null); } @@ -125,7 +125,6 @@ public static String readString(DataInput in) throws IOException { return new String(buffer, "UTF-8"); } - /** * Write a String array as a Nework Int N, followed by Int N Byte Array Strings. * Could be generalised using introspection. @@ -170,7 +169,6 @@ public static String[] readStringArray(DataInput in) throws IOException { return s; } - /** * Write a String array as a Nework Int N, followed by Int N Byte Array Strings. * Could be generalised using introspection. Handles null arrays and null values. @@ -187,7 +185,6 @@ public static String[] readCompressedStringArray(DataInput in) throws IOExceptio return s; } - /** * Test Utility Method Display Byte Array. */ @@ -206,7 +203,6 @@ public static void displayByteArray(byte[] record) { System.out.println(); } - /** * Serializes an integer to a binary stream with zero-compressed encoding. * For -120 <= i <= 127, only one byte is used with the actual value. @@ -268,9 +264,9 @@ public static void writeVLong(DataOutput stream, long i) throws IOException { } } - /** * Reads a zero-compressed encoded long from input stream and returns it. + * * @param stream Binary input stream * @return deserialized long from stream. */ @@ -291,6 +287,7 @@ public static long readVLong(DataInput stream) throws IOException { /** * Reads a zero-compressed encoded integer from input stream and returns it. + * * @param stream Binary input stream * @return deserialized integer from stream. */ @@ -300,6 +297,7 @@ public static int readVInt(DataInput stream) throws IOException { /** * Given the first byte of a vint/vlong, determine the sign. + * * @param value the first byte * @return is the value negative */ @@ -309,6 +307,7 @@ public static boolean isNegativeVInt(byte value) { /** * Parse the first byte of a vint/vlong to determine the number of bytes. + * * @param value the first byte of the vint/vlong * @return the total number of bytes (1 to 9) */ @@ -323,6 +322,7 @@ public static int decodeVIntSize(byte value) { /** * Get the encoded length if an integer is stored in a variable-length format. + * * @return the encoded length */ public static int getVIntSize(long i) { @@ -341,6 +341,7 @@ public static int getVIntSize(long i) { /** * Skip len number of bytes in input streamin. + * * @param in input stream * @param len number of bytes to skip * @throws IOException when skipped less number of bytes diff --git a/storm-client/src/jvm/org/apache/storm/utils/ZookeeperAuthInfo.java b/storm-client/src/jvm/org/apache/storm/utils/ZookeeperAuthInfo.java index e6954106839..42609fb505a 100644 --- a/storm-client/src/jvm/org/apache/storm/utils/ZookeeperAuthInfo.java +++ b/storm-client/src/jvm/org/apache/storm/utils/ZookeeperAuthInfo.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -16,7 +22,6 @@ import java.util.Map; import org.apache.storm.Config; - public class ZookeeperAuthInfo { public String scheme; public byte[] payload = null; diff --git a/storm-client/src/jvm/org/apache/storm/validation/ConfigValidation.java b/storm-client/src/jvm/org/apache/storm/validation/ConfigValidation.java index 4ebef6e4325..4fea5620bf8 100644 --- a/storm-client/src/jvm/org/apache/storm/validation/ConfigValidation.java +++ b/storm-client/src/jvm/org/apache/storm/validation/ConfigValidation.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -29,7 +35,6 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; - import org.apache.storm.Config; import org.apache.storm.utils.ObjectReader; import org.apache.storm.utils.Utils; @@ -42,13 +47,14 @@ */ public class ConfigValidation { private static final Logger LOG = LoggerFactory.getLogger(ConfigValidation.class); - //We follow the model of service loaders (Even though it is not a service). - private static final String CONFIG_CLASSES_NAME = "META-INF/services/" + Validated.class.getName(); + // We follow the model of service loaders (Even though it is not a service). + private static final String CONFIG_CLASSES_NAME = "META-INF/services/" + Validated.class + .getName(); /* * Validator definitions */ - //The following come from the JVm Specification table 4.4 + // The following come from the JVm Specification table 4.4 // https://docs.oracle.com/javase/specs/jvms/se7/html/jvms-4.html#jvms-4.5 private static final int ACC_PUBLIC = 0x0001; private static final int ACC_STATIC = 0x0008; @@ -63,7 +69,8 @@ public static synchronized List> getConfigClasses() { classesToScan.add(Config.class.getName()); for (URL url : Utils.findResources(CONFIG_CLASSES_NAME)) { try { - try (BufferedReader in = new BufferedReader(new InputStreamReader(url.openStream()))) { + try (BufferedReader in = new BufferedReader(new InputStreamReader(url + .openStream()))) { String line; while ((line = in.readLine()) != null) { line = line.replaceAll("#.*$", "").trim(); @@ -90,7 +97,7 @@ public static synchronized List> getConfigClasses() { } /** - * Validates a field given field name as string uses Config.java as the default config class + * Validates a field given field name as string uses Config.java as the default config class. * * @param fieldName provided as a string * @param conf map of confs @@ -106,13 +113,14 @@ public static void validateField(String fieldName, Map conf) { * @param conf map of confs * @param configs config class */ - public static void validateField(String fieldName, Map conf, List> configs) { + public static void validateField(String fieldName, Map conf, + List> configs) { Field field = null; for (Class clazz : configs) { try { field = clazz.getField(fieldName); } catch (NoSuchFieldException e) { - //Ignored + // Ignored } } if (field == null) { @@ -122,7 +130,8 @@ public static void validateField(String fieldName, Map conf, Lis } /** - * Validates a field given field. Calls correct ValidatorField method based on which fields are declared for the corresponding + * Validates a field given field. Calls correct ValidatorField method based on which fields are + * declared for the corresponding * annotation. * * @param field field that needs to be validated @@ -147,7 +156,7 @@ private static void validateField(Field field, Map conf, boolean String type = annotation.annotationType().getName(); Class validatorClass = null; Class[] classes = ConfigValidationAnnotations.class.getDeclaredClasses(); - //check if annotation is one of our + // check if annotation is one of our for (Class clazz : classes) { if (clazz.getName().equals(type)) { validatorClass = clazz; @@ -158,24 +167,29 @@ private static void validateField(Field field, Map conf, boolean Object v = validatorClass.cast(annotation); String key = (String) field.get(null); Class clazz = (Class) validatorClass - .getMethod(ConfigValidationAnnotations.ValidatorParams.VALIDATOR_CLASS).invoke(v); - //run each validator only in its phase, so cross-field rules are deferred to pass 2. + .getMethod(ConfigValidationAnnotations.ValidatorParams.VALIDATOR_CLASS) + .invoke(v); + // run each validator only in its phase, so cross-field rules are deferred to + // pass 2. boolean isCombo = ComboValidator.class.isAssignableFrom(clazz); if (isCombo != comboPhase) { continue; } Map params = getParamsFromAnnotation(validatorClass, v); - //two constructor signatures used to initialize validators. - //One constructor takes input a Map of arguments, the other doesn't take any arguments (default constructor) - //If validator has a constructor that takes a Map as an argument call that constructor + // two constructor signatures used to initialize validators. + // One constructor takes input a Map of arguments, the other doesn't take any + // arguments (default constructor) + // If validator has a constructor that takes a Map as an argument call that + // constructor Object o; if (hasConstructor(clazz, Map.class)) { o = clazz.getConstructor(Map.class).newInstance(params); - } else { //If not call default constructor + } else { // If not call default constructor o = clazz.newInstance(); } if (isCombo) { - //cross-field rule: pass the whole conf, keyed off the annotated field's name. + // cross-field rule: pass the whole conf, keyed off the annotated field's + // name. ((ComboValidator) o).validateComboFields(conf); } else { ((Validator) o).validateField(field.getName(), conf.get(key)); @@ -189,6 +203,7 @@ private static void validateField(Field field, Map conf, boolean /** * Validate topology conf. + * * @param topoConf The topology conf. */ public static void validateTopoConf(Map topoConf) { @@ -221,20 +236,22 @@ public static void validateFields(Map conf, List> class try { keyObj = field.get(null); } catch (IllegalAccessException e) { - //This should not happen because we checked for PUBLIC in isFieldAllowed + // This should not happen because we checked for PUBLIC in isFieldAllowed throw new RuntimeException(e); } - //make sure that defined key is string in case wrong stuff got put into Config.java + // make sure that defined key is string in case wrong stuff got put into Config.java if (keyObj instanceof String && conf.containsKey((String) keyObj)) { presentFields.add(field); } } } - //Pass 1: validate each present field on its own, so the whole conf is individually valid first. + // Pass 1: validate each present field on its own, so the whole conf is individually valid + // first. for (Field field : presentFields) { validateField(field, conf, false); } - //Pass 2: cross-field rules, run only after pass 1 so a ComboValidator sees an already-valid conf. + // Pass 2: cross-field rules, run only after pass 1 so a ComboValidator sees an + // already-valid conf. for (Field field : presentFields) { validateField(field, conf, true); } @@ -243,7 +260,8 @@ public static void validateFields(Map conf, List> class public static boolean isFieldAllowed(Field field) { return field.getAnnotation(NotConf.class) == null && String.class.equals(field.getType()) - && ((field.getModifiers() & DESIRED_FIELD_ACC) == DESIRED_FIELD_ACC) && !field.isSynthetic(); + && ((field.getModifiers() & DESIRED_FIELD_ACC) == DESIRED_FIELD_ACC) && !field + .isSynthetic(); } private static Map getParamsFromAnnotation(Class validatorClass, Object v) @@ -303,7 +321,8 @@ public static class NotNullValidator extends Validator { @Override public void validateField(String name, Object o) { if (o == null) { - throw new IllegalArgumentException("Field " + name + "cannot be null! Actual value: " + o); + throw new IllegalArgumentException("Field " + name + + "cannot be null! Actual value: " + o); } } } @@ -327,7 +346,8 @@ public static void validateField(String name, Class type, Object o) { return; } throw new IllegalArgumentException( - "Field " + name + " must be of type " + type + ". Object: " + o + " actual type: " + o.getClass()); + "Field " + name + " must be of type " + type + ". Object: " + o + " actual type: " + + o.getClass()); } @Override @@ -357,7 +377,8 @@ public static void validateField(String name, Class baseType, Object actualTy return; } throw new IllegalArgumentException( - "Field " + name + " must represent a type that derives from '" + baseType + "'. Specified type = " + actualTypeName); + "Field " + name + " must represent a type that derives from '" + baseType + + "'. Specified type = " + actualTypeName); } catch (ClassNotFoundException e) { throw new IllegalArgumentException(e.getMessage()); } @@ -379,9 +400,11 @@ public StringValidator() { public StringValidator(Map params) { this.acceptedValues = - new HashSet(Arrays.asList((String[]) params.get(ConfigValidationAnnotations.ValidatorParams.ACCEPTED_VALUES))); + new HashSet(Arrays.asList((String[]) params + .get(ConfigValidationAnnotations.ValidatorParams.ACCEPTED_VALUES))); - if (this.acceptedValues.isEmpty() || (this.acceptedValues.size() == 1 && this.acceptedValues.contains(""))) { + if (this.acceptedValues.isEmpty() || (this.acceptedValues.size() == 1 + && this.acceptedValues.contains(""))) { this.acceptedValues = null; } } @@ -392,7 +415,8 @@ public void validateField(String name, Object o) { if (this.acceptedValues != null) { if (!this.acceptedValues.contains((String) o)) { throw new IllegalArgumentException( - "Field " + name + " is not an accepted value. Value: " + o + " Accepted values: " + this.acceptedValues); + "Field " + name + " is not an accepted value. Value: " + o + + " Accepted values: " + this.acceptedValues); } } } @@ -443,7 +467,8 @@ public void validateInteger(String name, Object o) { return; } } - throw new IllegalArgumentException("Field " + name + " must be an Integer within type range."); + throw new IllegalArgumentException("Field " + name + + " must be an Integer within type range."); } } @@ -463,7 +488,8 @@ public void validateLong(String name, Object o) { if (o instanceof Number && ((Number) o).longValue() == ((Number) o).doubleValue()) { return; } - throw new IllegalArgumentException("Field " + name + " must be an Long within type range."); + throw new IllegalArgumentException("Field " + name + + " must be an Long within type range."); } } @@ -479,21 +505,24 @@ public void validateField(String name, Object o) { } ConfigValidationUtils.NestableFieldValidator validator = ConfigValidationUtils.mapFv(ConfigValidationUtils.fv(String.class, false), - ConfigValidationUtils.listFv(String.class, false), false); + ConfigValidationUtils.listFv(String.class, + false), false); validator.validateField(name, o); @SuppressWarnings("unchecked") Map> mapObject = (Map>) o; if (!mapObject.containsKey("hosts")) { - throw new IllegalArgumentException(name + " should contain Map entry with key: hosts"); + throw new IllegalArgumentException(name + + " should contain Map entry with key: hosts"); } if (!mapObject.containsKey("groups")) { - throw new IllegalArgumentException(name + " should contain Map entry with key: groups"); + throw new IllegalArgumentException(name + + " should contain Map entry with key: groups"); } } } /** - * validates a list of has no duplicates. + * Validates a list of has no duplicates. */ public static class NoDuplicateInListValidator extends Validator { @@ -502,12 +531,13 @@ public void validateField(String name, Object field) { if (field == null) { return; } - //check if iterable + // check if iterable SimpleTypeValidator.validateField(name, Iterable.class, field); HashSet objectSet = new HashSet(); for (Object o : (Iterable) field) { if (objectSet.contains(o)) { - throw new IllegalArgumentException(name + " should contain no duplicate elements. Duplicated element: " + o); + throw new IllegalArgumentException(name + + " should contain no duplicate elements. Duplicated element: " + o); } objectSet.add(o); } @@ -519,7 +549,8 @@ public void validateField(String name, Object field) { */ public static class StringOrStringListValidator extends Validator { - private ConfigValidationUtils.FieldValidator fv = ConfigValidationUtils.listFv(String.class, false); + private ConfigValidationUtils.FieldValidator fv = ConfigValidationUtils.listFv(String.class, + false); @Override public void validateField(String name, Object o) { @@ -546,22 +577,26 @@ public void validateField(String name, Object o) { if (o instanceof Iterable) { for (Object e : (Iterable) o) { if (e instanceof Map) { - for (Map.Entry entry : ((Map) e).entrySet()) { + for (Map.Entry entry : ((Map) e) + .entrySet()) { if (!(entry.getKey() instanceof String) || !(entry.getValue() instanceof String)) { throw new IllegalArgumentException( - "Each element of the list " + name + " must be a String or a Map of Strings"); + "Each element of the list " + name + + " must be a String or a Map of Strings"); } } } else if (!(e instanceof String)) { throw new IllegalArgumentException( - "Each element of the list " + name + " must be a String or a Map of Strings"); + "Each element of the list " + name + + " must be a String or a Map of Strings"); } } return; } throw new IllegalArgumentException( - "Field " + name + " must be an Iterable containing only Strings or Maps of Strings"); + "Field " + name + + " must be an Iterable containing only Strings or Maps of Strings"); } } @@ -610,7 +645,8 @@ public void validateField(String name, Object o) { ObjectInputFilter.Config.createFilter(pattern); } catch (IllegalArgumentException e) { throw new IllegalArgumentException( - "Field " + name + " is not a valid JEP-290 serial-filter pattern: '" + pattern + "'", e); + "Field " + name + " is not a valid JEP-290 serial-filter pattern: '" + pattern + + "'", e); } } } @@ -627,7 +663,8 @@ public ListEntryTypeValidator(Map params) { } public static void validateField(String name, Class type, Object o) { - ConfigValidationUtils.NestableFieldValidator validator = ConfigValidationUtils.listFv(type, false); + ConfigValidationUtils.NestableFieldValidator validator = ConfigValidationUtils + .listFv(type, false); validator.validateField(name, o); } @@ -638,7 +675,8 @@ public void validateField(String name, Object o) { } /** - * Validates each entry in a list against a list of custom Validators. Each validator in the list of validators must inherit or be an + * Validates each entry in a list against a list of custom Validators. Each validator in the + * list of validators must inherit or be an * instance of Validator class */ public static class ListEntryCustomValidator extends Validator { @@ -646,7 +684,8 @@ public static class ListEntryCustomValidator extends Validator { private Class[] entryValidators; public ListEntryCustomValidator(Map params) { - this.entryValidators = (Class[]) params.get(ConfigValidationAnnotations.ValidatorParams.ENTRY_VALIDATOR_CLASSES); + this.entryValidators = (Class[]) params + .get(ConfigValidationAnnotations.ValidatorParams.ENTRY_VALIDATOR_CLASSES); } public static void validateField(String name, Class[] validators, Object o) @@ -654,7 +693,7 @@ public static void validateField(String name, Class[] validators, Object o) if (o == null) { return; } - //check if iterable + // check if iterable SimpleTypeValidator.validateField(name, Iterable.class, o); for (Object entry : (Iterable) o) { for (Class validator : validators) { @@ -664,7 +703,8 @@ public static void validateField(String name, Class[] validators, Object o) } else { LOG.warn( "validator: {} cannot be used in ListEntryCustomValidator. " - + "Individual entry validators must be an instance of Validator class", + + "Individual entry validators must be an instance of " + + "Validator class", validator.getName()); } } @@ -682,7 +722,7 @@ public void validateField(String name, Object o) { } /** - * validates each key and value in a map of a certain type. + * Validates each key and value in a map of a certain type. */ public static class MapEntryTypeValidator extends Validator { @@ -690,12 +730,16 @@ public static class MapEntryTypeValidator extends Validator { private Class valueType; public MapEntryTypeValidator(Map params) { - this.keyType = (Class) params.get(ConfigValidationAnnotations.ValidatorParams.KEY_TYPE); - this.valueType = (Class) params.get(ConfigValidationAnnotations.ValidatorParams.VALUE_TYPE); + this.keyType = (Class) params + .get(ConfigValidationAnnotations.ValidatorParams.KEY_TYPE); + this.valueType = (Class) params + .get(ConfigValidationAnnotations.ValidatorParams.VALUE_TYPE); } - public static void validateField(String name, Class keyType, Class valueType, Object o) { - ConfigValidationUtils.NestableFieldValidator validator = ConfigValidationUtils.mapFv(keyType, valueType, false); + public static void validateField(String name, Class keyType, Class valueType, + Object o) { + ConfigValidationUtils.NestableFieldValidator validator = ConfigValidationUtils + .mapFv(keyType, valueType, false); validator.validateField(name, o); } @@ -706,7 +750,7 @@ public void validateField(String name, Object o) { } /** - * validates each key and each value against the respective arrays of validators. + * Validates each key and each value against the respective arrays of validators. */ public static class MapEntryCustomValidator extends Validator { @@ -714,17 +758,20 @@ public static class MapEntryCustomValidator extends Validator { private Class[] valueValidators; public MapEntryCustomValidator(Map params) { - this.keyValidators = (Class[]) params.get(ConfigValidationAnnotations.ValidatorParams.KEY_VALIDATOR_CLASSES); - this.valueValidators = (Class[]) params.get(ConfigValidationAnnotations.ValidatorParams.VALUE_VALIDATOR_CLASSES); + this.keyValidators = (Class[]) params + .get(ConfigValidationAnnotations.ValidatorParams.KEY_VALIDATOR_CLASSES); + this.valueValidators = (Class[]) params + .get(ConfigValidationAnnotations.ValidatorParams.VALUE_VALIDATOR_CLASSES); } @SuppressWarnings("unchecked") - public static void validateField(String name, Class[] keyValidators, Class[] valueValidators, Object o) + public static void validateField(String name, Class[] keyValidators, + Class[] valueValidators, Object o) throws IllegalAccessException, InstantiationException, NoSuchMethodException, InvocationTargetException { if (o == null) { return; } - //check if Map + // check if Map SimpleTypeValidator.validateField(name, Map.class, o); for (Map.Entry entry : ((Map) o).entrySet()) { for (Class kv : keyValidators) { @@ -733,19 +780,24 @@ public static void validateField(String name, Class[] keyValidators, Class ((Validator) keyValidator).validateField(name + " Map key", entry.getKey()); } else { LOG.warn( - "validator: {} cannot be used in MapEntryCustomValidator to validate keys. " - + "Individual entry validators must be an instance of Validator class", + "validator: {} cannot be used in MapEntryCustomValidator to validate " + + "keys. " + + "Individual entry validators must be an instance of " + + "Validator class", kv.getName()); } } for (Class vv : valueValidators) { Object valueValidator = vv.getConstructor().newInstance(); if (valueValidator instanceof Validator) { - ((Validator) valueValidator).validateField(name + " Map value", entry.getValue()); + ((Validator) valueValidator).validateField(name + " Map value", entry + .getValue()); } else { LOG.warn( - "validator: {} cannot be used in MapEntryCustomValidator to validate values. " - + "Individual entry validators must be an instance of Validator class", + "validator: {} cannot be used in MapEntryCustomValidator to validate " + + "values. " + + "Individual entry validators must be an instance of " + + "Validator class", vv.getName()); } } @@ -774,7 +826,8 @@ public PositiveNumberValidator() { } public PositiveNumberValidator(Map params) { - this.includeZero = (boolean) params.get(ConfigValidationAnnotations.ValidatorParams.INCLUDE_ZERO); + this.includeZero = (boolean) params + .get(ConfigValidationAnnotations.ValidatorParams.INCLUDE_ZERO); } public static void validateField(String name, boolean includeZero, Object o) { @@ -810,7 +863,8 @@ public void validateField(String name, Object o) { } SimpleTypeValidator.validateField(name, Map.class, o); if (!((Map) o).containsKey("class")) { - throw new IllegalArgumentException("Field " + name + " must have map entry with key: class"); + throw new IllegalArgumentException("Field " + name + + " must have map entry with key: class"); } SimpleTypeValidator.validateField(name, String.class, ((Map) o).get("class")); @@ -826,10 +880,12 @@ public void validateField(String name, Object o) { } SimpleTypeValidator.validateField(name, Map.class, o); if (!((Map) o).containsKey("class")) { - throw new IllegalArgumentException("Field " + name + " must have map entry with key: class"); + throw new IllegalArgumentException("Field " + name + + " must have map entry with key: class"); } if (!((Map) o).containsKey("parallelism.hint")) { - throw new IllegalArgumentException("Field " + name + " must have map entry with key: parallelism.hint"); + throw new IllegalArgumentException("Field " + name + + " must have map entry with key: parallelism.hint"); } SimpleTypeValidator.validateField(name, String.class, ((Map) o).get("class")); @@ -848,7 +904,8 @@ public void validateField(String name, Object o) { } SimpleTypeValidator.validateField(name, Map.class, o); if (!((Map) o).containsKey(CLASS)) { - throw new IllegalArgumentException("Field " + name + " must have map entry with key: class"); + throw new IllegalArgumentException("Field " + name + + " must have map entry with key: class"); } if (((Map) o).containsKey(FILTER)) { Map filterMap = (Map) ((Map) o).get(FILTER); @@ -871,7 +928,8 @@ public void validateField(String name, Object o) { int level = (Integer) o; if (level < MIN_LEVEL || level > MAX_LEVEL) { throw new IllegalArgumentException( - String.format("Field '%s' is invalid: %d. Zstd compression level must be between %d and %d.", + String.format("Field '%s' is invalid: %d. Zstd compression level must be " + + "between %d and %d.", name, level, MIN_LEVEL, MAX_LEVEL) ); } @@ -887,13 +945,15 @@ public void validateField(String name, Object o) { } SimpleTypeValidator.validateField(name, Map.class, o); if (!((Map) o).containsKey("class")) { - throw new IllegalArgumentException("Field " + name + " must have map entry with key: class"); + throw new IllegalArgumentException("Field " + name + + " must have map entry with key: class"); } SimpleTypeValidator.validateField(name, String.class, ((Map) o).get("class")); if (((Map) o).containsKey("arguments")) { - SimpleTypeValidator.validateField(name, Map.class, ((Map) o).get("arguments")); + SimpleTypeValidator.validateField(name, Map.class, ((Map) o) + .get("arguments")); } } } @@ -903,7 +963,8 @@ public static class MapOfStringToMapOfStringToObjectValidator extends Validator public void validateField(String name, Object o) { ConfigValidationUtils.NestableFieldValidator validator = ConfigValidationUtils.mapFv(ConfigValidationUtils.fv(String.class, false), - ConfigValidationUtils.mapFv(String.class, Object.class, true), true); + ConfigValidationUtils.mapFv(String.class, Object.class, + true), true); validator.validateField(name, o); } } @@ -920,7 +981,8 @@ public void validateField(String name, Object o) { || ((String) o).equals("KERBEROS"))) { return; } - throw new IllegalArgumentException("Field " + name + " must be one of \"NONE\", \"DIGEST\", or \"KERBEROS\""); + throw new IllegalArgumentException("Field " + name + + " must be one of \"NONE\", \"DIGEST\", or \"KERBEROS\""); } } @@ -946,8 +1008,10 @@ public static class CustomIsExactlyOneOfValidators extends Validator { private List validatorClassNames; public CustomIsExactlyOneOfValidators(Map params) { - this.subValidators = (Class[]) params.get(ConfigValidationAnnotations.ValidatorParams.VALUE_VALIDATOR_CLASSES); - this.validatorClassNames = Arrays.asList(subValidators).stream().map(x -> x.getName()).collect(Collectors.toList()); + this.subValidators = (Class[]) params + .get(ConfigValidationAnnotations.ValidatorParams.VALUE_VALIDATOR_CLASSES); + this.validatorClassNames = Arrays.asList(subValidators).stream().map(x -> x.getName()) + .collect(Collectors.toList()); } @Override @@ -967,35 +1031,45 @@ public void validateField(String name, Object o) { } if (valueValidator instanceof Validator) { try { - ((Validator) valueValidator).validateField(name + " " + vv.getSimpleName() + " value", o); + ((Validator) valueValidator).validateField(name + " " + vv.getSimpleName() + + " value", o); selectedValidators.add(vv.getName()); } catch (Exception ex) { - // only one will pass, so ignore all validation errors - stored for future use + // only one will pass, so ignore all validation errors - stored for future + // use validatorExceptions.put(vv.getName(), ex); } } else { - String err = String.format("validator: %s cannot be used in CustomExactlyOneOfValidators to validate values. " - + "Individual entry validators must a instance of Validator class", vv.getName()); + String err = String + .format("validator: %s cannot be used in CustomExactlyOneOfValidators " + + "to validate values. " + + "Individual entry validators must a instance of Validator class", vv + .getName()); LOG.warn(err); } } // check if one and only one validation succeeded if (selectedValidators.isEmpty()) { String parseErrs = String.join(";\n\t", validatorExceptions.entrySet().stream() - .map(e -> String.format("%s:%s", e.getKey(), e.getValue())).collect(Collectors.toList())); - String err = String.format("Field %s must be one of %s; parse errors are \n\t%s", name, + .map(e -> String.format("%s:%s", e.getKey(), e.getValue())) + .collect(Collectors.toList())); + String err = String.format("Field %s must be one of %s; parse errors are \n\t%s", + name, String.join(", ", validatorClassNames), parseErrs); throw new IllegalArgumentException(err); } if (selectedValidators.size() > 1) { - throw new IllegalArgumentException("Field " + name + " must match exactly one of " + String.join(", ", selectedValidators)); + throw new IllegalArgumentException("Field " + name + " must match exactly one of " + + String.join(", ", selectedValidators)); } } } public static class RasConstraintsTypeValidator extends Validator { - public static final String CONSTRAINT_TYPE_MAX_NODE_CO_LOCATION_CNT = "maxNodeCoLocationCnt"; - public static final String CONSTRAINT_TYPE_INCOMPATIBLE_COMPONENTS = "incompatibleComponents"; + public static final String CONSTRAINT_TYPE_MAX_NODE_CO_LOCATION_CNT = + "maxNodeCoLocationCnt"; + public static final String CONSTRAINT_TYPE_INCOMPATIBLE_COMPONENTS = + "incompatibleComponents"; @Override public void validateField(String name, Object o) { @@ -1011,7 +1085,9 @@ public void validateField(String name, Object o) { String comp1 = entry1.getKey(); Object o2 = entry1.getValue(); if (!(o2 instanceof Map)) { - String err = String.format("Field %s, component %s, expecting constraints Map with keys [\"%s\", \"%s\"], in \"%s\"", + String err = String + .format("Field %s, component %s, expecting constraints Map with keys " + + "[\"%s\", \"%s\"], in \"%s\"", name, comp1, CONSTRAINT_TYPE_MAX_NODE_CO_LOCATION_CNT, CONSTRAINT_TYPE_INCOMPATIBLE_COMPONENTS, o); throw new IllegalArgumentException(err); } @@ -1024,7 +1100,9 @@ public void validateField(String name, Object o) { try { Integer.parseInt("" + o3); } catch (Exception ex) { - String err = String.format("Field %s, component %s, constraint %s should be a number, not \"%s\"", + String err = String + .format("Field %s, component %s, constraint %s should be " + + "a number, not \"%s\"", name, comp1, constraintType, o3); throw new IllegalArgumentException(err); } @@ -1039,9 +1117,11 @@ public void validateField(String name, Object o) { continue; } String err = String.format( - "Field %s, component %s, constraintType \"%s\", expecting incompatible component-name, " + "Field %s, component %s, constraintType \"%s\", " + + "expecting incompatible component-name, " + "found instance of class \"%s\" value \"%s\"", - name, comp1, constraintType, o3.getClass().getName(), otherComp); + name, comp1, constraintType, o3.getClass() + .getName(), otherComp); throw new IllegalArgumentException(err); } } @@ -1049,7 +1129,8 @@ public void validateField(String name, Object o) { default: String err = String.format( - "Field %s, component %s, has unsupported constraintType \"%s\", expecting one of [\"%s\", \"%s\"], " + "Field %s, component %s, has unsupported constraintType " + + "\"%s\", expecting one of [\"%s\", \"%s\"], " + "in \"%s\"", name, comp1, constraintType, CONSTRAINT_TYPE_MAX_NODE_CO_LOCATION_CNT, CONSTRAINT_TYPE_INCOMPATIBLE_COMPONENTS, o); @@ -1070,10 +1151,12 @@ public void validateField(String name, Object o) { SimpleTypeValidator.validateField(name, Map.class, o); Map m = (Map) o; if (!m.containsKey("cpu")) { - throw new IllegalArgumentException("Field " + name + " must have map entry with key: cpu"); + throw new IllegalArgumentException("Field " + name + + " must have map entry with key: cpu"); } if (!m.containsKey("memory")) { - throw new IllegalArgumentException("Field " + name + " must have map entry with key: memory"); + throw new IllegalArgumentException("Field " + name + + " must have map entry with key: memory"); } SimpleTypeValidator.validateField(name, Number.class, m.get("cpu")); @@ -1086,7 +1169,8 @@ public static class ImplementsClassValidator extends Validator { Class classImplements; public ImplementsClassValidator(Map params) { - this.classImplements = (Class) params.get(ConfigValidationAnnotations.ValidatorParams.IMPLEMENTS_CLASS); + this.classImplements = (Class) params + .get(ConfigValidationAnnotations.ValidatorParams.IMPLEMENTS_CLASS); } @Override @@ -1100,18 +1184,23 @@ public void validateField(String name, Object o) { Class objectClass = Class.forName(className); if (!this.classImplements.isAssignableFrom(objectClass)) { throw new IllegalArgumentException("Field " + name + " with value " + o - + " does not implement " + this.classImplements.getName()); + + " does not implement " + + this.classImplements.getName()); } } catch (ClassNotFoundException e) { - //To support topologies of older version to run, we might have to loose the constraints so that - //the configs of older version can pass the validation. + // To support topologies of older version to run, we might have to loose the + // constraints so that + // the configs of older version can pass the validation. if (className.startsWith("backtype.storm")) { LOG.warn("ClassNotFoundException: {}", className); - LOG.warn("Replace backtype.storm with org.apache.storm and try to validate again"); - LOG.warn("We loosen some constraints here to support topologies of older version running on the current version"); + LOG.warn("Replace backtype.storm with org.apache.storm and try to validate " + + "again"); + LOG.warn("We loosen some constraints here to support topologies of older " + + "version running on the current version"); validateField(name, className.replace("backtype.storm", "org.apache.storm")); } else { - throw new RuntimeException("Failed to validate config " + name + " with value " + className, e); + throw new RuntimeException("Failed to validate config " + name + " with value " + + className, e); } } } @@ -1133,42 +1222,53 @@ public void validateField(String name, Object o) throws IllegalArgumentException for (Object entry2 : (List) entry1) { if (!(entry2 instanceof String)) { throw new IllegalArgumentException( - "Field " + name + " must be an Iterable containing only List of List of Strings"); + "Field " + name + + " must be an Iterable containing only List of List of " + + "Strings"); } } } else { throw new IllegalArgumentException( - "Field " + name + " must be an Iterable containing only List of List of Strings"); + "Field " + name + + " must be an Iterable containing only List of List of Strings"); } } } else { throw new IllegalArgumentException( - "Field " + name + " must be an Iterable containing only List of List of Strings"); + "Field " + name + + " must be an Iterable containing only List of List of Strings"); } } } - /** - * Rejects enabling upstream feedback without EWMA stats. The feedback loop only carries EWMA jitter - * statistics, which are populated solely when {@link Config#TOPOLOGY_STATS_EWMA_ENABLE} is on; with + * Rejects enabling upstream feedback without EWMA stats. The feedback loop only carries EWMA + * jitter + * statistics, which are populated solely when {@link Config#TOPOLOGY_STATS_EWMA_ENABLE} is on; + * with * {@link Config#TOPOLOGY_UPSTREAM_FEEDBACK_ENABLE} enabled but EWMA left at its default ({@code false}), - * every feedback record is empty and a jitter-aware grouping silently degrades to its load-aware + * every feedback record is empty and a jitter-aware grouping silently degrades to its + * load-aware * fallback forever. */ public static class UpstreamFeedbackValidator extends ComboValidator { @Override public void validateComboFields(Map conf) { boolean feedbackEnabled = - ObjectReader.getBoolean(conf.get(Config.TOPOLOGY_UPSTREAM_FEEDBACK_ENABLE), false); + ObjectReader.getBoolean(conf.get(Config.TOPOLOGY_UPSTREAM_FEEDBACK_ENABLE), + false); boolean ewmaEnabled = ObjectReader.getBoolean(conf.get(Config.TOPOLOGY_STATS_EWMA_ENABLE), false); if (feedbackEnabled && !ewmaEnabled) { throw new IllegalArgumentException( - Config.TOPOLOGY_UPSTREAM_FEEDBACK_ENABLE + " requires " + Config.TOPOLOGY_STATS_EWMA_ENABLE - + "=true: the feedback loop only carries EWMA jitter stats, which are produced solely " - + "when EWMA is enabled. Enable " + Config.TOPOLOGY_STATS_EWMA_ENABLE + ", or disable " - + Config.TOPOLOGY_UPSTREAM_FEEDBACK_ENABLE + " (it is otherwise a no-op)."); + Config.TOPOLOGY_UPSTREAM_FEEDBACK_ENABLE + " requires " + + Config.TOPOLOGY_STATS_EWMA_ENABLE + + "=true: the feedback loop only carries EWMA jitter stats, which " + + "are produced solely " + + "when EWMA is enabled. Enable " + + Config.TOPOLOGY_STATS_EWMA_ENABLE + ", or disable " + + Config.TOPOLOGY_UPSTREAM_FEEDBACK_ENABLE + + " (it is otherwise a no-op)."); } } } diff --git a/storm-client/src/jvm/org/apache/storm/validation/ConfigValidationAnnotations.java b/storm-client/src/jvm/org/apache/storm/validation/ConfigValidationAnnotations.java index 79633606528..271d9ff150a 100644 --- a/storm-client/src/jvm/org/apache/storm/validation/ConfigValidationAnnotations.java +++ b/storm-client/src/jvm/org/apache/storm/validation/ConfigValidationAnnotations.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -18,10 +24,14 @@ import java.lang.annotation.Target; /** - * Note: every annotation interface must have method `validatorClass()` For every annotation there must validator class to do the validation - * To add another annotation for config validation, add another annotation @interface class. Implement the corresponding validator logic in - * a class in ConfigValidation. Make sure validateField method in ConfigValidation knows how to use the validator and which method - * definition/parameters to pass in based on what fields are in the annotation. By default, params of annotations will be passed into a + * Note: every annotation interface must have method `validatorClass()` For every annotation there + * must validator class to do the validation + * To add another annotation for config validation, add another annotation @interface class. + * Implement the corresponding validator logic in + * a class in ConfigValidation. Make sure validateField method in ConfigValidation knows how to use + * the validator and which method + * definition/parameters to pass in based on what fields are in the annotation. By default, params + * of annotations will be passed into a * constructor that takes a Map as a parameter. */ public class ConfigValidationAnnotations { @@ -53,7 +63,7 @@ public class ConfigValidationAnnotations { } /** - * validates each entry in a list is of a certain type. + * Validates each entry in a list is of a certain type. */ @Retention(RetentionPolicy.RUNTIME) @Target(ElementType.FIELD) @@ -111,7 +121,8 @@ public class ConfigValidationAnnotations { } /** - * Validates each entry in a list with a list of validators Validators with fields: validatorClass and entryValidatorClass. + * Validates each entry in a list with a list of validators Validators with fields: + * validatorClass and entryValidatorClass. */ @Retention(RetentionPolicy.RUNTIME) @Target(ElementType.FIELD) @@ -129,7 +140,8 @@ public class ConfigValidationAnnotations { /** - * Validates the type of each key and value in a map Validator with fields: validatorClass, keyValidatorClass, valueValidatorClass. + * Validates the type of each key and value in a map Validator with fields: validatorClass, + * keyValidatorClass, valueValidatorClass. */ @Retention(RetentionPolicy.RUNTIME) @Target(ElementType.FIELD) @@ -142,7 +154,8 @@ public class ConfigValidationAnnotations { } /** - * Validates a each key and value in a Map with a list of validators Validator with fields: validatorClass, keyValidatorClasses, + * Validates a each key and value in a Map with a list of validators Validator with fields: + * validatorClass, keyValidatorClasses, * valueValidatorClasses. */ @Retention(RetentionPolicy.RUNTIME) @@ -156,7 +169,8 @@ public class ConfigValidationAnnotations { } /** - * Checks if a number is positive and whether zero inclusive Validator with fields: validatorClass, includeZero. + * Checks if a number is positive and whether zero inclusive Validator with fields: + * validatorClass, includeZero. */ @Retention(RetentionPolicy.RUNTIME) @Target(ElementType.FIELD) @@ -205,9 +219,12 @@ public class ConfigValidationAnnotations { } /** - * For custom cross-field validators. Unlike {@link CustomValidator}, whose validator receives a single - * field's value, the referenced {@link ConfigValidation.ComboValidator} receives the whole configuration - * so it can enforce dependencies between keys. Place it on the field whose presence triggers the rule. + * For custom cross-field validators. Unlike {@link CustomValidator}, whose validator receives a + * single + * field's value, the referenced {@link ConfigValidation.ComboValidator} receives the whole + * configuration + * so it can enforce dependencies between keys. Place it on the field whose presence triggers + * the rule. */ @Retention(RetentionPolicy.RUNTIME) @Target(ElementType.FIELD) diff --git a/storm-client/src/jvm/org/apache/storm/validation/ConfigValidationUtils.java b/storm-client/src/jvm/org/apache/storm/validation/ConfigValidationUtils.java index 21d2cde1d38..138b4367bab 100644 --- a/storm-client/src/jvm/org/apache/storm/validation/ConfigValidationUtils.java +++ b/storm-client/src/jvm/org/apache/storm/validation/ConfigValidationUtils.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -62,7 +68,8 @@ public static NestableFieldValidator listFv(Class cls, boolean notNull) { * * @param validator used to validate each item in the list * @param notNull whether or not a value of null is valid - * @return a NestableFieldValidator for a list with each item validated by a different validator. + * @return a NestableFieldValidator for a list with each item validated by a different + * validator. */ public static NestableFieldValidator listFv(final NestableFieldValidator validator, final boolean notNull) { @@ -128,7 +135,8 @@ public void validateField(String pd, String name, Object field) } } if (field instanceof Map) { - for (Map.Entry entry : ((Map) field).entrySet()) { + for (Map.Entry entry : ((Map) field) + .entrySet()) { key.validateField("Each key of the map ", name, entry.getKey()); val.validateField("Each value in the map ", name, entry.getValue()); } @@ -171,6 +179,7 @@ public void validateField(String name, Object field) throws IllegalArgumentExcep * @param field The field to be validated. * @throws IllegalArgumentException if the field fails validation. */ - public abstract void validateField(String pd, String name, Object field) throws IllegalArgumentException; + public abstract void validateField(String pd, String name, + Object field) throws IllegalArgumentException; } } diff --git a/storm-client/src/jvm/org/apache/storm/validation/NotConf.java b/storm-client/src/jvm/org/apache/storm/validation/NotConf.java index 50063f1771a..6bbde52c05c 100644 --- a/storm-client/src/jvm/org/apache/storm/validation/NotConf.java +++ b/storm-client/src/jvm/org/apache/storm/validation/NotConf.java @@ -1,19 +1,25 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

      * http://www.apache.org/licenses/LICENSE-2.0 *

      - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.validation; /** - * Annotation that can be used to explicitly call out public static final String fields that are not configs. + * Annotation that can be used to explicitly call out public static final String fields that are not + * configs. */ public @interface NotConf { diff --git a/storm-client/src/jvm/org/apache/storm/validation/Validated.java b/storm-client/src/jvm/org/apache/storm/validation/Validated.java index b19af76c850..4b6afea489f 100644 --- a/storm-client/src/jvm/org/apache/storm/validation/Validated.java +++ b/storm-client/src/jvm/org/apache/storm/validation/Validated.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -16,5 +22,5 @@ * An interface that is used to inform config validation what to look at. */ public interface Validated { - //Empty + // Empty } diff --git a/storm-client/src/jvm/org/apache/storm/windowing/CountEvictionPolicy.java b/storm-client/src/jvm/org/apache/storm/windowing/CountEvictionPolicy.java index 910b600eecb..4427e9aea28 100644 --- a/storm-client/src/jvm/org/apache/storm/windowing/CountEvictionPolicy.java +++ b/storm-client/src/jvm/org/apache/storm/windowing/CountEvictionPolicy.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/windowing/CountTriggerPolicy.java b/storm-client/src/jvm/org/apache/storm/windowing/CountTriggerPolicy.java index 096a02c3d99..83e9da97a3d 100644 --- a/storm-client/src/jvm/org/apache/storm/windowing/CountTriggerPolicy.java +++ b/storm-client/src/jvm/org/apache/storm/windowing/CountTriggerPolicy.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -15,7 +21,8 @@ import java.util.concurrent.atomic.AtomicInteger; /** - * A trigger that tracks event counts and calls back {@link TriggerHandler#onTrigger()} when the count threshold is hit. + * A trigger that tracks event counts and calls back {@link TriggerHandler#onTrigger()} when the + * count threshold is hit. * * @param the type of event tracked by this policy. */ @@ -26,7 +33,8 @@ public class CountTriggerPolicy implements TriggerPolicy { private final EvictionPolicy evictionPolicy; private boolean started; - public CountTriggerPolicy(int count, TriggerHandler handler, EvictionPolicy evictionPolicy) { + public CountTriggerPolicy(int count, TriggerHandler handler, EvictionPolicy evictionPolicy) { this.count = count; this.currentCount = new AtomicInteger(); this.handler = handler; diff --git a/storm-client/src/jvm/org/apache/storm/windowing/DefaultEvictionContext.java b/storm-client/src/jvm/org/apache/storm/windowing/DefaultEvictionContext.java index 6f432da3c7b..2ab720d8745 100644 --- a/storm-client/src/jvm/org/apache/storm/windowing/DefaultEvictionContext.java +++ b/storm-client/src/jvm/org/apache/storm/windowing/DefaultEvictionContext.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -30,7 +36,8 @@ public DefaultEvictionContext(Long referenceTime, Long currentCount, Long slidin this(referenceTime, currentCount, slidingCount, null); } - public DefaultEvictionContext(Long referenceTime, Long currentCount, Long slidingCount, Long slidingInterval) { + public DefaultEvictionContext(Long referenceTime, Long currentCount, Long slidingCount, + Long slidingInterval) { this.referenceTime = referenceTime; this.currentCount = currentCount; this.slidingCount = slidingCount; diff --git a/storm-client/src/jvm/org/apache/storm/windowing/Event.java b/storm-client/src/jvm/org/apache/storm/windowing/Event.java index c5372415099..577a654a8f2 100644 --- a/storm-client/src/jvm/org/apache/storm/windowing/Event.java +++ b/storm-client/src/jvm/org/apache/storm/windowing/Event.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -19,7 +25,8 @@ */ public interface Event { /** - * The event timestamp in millis. This could be the time when the source generated the tuple or the time when the tuple was received by + * The event timestamp in millis. This could be the time when the source generated the tuple or + * the time when the tuple was received by * a bolt. * * @return the event timestamp in milliseconds. @@ -34,7 +41,8 @@ public interface Event { T get(); /** - * If this is a watermark event or not. Watermark events are used for tracking time while processing event based ts. + * If this is a watermark event or not. Watermark events are used for tracking time while + * processing event based ts. * * @return true if this is a watermark event */ diff --git a/storm-client/src/jvm/org/apache/storm/windowing/EventImpl.java b/storm-client/src/jvm/org/apache/storm/windowing/EventImpl.java index 012e3b91675..a51d41e754a 100644 --- a/storm-client/src/jvm/org/apache/storm/windowing/EventImpl.java +++ b/storm-client/src/jvm/org/apache/storm/windowing/EventImpl.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/windowing/EvictionContext.java b/storm-client/src/jvm/org/apache/storm/windowing/EvictionContext.java index 1f600d671bf..53c0fd2d2bc 100644 --- a/storm-client/src/jvm/org/apache/storm/windowing/EvictionContext.java +++ b/storm-client/src/jvm/org/apache/storm/windowing/EvictionContext.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -17,7 +23,8 @@ */ public interface EvictionContext { /** - * Returns the reference time that the eviction policy could use to evict the events. In the case of event time processing, this would + * Returns the reference time that the eviction policy could use to evict the events. In the + * case of event time processing, this would * be the watermark time. * * @return the reference time in millis @@ -31,7 +38,6 @@ public interface EvictionContext { */ Long getSlidingCount(); - /** * Returns the sliding interval for time based windows. * @@ -40,7 +46,8 @@ public interface EvictionContext { Long getSlidingInterval(); /** - * Returns the current count of events in the queue up to the reference time based on which count based evictions can be performed. + * Returns the current count of events in the queue up to the reference time based on which + * count based evictions can be performed. * * @return the current count */ diff --git a/storm-client/src/jvm/org/apache/storm/windowing/EvictionPolicy.java b/storm-client/src/jvm/org/apache/storm/windowing/EvictionPolicy.java index b95414e8de1..dd5d539a861 100644 --- a/storm-client/src/jvm/org/apache/storm/windowing/EvictionPolicy.java +++ b/storm-client/src/jvm/org/apache/storm/windowing/EvictionPolicy.java @@ -1,33 +1,43 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.windowing; /** - * Eviction policy tracks events and decides whether an event should be evicted from the window or not. + * Eviction policy tracks events and decides whether an event should be evicted from the window or + * not. * * @param the type of event that is tracked. */ public interface EvictionPolicy { /** - * Decides if an event should be expired from the window, processed in the current window or kept for later processing. + * Decides if an event should be expired from the window, processed in the current window or + * kept for later processing. * * @param event the input event - * @return the {@link org.apache.storm.windowing.EvictionPolicy.Action} to be taken based on the input event + * @return the {@link org.apache.storm.windowing.EvictionPolicy.Action} to be taken based on the + * input event */ Action evict(Event event); /** - * Tracks the event to later decide whether {@link EvictionPolicy#evict(Event)} should evict it or not. + * Tracks the event to later decide whether {@link EvictionPolicy#evict(Event)} should evict it + * or not. * * @param event the input event to be tracked */ @@ -41,7 +51,8 @@ public interface EvictionPolicy { EvictionContext getContext(); /** - * Sets a context in the eviction policy that can be used while evicting the events. E.g. For TimeEvictionPolicy, this could be used to + * Sets a context in the eviction policy that can be used while evicting the events. E.g. For + * TimeEvictionPolicy, this could be used to * set the reference timestamp. * * @param context the eviction context @@ -54,7 +65,8 @@ public interface EvictionPolicy { void reset(); /** - * Return runtime state to be checkpointed by the framework for restoring the eviction policy in case of failures. + * Return runtime state to be checkpointed by the framework for restoring the eviction policy in + * case of failures. * * @return the state */ @@ -72,19 +84,20 @@ public interface EvictionPolicy { */ enum Action { /** - * expire the event and remove it from the queue. + * Expire the event and remove it from the queue. */ EXPIRE, /** - * process the event in the current window of events. + * Process the event in the current window of events. */ PROCESS, /** - * don't include in the current window but keep the event in the queue for evaluating as a part of future windows. + * don't include in the current window but keep the event in the queue for evaluating as a + * part of future windows. */ KEEP, /** - * stop processing the queue, there cannot be anymore events satisfying the eviction policy. + * Stop processing the queue, there cannot be anymore events satisfying the eviction policy. */ STOP } diff --git a/storm-client/src/jvm/org/apache/storm/windowing/StatefulWindowManager.java b/storm-client/src/jvm/org/apache/storm/windowing/StatefulWindowManager.java index 02225405788..bab2e4a8112 100644 --- a/storm-client/src/jvm/org/apache/storm/windowing/StatefulWindowManager.java +++ b/storm-client/src/jvm/org/apache/storm/windowing/StatefulWindowManager.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -40,7 +46,8 @@ public StatefulWindowManager(WindowLifecycleListener lifecycleListener) { * @param queue a collection where the events in the window can be enqueued.
      * Note: This collection has to be thread safe. */ - public StatefulWindowManager(WindowLifecycleListener lifecycleListener, Collection> queue) { + public StatefulWindowManager(WindowLifecycleListener lifecycleListener, + Collection> queue) { super(lifecycleListener, queue); } @@ -51,7 +58,8 @@ public boolean hasNext() { if (status.isValid()) { return inner.hasNext(); } - throw new IllegalStateException("Stale iterator, the iterator is valid only within the corresponding execute"); + throw new IllegalStateException("Stale iterator, the iterator is valid only " + + "within the corresponding execute"); } @Override @@ -59,7 +67,8 @@ public T next() { if (status.isValid()) { return inner.next(); } - throw new IllegalStateException("Stale iterator, the iterator is valid only within the corresponding execute"); + throw new IllegalStateException("Stale iterator, the iterator is valid only " + + "within the corresponding execute"); } }; } @@ -93,10 +102,12 @@ public Iterator get() { } return expiringIterator(res, status); } - throw new IllegalStateException("Stale window, the window is valid only within the corresponding execute"); + throw new IllegalStateException("Stale window, the window is valid only " + + "within the corresponding execute"); } }; - windowLifecycleListener.onActivation(wrapper, null, null, evictionPolicy.getContext().getReferenceTime()); + windowLifecycleListener.onActivation(wrapper, null, null, evictionPolicy.getContext() + .getReferenceTime()); // invalidate the iterator status.invalidate(); } else { diff --git a/storm-client/src/jvm/org/apache/storm/windowing/TimeEvictionPolicy.java b/storm-client/src/jvm/org/apache/storm/windowing/TimeEvictionPolicy.java index 8e157508b7f..da9742f9ca9 100644 --- a/storm-client/src/jvm/org/apache/storm/windowing/TimeEvictionPolicy.java +++ b/storm-client/src/jvm/org/apache/storm/windowing/TimeEvictionPolicy.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -26,7 +32,8 @@ public class TimeEvictionPolicy implements EvictionPolicy private long delta; /** - * Constructs a TimeEvictionPolicy that evicts events older than the given window length in millis. + * Constructs a TimeEvictionPolicy that evicts events older than the given window length in + * millis. * * @param windowLength the duration in milliseconds */ @@ -39,7 +46,8 @@ public TimeEvictionPolicy(int windowLength) { */ @Override public Action evict(Event event) { - long now = evictionContext == null ? System.currentTimeMillis() : evictionContext.getReferenceTime(); + long now = evictionContext == null ? System.currentTimeMillis() : evictionContext + .getReferenceTime(); long diff = now - event.getTimestamp(); if (diff >= (windowLength + delta)) { return Action.EXPIRE; @@ -68,7 +76,8 @@ public void setContext(EvictionContext context) { if (prevContext == null) { delta = Integer.MAX_VALUE; // consider all events for the initial window } else { - delta = context.getReferenceTime() - prevContext.getReferenceTime() - context.getSlidingInterval(); + delta = context.getReferenceTime() - prevContext.getReferenceTime() - context + .getSlidingInterval(); if (Math.abs(delta) > 100) { LOG.warn("Possible clock drift or long running computation in window; " + "Previous eviction time: {}, current eviction time: {}", diff --git a/storm-client/src/jvm/org/apache/storm/windowing/TimeTriggerPolicy.java b/storm-client/src/jvm/org/apache/storm/windowing/TimeTriggerPolicy.java index 31f83c0bb31..2a35e21311f 100644 --- a/storm-client/src/jvm/org/apache/storm/windowing/TimeTriggerPolicy.java +++ b/storm-client/src/jvm/org/apache/storm/windowing/TimeTriggerPolicy.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -38,7 +44,8 @@ public TimeTriggerPolicy(long millis, TriggerHandler handler) { this(millis, handler, null); } - public TimeTriggerPolicy(long millis, TriggerHandler handler, EvictionPolicy evictionPolicy) { + public TimeTriggerPolicy(long millis, TriggerHandler handler, EvictionPolicy evictionPolicy) { this.duration = millis; this.handler = handler; ThreadFactory threadFactory = new ThreadFactoryBuilder() @@ -61,7 +68,8 @@ public void reset() { @Override public void start() { - executorFuture = executor.scheduleAtFixedRate(newTriggerTask(), duration, duration, TimeUnit.MILLISECONDS); + executorFuture = executor.scheduleAtFixedRate(newTriggerTask(), duration, duration, + TimeUnit.MILLISECONDS); } @Override @@ -108,14 +116,16 @@ private Runnable newTriggerTask() { return new Runnable() { @Override public void run() { - // do not process current timestamp since tuples might arrive while the trigger is executing + // do not process current timestamp since tuples might arrive while the trigger is + // executing long now = System.currentTimeMillis() - 1; try { /* * set the current timestamp as the reference time for the eviction policy * to evict the events */ - evictionPolicy.setContext(new DefaultEvictionContext(now, null, null, duration)); + evictionPolicy.setContext(new DefaultEvictionContext(now, null, null, + duration)); handler.onTrigger(); } catch (Throwable th) { LOG.error("handler.onTrigger failed ", th); diff --git a/storm-client/src/jvm/org/apache/storm/windowing/TimestampExtractor.java b/storm-client/src/jvm/org/apache/storm/windowing/TimestampExtractor.java index 8074ae236a9..425cf05167f 100644 --- a/storm-client/src/jvm/org/apache/storm/windowing/TimestampExtractor.java +++ b/storm-client/src/jvm/org/apache/storm/windowing/TimestampExtractor.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/windowing/TriggerHandler.java b/storm-client/src/jvm/org/apache/storm/windowing/TriggerHandler.java index 97ea5af13f9..f5ed41db54c 100644 --- a/storm-client/src/jvm/org/apache/storm/windowing/TriggerHandler.java +++ b/storm-client/src/jvm/org/apache/storm/windowing/TriggerHandler.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -19,7 +25,8 @@ public interface TriggerHandler { /** * The code to execute when the {@link TriggerPolicy} condition is satisfied. * - * @return true if the window was evaluated with at least one event in the window, false otherwise + * @return true if the window was evaluated with at least one event in the window, false + * otherwise */ boolean onTrigger(); } diff --git a/storm-client/src/jvm/org/apache/storm/windowing/TriggerPolicy.java b/storm-client/src/jvm/org/apache/storm/windowing/TriggerPolicy.java index d53d399f9d0..4aed1b6fe19 100644 --- a/storm-client/src/jvm/org/apache/storm/windowing/TriggerPolicy.java +++ b/storm-client/src/jvm/org/apache/storm/windowing/TriggerPolicy.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -26,12 +32,13 @@ public interface TriggerPolicy { void track(Event event); /** - * resets the trigger policy. + * Resets the trigger policy. */ void reset(); /** - * Starts the trigger policy. This can be used during recovery to start the triggers after recovery is complete. + * Starts the trigger policy. This can be used during recovery to start the triggers after + * recovery is complete. */ void start(); @@ -41,7 +48,8 @@ public interface TriggerPolicy { void shutdown(); /** - * Return runtime state to be checkpointed by the framework for restoring the trigger policy in case of failures. + * Return runtime state to be checkpointed by the framework for restoring the trigger policy in + * case of failures. * * @return the state */ diff --git a/storm-client/src/jvm/org/apache/storm/windowing/TupleWindow.java b/storm-client/src/jvm/org/apache/storm/windowing/TupleWindow.java index 08663cef7be..a19f22734ca 100644 --- a/storm-client/src/jvm/org/apache/storm/windowing/TupleWindow.java +++ b/storm-client/src/jvm/org/apache/storm/windowing/TupleWindow.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/windowing/TupleWindowImpl.java b/storm-client/src/jvm/org/apache/storm/windowing/TupleWindowImpl.java index bbf86147eb3..3b71bb98107 100644 --- a/storm-client/src/jvm/org/apache/storm/windowing/TupleWindowImpl.java +++ b/storm-client/src/jvm/org/apache/storm/windowing/TupleWindowImpl.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -89,7 +95,8 @@ public boolean equals(Object o) { if (newTuples != null ? !newTuples.equals(that.newTuples) : that.newTuples != null) { return false; } - return expiredTuples != null ? expiredTuples.equals(that.expiredTuples) : that.expiredTuples == null; + return expiredTuples != null ? expiredTuples + .equals(that.expiredTuples) : that.expiredTuples == null; } diff --git a/storm-client/src/jvm/org/apache/storm/windowing/TupleWindowIterImpl.java b/storm-client/src/jvm/org/apache/storm/windowing/TupleWindowIterImpl.java index ba8ba1ff2d1..8de7c367c3b 100644 --- a/storm-client/src/jvm/org/apache/storm/windowing/TupleWindowIterImpl.java +++ b/storm-client/src/jvm/org/apache/storm/windowing/TupleWindowIterImpl.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/windowing/WaterMarkEvent.java b/storm-client/src/jvm/org/apache/storm/windowing/WaterMarkEvent.java index 57a982baae4..8c4b1885cee 100644 --- a/storm-client/src/jvm/org/apache/storm/windowing/WaterMarkEvent.java +++ b/storm-client/src/jvm/org/apache/storm/windowing/WaterMarkEvent.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/windowing/WaterMarkEventGenerator.java b/storm-client/src/jvm/org/apache/storm/windowing/WaterMarkEventGenerator.java index 060f0d1b114..110f5e689ed 100644 --- a/storm-client/src/jvm/org/apache/storm/windowing/WaterMarkEventGenerator.java +++ b/storm-client/src/jvm/org/apache/storm/windowing/WaterMarkEventGenerator.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -28,8 +34,10 @@ import org.slf4j.LoggerFactory; /** - * Tracks tuples across input streams and periodically emits watermark events. Watermark event timestamp is the minimum of the latest tuple - * timestamps across all the input streams (minus the lag). Once a watermark event is emitted any tuple coming with an earlier timestamp can + * Tracks tuples across input streams and periodically emits watermark events. Watermark event + * timestamp is the minimum of the latest tuple + * timestamps across all the input streams (minus the lag). Once a watermark event is emitted any + * tuple coming with an earlier timestamp can * be considered as late events. */ public class WaterMarkEventGenerator implements Runnable { @@ -47,8 +55,10 @@ public class WaterMarkEventGenerator implements Runnable { * Creates a new WatermarkEventGenerator. * * @param windowManager The window manager this generator will submit watermark events to - * @param intervalMs The generator will check if it should generate a watermark event with this interval - * @param eventTsLagMs The max allowed lag behind the last watermark event before an event is considered late + * @param intervalMs The generator will check if it should generate a watermark event with this + * interval + * @param eventTsLagMs The max allowed lag behind the last watermark event before an event is + * considered late * @param inputStreams The input streams this generator is expected to handle */ public WaterMarkEventGenerator(WindowManager windowManager, int intervalMs, @@ -68,7 +78,8 @@ public WaterMarkEventGenerator(WindowManager windowManager, int intervalMs, } /** - * Tracks the timestamp of the event in the stream, returns true if the event can be considered for processing or false if its a late + * Tracks the timestamp of the event in the stream, returns true if the event can be considered + * for processing or false if its a late * event. */ public boolean track(GlobalStreamId stream, long ts) { @@ -124,7 +135,8 @@ private void checkFailures() { } public void start() { - this.executorFuture = executorService.scheduleAtFixedRate(this, interval, interval, TimeUnit.MILLISECONDS); + this.executorFuture = executorService.scheduleAtFixedRate(this, interval, interval, + TimeUnit.MILLISECONDS); } public void shutdown() { diff --git a/storm-client/src/jvm/org/apache/storm/windowing/WatermarkCountEvictionPolicy.java b/storm-client/src/jvm/org/apache/storm/windowing/WatermarkCountEvictionPolicy.java index ca6201a6ace..9887c1e1525 100644 --- a/storm-client/src/jvm/org/apache/storm/windowing/WatermarkCountEvictionPolicy.java +++ b/storm-client/src/jvm/org/apache/storm/windowing/WatermarkCountEvictionPolicy.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -16,7 +22,8 @@ import org.apache.storm.streams.Pair; /** - * An eviction policy that tracks count based on watermark ts and evicts events up to the watermark based on a threshold count. + * An eviction policy that tracks count based on watermark ts and evicts events up to the watermark + * based on a threshold count. * * @param the type of event tracked by this policy. */ @@ -35,14 +42,17 @@ public WatermarkCountEvictionPolicy(int count) { @Override public Action evict(Event event) { if (getContext() == null) { - //It is possible to get asked about eviction before we have a context, due to WindowManager.compactWindow. - //In this case we should hold on to all the events. When the first watermark is received, the context will be set, - //and the events will be reevaluated for eviction + // It is possible to get asked about eviction before we have a context, due to + // WindowManager.compactWindow. + // In this case we should hold on to all the events. When the first watermark is + // received, the context will be set, + // and the events will be reevaluated for eviction return Action.STOP; } Action action; - if (event.getTimestamp() <= getContext().getReferenceTime() && processed < currentCount.get()) { + if (event.getTimestamp() <= getContext().getReferenceTime() && processed < currentCount + .get()) { action = doEvict(event); if (action == Action.PROCESS) { ++processed; diff --git a/storm-client/src/jvm/org/apache/storm/windowing/WatermarkCountTriggerPolicy.java b/storm-client/src/jvm/org/apache/storm/windowing/WatermarkCountTriggerPolicy.java index 30551874f79..eba52f755e0 100644 --- a/storm-client/src/jvm/org/apache/storm/windowing/WatermarkCountTriggerPolicy.java +++ b/storm-client/src/jvm/org/apache/storm/windowing/WatermarkCountTriggerPolicy.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -15,7 +21,8 @@ import java.util.List; /** - * A trigger policy that tracks event counts and sets the context for eviction policy to evict based on latest watermark time. + * A trigger policy that tracks event counts and sets the context for eviction policy to evict based + * on latest watermark time. * * @param the type of event tracked by this policy. */ @@ -59,13 +66,15 @@ public void shutdown() { } /** - * Triggers all the pending windows up to the waterMarkEvent timestamp based on the sliding interval count. + * Triggers all the pending windows up to the waterMarkEvent timestamp based on the sliding + * interval count. * * @param waterMarkEvent the watermark event */ private void handleWaterMarkEvent(Event waterMarkEvent) { long watermarkTs = waterMarkEvent.getTimestamp(); - List eventTs = windowManager.getSlidingCountTimestamps(lastProcessedTs, watermarkTs, count); + List eventTs = windowManager.getSlidingCountTimestamps(lastProcessedTs, watermarkTs, + count); for (long ts : eventTs) { evictionPolicy.setContext(new DefaultEvictionContext(ts, null, Long.valueOf(count))); handler.onTrigger(); diff --git a/storm-client/src/jvm/org/apache/storm/windowing/WatermarkTimeEvictionPolicy.java b/storm-client/src/jvm/org/apache/storm/windowing/WatermarkTimeEvictionPolicy.java index 7638849917b..381498646da 100644 --- a/storm-client/src/jvm/org/apache/storm/windowing/WatermarkTimeEvictionPolicy.java +++ b/storm-client/src/jvm/org/apache/storm/windowing/WatermarkTimeEvictionPolicy.java @@ -1,25 +1,33 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.windowing; /** - * An eviction policy that evicts events based on time duration taking watermark time and event lag into account. + * An eviction policy that evicts events based on time duration taking watermark time and event lag + * into account. */ public class WatermarkTimeEvictionPolicy extends TimeEvictionPolicy { private final int lag; /** - * Constructs a WatermarkTimeEvictionPolicy that evicts events older than the given window length in millis. + * Constructs a WatermarkTimeEvictionPolicy that evicts events older than the given window + * length in millis. * * @param windowLength the window length in milliseconds */ @@ -28,7 +36,8 @@ public WatermarkTimeEvictionPolicy(int windowLength) { } /** - * Constructs a WatermarkTimeEvictionPolicy that evicts events older than the given window length in millis. The lag parameter can be + * Constructs a WatermarkTimeEvictionPolicy that evicts events older than the given window + * length in millis. The lag parameter can be * used in the case of event based ts to break the queue scan early. * * @param windowLength the window length in milliseconds @@ -42,15 +51,18 @@ public WatermarkTimeEvictionPolicy(int windowLength, int lag) { /** * {@inheritDoc} *

      - * Keeps events with future ts in the queue for processing in the next window. If the ts difference is more than the lag, stops scanning + * Keeps events with future ts in the queue for processing in the next window. If the ts + * difference is more than the lag, stops scanning * the queue for the current window. */ @Override public Action evict(Event event) { if (evictionContext == null) { - //It is possible to get asked about eviction before we have a context, due to WindowManager.compactWindow. - //In this case we should hold on to all the events. When the first watermark is received, the context will be set, - //and the events will be reevaluated for eviction + // It is possible to get asked about eviction before we have a context, due to + // WindowManager.compactWindow. + // In this case we should hold on to all the events. When the first watermark is + // received, the context will be set, + // and the events will be reevaluated for eviction return Action.STOP; } diff --git a/storm-client/src/jvm/org/apache/storm/windowing/WatermarkTimeTriggerPolicy.java b/storm-client/src/jvm/org/apache/storm/windowing/WatermarkTimeTriggerPolicy.java index 9758eb83f8e..322039c1663 100644 --- a/storm-client/src/jvm/org/apache/storm/windowing/WatermarkTimeTriggerPolicy.java +++ b/storm-client/src/jvm/org/apache/storm/windowing/WatermarkTimeTriggerPolicy.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -16,7 +22,8 @@ import org.slf4j.LoggerFactory; /** - * Handles watermark events and triggers {@link TriggerHandler#onTrigger()} for each window interval that has events to be processed up to + * Handles watermark events and triggers {@link TriggerHandler#onTrigger()} for each window interval + * that has events to be processed up to * the watermark ts. */ public class WatermarkTimeTriggerPolicy implements TriggerPolicy { @@ -28,7 +35,8 @@ public class WatermarkTimeTriggerPolicy implements TriggerPolicy { private volatile long nextWindowEndTs; private boolean started; - public WatermarkTimeTriggerPolicy(long slidingIntervalMs, TriggerHandler handler, EvictionPolicy evictionPolicy, + public WatermarkTimeTriggerPolicy(long slidingIntervalMs, TriggerHandler handler, + EvictionPolicy evictionPolicy, WindowManager windowManager) { this.slidingIntervalMs = slidingIntervalMs; this.handler = handler; @@ -60,7 +68,8 @@ public void shutdown() { } /** - * Invokes the trigger all pending windows up to the watermark timestamp. The end ts of the window is set in the eviction policy context + * Invokes the trigger all pending windows up to the watermark timestamp. The end ts of the + * window is set in the eviction policy context * so that the events falling within that window can be processed. */ private void handleWaterMarkEvent(Event event) { @@ -81,7 +90,8 @@ private void handleWaterMarkEvent(Event event) { long ts = getNextAlignedWindowTs(windowEndTs, watermarkTs); LOG.debug("Next aligned window end ts {}", ts); if (ts == Long.MAX_VALUE) { - LOG.debug("No events to process between {} and watermark ts {}", windowEndTs, watermarkTs); + LOG.debug("No events to process between {} and watermark ts {}", windowEndTs, + watermarkTs); break; } windowEndTs = ts; @@ -91,12 +101,14 @@ private void handleWaterMarkEvent(Event event) { } /** - * Computes the next window by scanning the events in the window and finds the next aligned window between the startTs and endTs. Return + * Computes the next window by scanning the events in the window and finds the next aligned + * window between the startTs and endTs. Return * the end ts of the next aligned window, i.e. the ts when the window should fire. * * @param startTs the start timestamp (excluding) * @param endTs the end timestamp (including) - * @return the aligned window end ts for the next window or Long.MAX_VALUE if there are no more events to be processed. + * @return the aligned window end ts for the next window or Long.MAX_VALUE if there are no more + * events to be processed. */ private long getNextAlignedWindowTs(long startTs, long endTs) { long nextTs = windowManager.getEarliestEventTs(startTs, endTs); diff --git a/storm-client/src/jvm/org/apache/storm/windowing/Window.java b/storm-client/src/jvm/org/apache/storm/windowing/Window.java index e2dc4236056..3c658d0616f 100644 --- a/storm-client/src/jvm/org/apache/storm/windowing/Window.java +++ b/storm-client/src/jvm/org/apache/storm/windowing/Window.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -24,10 +30,12 @@ public interface Window { /** * Gets the list of events in the window. - *

      - * Note: If the number of tuples in windows is huge, invoking {@code get} would - * load all the tuples into memory and may throw an OOM exception. Use windowing with persistence ({@link - * BaseStatefulWindowedBolt#withPersistence()}) and {@link Window#getIter} to retrieve an iterator over the events in the window. + * + *

      Note: If the number of tuples in windows is huge, invoking {@code get} would + * load all the tuples into memory and may throw an OOM exception. Use windowing with + * persistence ({@link + * BaseStatefulWindowedBolt#withPersistence()}) and {@link Window#getIter} to retrieve an + * iterator over the events in the window. *

      * * @return the list of events in the window. @@ -36,44 +44,52 @@ public interface Window { /** * Returns an iterator over the events in the window. - *

      - * Note: This is only supported when using windowing with persistence {@link BaseStatefulWindowedBolt#withPersistence()}. + * + *

      Note: This is only supported when using windowing with persistence {@link + * BaseStatefulWindowedBolt#withPersistence()}. *

      * * @return an {@link Iterator} over the events in the current window. * - * @throws UnsupportedOperationException if not using {@link BaseStatefulWindowedBolt#withPersistence()} + * @throws UnsupportedOperationException if not using {@link + * BaseStatefulWindowedBolt#withPersistence()} */ default Iterator getIter() { throw new UnsupportedOperationException("Not implemented"); } /** - * Get the list of newly added events in the window since the last time the window was generated. - *

      - * Note: This is not supported when using windowing with persistence ({@link BaseStatefulWindowedBolt#withPersistence()}). + * Get the list of newly added events in the window since the last time the window was + * generated. + * + *

      Note: This is not supported when using windowing with persistence ({@link + * BaseStatefulWindowedBolt#withPersistence()}). *

      * * @return the list of newly added events in the window. * - * @throws UnsupportedOperationException if using {@link BaseStatefulWindowedBolt#withPersistence()} + * @throws UnsupportedOperationException if using {@link + * BaseStatefulWindowedBolt#withPersistence()} */ List getNew(); /** * Get the list of events expired from the window since the last time the window was generated. - *

      - * Note: This is not supported when using windowing with persistence ({@link BaseStatefulWindowedBolt#withPersistence()}). + * + *

      Note: This is not supported when using windowing with persistence ({@link + * BaseStatefulWindowedBolt#withPersistence()}). *

      * * @return the list of events expired from the window. * - * @throws UnsupportedOperationException if using {@link BaseStatefulWindowedBolt#withPersistence()} + * @throws UnsupportedOperationException if using {@link + * BaseStatefulWindowedBolt#withPersistence()} */ List getExpired(); /** - * If processing based on event time, returns the window end time based on watermark otherwise returns the window end time based on + * If processing based on event time, returns the window end time based on watermark otherwise + * returns the window end time based on * processing time. * * @return the window end timestamp @@ -81,7 +97,8 @@ default Iterator getIter() { Long getEndTimestamp(); /** - * Returns the window start timestamp. Will return null if the window length is not based on time duration. + * Returns the window start timestamp. Will return null if the window length is not based on + * time duration. * * @return the window start timestamp or null if the window length is not time based */ diff --git a/storm-client/src/jvm/org/apache/storm/windowing/WindowLifecycleListener.java b/storm-client/src/jvm/org/apache/storm/windowing/WindowLifecycleListener.java index ea9db39c072..17cbcf42a19 100644 --- a/storm-client/src/jvm/org/apache/storm/windowing/WindowLifecycleListener.java +++ b/storm-client/src/jvm/org/apache/storm/windowing/WindowLifecycleListener.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -37,20 +43,24 @@ public interface WindowLifecycleListener { * @param expired the expired events since last activation. * @param referenceTime the reference (event or processing) time that resulted in activation */ - default void onActivation(List events, List newEvents, List expired, Long referenceTime) { + default void onActivation(List events, List newEvents, List expired, + Long referenceTime) { throw new UnsupportedOperationException("Not implemented"); } /** - * Called on activation of the window due to the {@link TriggerPolicy}. This is typically invoked when the windows are persisted in + * Called on activation of the window due to the {@link TriggerPolicy}. This is typically + * invoked when the windows are persisted in * state and is huge to be loaded entirely in memory. * * @param eventsIt a supplier of iterator over the list of current events in the window - * @param newEventsIt a supplier of iterator over the newly added events since the last ativation + * @param newEventsIt a supplier of iterator over the newly added events since the last + * ativation * @param expiredIt a supplier of iterator over the expired events since the last activation * @param referenceTime the reference (event or processing) time that resulted in activation */ - default void onActivation(Supplier> eventsIt, Supplier> newEventsIt, Supplier> expiredIt, + default void onActivation(Supplier> eventsIt, Supplier> newEventsIt, + Supplier> expiredIt, Long referenceTime) { throw new UnsupportedOperationException("Not implemented"); } diff --git a/storm-client/src/jvm/org/apache/storm/windowing/WindowManager.java b/storm-client/src/jvm/org/apache/storm/windowing/WindowManager.java index 9043020339f..6ab5b4085fe 100644 --- a/storm-client/src/jvm/org/apache/storm/windowing/WindowManager.java +++ b/storm-client/src/jvm/org/apache/storm/windowing/WindowManager.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -33,7 +39,8 @@ import org.slf4j.LoggerFactory; /** - * Tracks a window of events and fires {@link WindowLifecycleListener} callbacks on expiry of events or activation of the window due to + * Tracks a window of events and fires {@link WindowLifecycleListener} callbacks on expiry of events + * or activation of the window due to * {@link TriggerPolicy}. * * @param the type of event in the window. @@ -42,7 +49,8 @@ public class WindowManager implements TriggerHandler { /** * Expire old events every EXPIRE_EVENTS_THRESHOLD to keep the window size in check. * - *

      Note that if the eviction policy is based on watermarks, events will not be evicted until a new watermark would cause them to be + *

      Note that if the eviction policy is based on watermarks, events will not be evicted until + * a new watermark would cause them to be * considered expired anyway, regardless of this limit */ public static final int EXPIRE_EVENTS_THRESHOLD = 100; @@ -152,8 +160,10 @@ public boolean onTrigger() { prevWindowEvents.clear(); if (!events.isEmpty()) { prevWindowEvents.addAll(windowEvents); - LOG.debug("invoking windowLifecycleListener onActivation, [{}] events in window.", events.size()); - windowLifecycleListener.onActivation(events, newEvents, expired, evictionPolicy.getContext().getReferenceTime()); + LOG.debug("invoking windowLifecycleListener onActivation, [{}] events in window.", + events.size()); + windowLifecycleListener.onActivation(events, newEvents, expired, evictionPolicy + .getContext().getReferenceTime()); } else { LOG.debug("No events in the window, skipping onActivation"); } @@ -169,7 +179,8 @@ public void shutdown() { } /** - * expires events that fall out of the window every EXPIRE_EVENTS_THRESHOLD so that the window does not grow too big. + * Expires events that fall out of the window every EXPIRE_EVENTS_THRESHOLD so that the window + * does not grow too big. */ protected void compactWindow() { if (eventsSinceLastExpiry.incrementAndGet() >= EXPIRE_EVENTS_THRESHOLD) { @@ -178,7 +189,8 @@ protected void compactWindow() { } /** - * feed the event to the eviction and trigger policies for bookkeeping and optionally firing the trigger. + * Feed the event to the eviction and trigger policies for bookkeeping and optionally firing the + * trigger. */ private void track(Event windowEvent) { evictionPolicy.track(windowEvent); @@ -186,9 +198,11 @@ private void track(Event windowEvent) { } /** - * Scan events in the queue, using the expiration policy to check if the event should be evicted or not. + * Scan events in the queue, using the expiration policy to check if the event should be evicted + * or not. * - * @param fullScan if set, will scan the entire queue; if not set, will stop as soon as an event not satisfying the expiration policy is + * @param fullScan if set, will scan the entire queue; if not set, will stop as soon as an event + * not satisfying the expiration policy is * found * @return the list of events to be processed as a part of the current window */ @@ -242,7 +256,8 @@ public long getEarliestEventTs(long startTs, long endTs) { } /** - * Scans the event queue and returns number of events having timestamp less than or equal to the reference time. + * Scans the event queue and returns number of events having timestamp less than or equal to the + * reference time. * * @param referenceTime the reference timestamp in millis * @return the count of events with timestamp less than or equal to referenceTime @@ -258,7 +273,8 @@ public int getEventCount(long referenceTime) { } /** - * Scans the event queue and returns the list of event ts falling between startTs (exclusive) and endTs (inclusive) at each sliding + * Scans the event queue and returns the list of event ts falling between startTs (exclusive) + * and endTs (inclusive) at each sliding * interval counts. * * @param startTs the start timestamp (exclusive) diff --git a/storm-client/src/jvm/org/apache/storm/windowing/persistence/SimpleWindowPartitionCache.java b/storm-client/src/jvm/org/apache/storm/windowing/persistence/SimpleWindowPartitionCache.java index f52dc128e7d..08cdddfe733 100644 --- a/storm-client/src/jvm/org/apache/storm/windowing/persistence/SimpleWindowPartitionCache.java +++ b/storm-client/src/jvm/org/apache/storm/windowing/persistence/SimpleWindowPartitionCache.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -23,7 +29,8 @@ import org.slf4j.LoggerFactory; /** - * A simple implementation that evicts the largest un-pinned entry from the cache. This works well for caching window partitions since the + * A simple implementation that evicts the largest un-pinned entry from the cache. This works well + * for caching window partitions since the * access pattern is mostly sequential scans. */ public class SimpleWindowPartitionCache implements WindowPartitionCache { @@ -37,7 +44,8 @@ public class SimpleWindowPartitionCache implements WindowPartitionCache removalListener, CacheLoader cacheLoader) { + private SimpleWindowPartitionCache(long maximumSize, RemovalListener removalListener, + CacheLoader cacheLoader) { if (maximumSize <= 0) { throw new IllegalArgumentException("maximumSize must be greater than 0"); } @@ -157,7 +165,8 @@ private void ensureCapacity() { if (!isPinned(next.getKey())) { it.remove(); if (removalListener != null) { - removalListener.onRemoval(next.getKey(), next.getValue(), RemovalCause.REPLACED); + removalListener.onRemoval(next.getKey(), next.getValue(), + RemovalCause.REPLACED); } --size; break; @@ -187,7 +196,8 @@ public SimpleWindowPartitionCacheBuilder maximumSize(long size) { } @Override - public SimpleWindowPartitionCacheBuilder removalListener(RemovalListener listener) { + public SimpleWindowPartitionCacheBuilder removalListener(RemovalListener listener) { removalListener = listener; return this; } diff --git a/storm-client/src/jvm/org/apache/storm/windowing/persistence/WindowPartitionCache.java b/storm-client/src/jvm/org/apache/storm/windowing/persistence/WindowPartitionCache.java index 2678c222e66..0dafaec9529 100644 --- a/storm-client/src/jvm/org/apache/storm/windowing/persistence/WindowPartitionCache.java +++ b/storm-client/src/jvm/org/apache/storm/windowing/persistence/WindowPartitionCache.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -31,7 +37,8 @@ public interface WindowPartitionCache { V get(K key); /** - * Get value from the cache or load the value pinning it so that the entry will never get evicted. + * Get value from the cache or load the value pinning it so that the entry will never get + * evicted. * * @param key the key * @return the value diff --git a/storm-client/src/jvm/org/apache/storm/windowing/persistence/WindowState.java b/storm-client/src/jvm/org/apache/storm/windowing/persistence/WindowState.java index ec2d0adc974..023c6a02267 100644 --- a/storm-client/src/jvm/org/apache/storm/windowing/persistence/WindowState.java +++ b/storm-client/src/jvm/org/apache/storm/windowing/persistence/WindowState.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

      - * http://www.apache.org/licenses/LICENSE-2.0 - *

      - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

      http://www.apache.org/licenses/LICENSE-2.0 + * + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -103,7 +109,8 @@ private Iterator getIds() { @Override public void remove() { if (removeFrom == null) { - throw new IllegalStateException("No calls to next() since last call to remove()"); + throw new IllegalStateException("No calls to next() since last call to " + + "remove()"); } removeFrom.remove(); removeFrom = null; @@ -222,14 +229,16 @@ private void initCache() { .maximumSize(size) .removalListener(new WindowPartitionCache.RemovalListener>() { @Override - public void onRemoval(Long pid, WindowPartition p, WindowPartitionCache.RemovalCause removalCause) { + public void onRemoval(Long pid, WindowPartition p, + WindowPartitionCache.RemovalCause removalCause) { Objects.requireNonNull(pid, "Null partition id"); Objects.requireNonNull(p, "Null window partition"); LOG.debug("onRemoval for id '{}', WindowPartition '{}'", pid, p); try { windowPartitionsLock.lock(pid); if (p.isEmpty() && pid != latestPartitionId) { - // if the empty partition was not invalidated by flush, but evicted from cache + // if the empty partition was not invalidated by flush, but evicted from + // cache if (removalCause != WindowPartitionCache.RemovalCause.EXPLICIT) { deletePartition(pid); windowPartitionsState.delete(pid); @@ -305,7 +314,8 @@ private void flush() { } // invalidate after releasing the lock // if the parition is pinned before we could invalidate, - // it will get invalidated in the next flush or when the entry gets evicted from the cache. + // it will get invalidated in the next flush or when the entry gets evicted from the + // cache. if (pidToInvalidate != null) { cache.invalidate(pidToInvalidate); } diff --git a/storm-client/src/jvm/org/apache/storm/zookeeper/ClientZookeeper.java b/storm-client/src/jvm/org/apache/storm/zookeeper/ClientZookeeper.java index e84742951d9..f4c120385a5 100644 --- a/storm-client/src/jvm/org/apache/storm/zookeeper/ClientZookeeper.java +++ b/storm-client/src/jvm/org/apache/storm/zookeeper/ClientZookeeper.java @@ -49,7 +49,8 @@ public class ClientZookeeper { private static ClientZookeeper _instance = INSTANCE; /** - * Provide an instance of this class for delegates to use. To mock out delegated methods, provide an instance of a subclass that + * Provide an instance of this class for delegates to use. To mock out delegated methods, + * provide an instance of a subclass that * overrides the implementation of the delegated method. * * @param u a ClientZookeeper instance @@ -59,7 +60,8 @@ public static void setInstance(ClientZookeeper u) { } /** - * Resets the singleton instance to the default. This is helpful to reset the class to its original functionality when mocking is no + * Resets the singleton instance to the default. This is helpful to reset the class to its + * original functionality when mocking is no * longer desired. */ public static void resetInstance() { @@ -70,14 +72,16 @@ public static void mkdirs(CuratorFramework zk, String path, List acls) { _instance.mkdirsImpl(zk, path, acls); } - public static CuratorFramework mkClient(Map conf, List servers, Object port, + public static CuratorFramework mkClient(Map conf, List servers, + Object port, String root, final WatcherCallBack watcher, Map authConf, DaemonType type) { return _instance.mkClientImpl(conf, servers, port, root, watcher, authConf, type); } // Deletes the state inside the zookeeper for a key, for which the // contents of the key starts with nimbus host port information - public static void deleteNodeBlobstore(CuratorFramework zk, String parentPath, String hostPortInfo) { + public static void deleteNodeBlobstore(CuratorFramework zk, String parentPath, + String hostPortInfo) { String normalizedParentPath = normalizePath(parentPath); List childPathList = null; if (existsNode(zk, normalizedParentPath, false)) { @@ -91,11 +95,13 @@ public static void deleteNodeBlobstore(CuratorFramework zk, String parentPath, S } } - public static String createNode(CuratorFramework zk, String path, byte[] data, CreateMode mode, List acls) { + public static String createNode(CuratorFramework zk, String path, byte[] data, CreateMode mode, + List acls) { String ret = null; try { String npath = normalizePath(path); - ret = zk.create().creatingParentsIfNeeded().withMode(mode).withACL(acls).forPath(npath, data); + ret = zk.create().creatingParentsIfNeeded().withMode(mode).withACL(acls).forPath(npath, + data); } catch (Exception e) { throw Utils.wrapInRuntime(e); } @@ -187,7 +193,8 @@ public static Stat setData(CuratorFramework zk, String path, byte[] data) { } } - public static Integer getVersion(CuratorFramework zk, String path, boolean watch) throws Exception { + public static Integer getVersion(CuratorFramework zk, String path, + boolean watch) throws Exception { String npath = normalizePath(path); // checkExists returns the Stat directly (null when absent) and still arms the watch when // watch is true. request 1. @@ -241,7 +248,8 @@ public static byte[] getData(CuratorFramework zk, String path, boolean watch) { * @param watch should a watch be enabled * @return null if no data is found, else the data with the version. */ - public static VersionedData getDataWithVersion(CuratorFramework zk, String path, boolean watch) { + public static VersionedData getDataWithVersion(CuratorFramework zk, String path, + boolean watch) { VersionedData data = null; String npath = normalizePath(path); Stat stats = new Stat(); @@ -300,13 +308,16 @@ public void mkdirsImpl(CuratorFramework zk, String path, List acls) { } } - public CuratorFramework mkClientImpl(Map conf, List servers, Object port, String root, + public CuratorFramework mkClientImpl(Map conf, List servers, + Object port, String root, final WatcherCallBack watcher, Map authConf, DaemonType type) { CuratorFramework fk; if (authConf != null) { - fk = CuratorUtils.newCurator(conf, servers, port, root, new ZookeeperAuthInfo(authConf), type.getDefaultZkAcls(conf)); + fk = CuratorUtils.newCurator(conf, servers, port, root, new ZookeeperAuthInfo(authConf), + type.getDefaultZkAcls(conf)); } else { - fk = CuratorUtils.newCurator(conf, servers, port, root, null, type.getDefaultZkAcls(conf)); + fk = CuratorUtils.newCurator(conf, servers, port, root, null, type + .getDefaultZkAcls(conf)); } fk.getCuratorListenable().addListener((unused, e) -> { diff --git a/storm-client/src/jvm/org/apache/storm/zookeeper/ZkEventTypes.java b/storm-client/src/jvm/org/apache/storm/zookeeper/ZkEventTypes.java index 1ec6d2db8af..9d24f474a4d 100644 --- a/storm-client/src/jvm/org/apache/storm/zookeeper/ZkEventTypes.java +++ b/storm-client/src/jvm/org/apache/storm/zookeeper/ZkEventTypes.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/src/jvm/org/apache/storm/zookeeper/ZkKeeperStates.java b/storm-client/src/jvm/org/apache/storm/zookeeper/ZkKeeperStates.java index 48b0ea19bbc..ffc75b3c6fb 100644 --- a/storm-client/src/jvm/org/apache/storm/zookeeper/ZkKeeperStates.java +++ b/storm-client/src/jvm/org/apache/storm/zookeeper/ZkKeeperStates.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/test/jvm/org/apache/storm/ConstantsTest.java b/storm-client/test/jvm/org/apache/storm/ConstantsTest.java index cdf930591ee..c3f17357582 100644 --- a/storm-client/test/jvm/org/apache/storm/ConstantsTest.java +++ b/storm-client/test/jvm/org/apache/storm/ConstantsTest.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm; @@ -21,14 +27,16 @@ public class ConstantsTest { @Test public void testEveryWhitelistedStreamIsControl() { for (String id : Constants.SYSTEM_CONTROL_STREAM_IDS) { - // fresh instance: the fast-reject gates must not depend on identity or a cached hashCode + // fresh instance: the fast-reject gates must not depend on identity or a cached + // hashCode assertTrue(Constants.isControlStreamId(new String(id.toCharArray())), id); } } @Test public void testHighVolumeSystemStreamsAreNotControl() { - // volume-proportional to the data plane: must stay on the data path (see SYSTEM_CONTROL_STREAM_IDS javadoc) + // volume-proportional to the data plane: must stay on the data path (see + // SYSTEM_CONTROL_STREAM_IDS javadoc) String[] dataPlaneSystemStreams = { "__ack_init", "__ack_ack", "__ack_fail", "__ack_reset_timeout", "__metrics", "__system", "__eventlog", "__heartbeat" @@ -40,8 +48,10 @@ public void testHighVolumeSystemStreamsAreNotControl() { @Test public void testUserStreamsAreNotControl() { - // includes ids whose length collides with a whitelisted id, to exercise the prefix/char gates - String[] userStreams = { "default", "s1", "stream", "__x", "_tick", "___tick", "abcdef", "abcdefghij" }; + // includes ids whose length collides with a whitelisted id, to exercise the prefix/char + // gates + String[] userStreams = + { "default", "s1", "stream", "__x", "_tick", "___tick", "abcdef", "abcdefghij" }; for (String id : userStreams) { assertFalse(Constants.isControlStreamId(id), id); } diff --git a/storm-client/test/jvm/org/apache/storm/PaceMakerStateStorageFactoryTest.java b/storm-client/test/jvm/org/apache/storm/PaceMakerStateStorageFactoryTest.java index 65590d41def..5c78819367a 100644 --- a/storm-client/test/jvm/org/apache/storm/PaceMakerStateStorageFactoryTest.java +++ b/storm-client/test/jvm/org/apache/storm/PaceMakerStateStorageFactoryTest.java @@ -18,6 +18,8 @@ package org.apache.storm; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -44,9 +46,6 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; - @ExtendWith(MockitoExtension.class) @SuppressWarnings("deprecation") public class PaceMakerStateStorageFactoryTest { @@ -59,7 +58,8 @@ public class PaceMakerStateStorageFactoryTest { private PacemakerClientPoolProxy clientPoolProxy; private PaceMakerStateStorage stateStorage; - public void createPaceMakerStateStorage(HBServerMessageType messageType, HBMessageData messageData) throws Exception { + public void createPaceMakerStateStorage(HBServerMessageType messageType, + HBMessageData messageData) throws Exception { HBMessage response = new HBMessage(messageType, messageData); when(clientMock.send(any())).thenReturn(response); clientPoolProxy = new PacemakerClientPoolProxy(); @@ -82,7 +82,8 @@ public void testSetWorkerHb() throws Exception { public void testSetWorkerHbResponseType() throws Exception { createPaceMakerStateStorage(HBServerMessageType.SEND_PULSE, null); assertThrows(RuntimeException.class, - () -> stateStorage.set_worker_hb("/foo", "data".getBytes(StandardCharsets.UTF_8), null)); + () -> stateStorage.set_worker_hb("/foo", "data".getBytes(StandardCharsets.UTF_8), + null)); } @Test @@ -106,9 +107,11 @@ public void testDeleteWorkerHbResponseType() throws Exception { public void testGetWorkerHb() throws Exception { HBPulse hbPulse = new HBPulse(); hbPulse.set_id("/foo"); - ClusterWorkerHeartbeat cwh = new ClusterWorkerHeartbeat("some-storm-id", new HashMap<>(), 1, 1); + ClusterWorkerHeartbeat cwh = new ClusterWorkerHeartbeat("some-storm-id", new HashMap<>(), 1, + 1); hbPulse.set_details(Utils.serialize(cwh)); - createPaceMakerStateStorage(HBServerMessageType.GET_PULSE_RESPONSE, HBMessageData.pulse(hbPulse)); + createPaceMakerStateStorage(HBServerMessageType.GET_PULSE_RESPONSE, HBMessageData + .pulse(hbPulse)); stateStorage.get_worker_hb("/foo", false); verify(clientMock).send(hbMessageCaptor.capture()); HBMessage sent = hbMessageCaptor.getValue(); @@ -132,7 +135,8 @@ public void testGetWorkerHbBadData() throws Exception { @Test public void testGetWorkerHbChildren() throws Exception { - createPaceMakerStateStorage(HBServerMessageType.GET_ALL_NODES_FOR_PATH_RESPONSE, HBMessageData.nodes(new HBNodes())); + createPaceMakerStateStorage(HBServerMessageType.GET_ALL_NODES_FOR_PATH_RESPONSE, + HBMessageData.nodes(new HBNodes())); stateStorage.get_worker_hb_children("/foo", false); verify(clientMock).send(hbMessageCaptor.capture()); HBMessage sent = hbMessageCaptor.getValue(); diff --git a/storm-client/test/jvm/org/apache/storm/TestConfigValidate.java b/storm-client/test/jvm/org/apache/storm/TestConfigValidate.java index 6105bc98737..27ec351edf9 100644 --- a/storm-client/test/jvm/org/apache/storm/TestConfigValidate.java +++ b/storm-client/test/jvm/org/apache/storm/TestConfigValidate.java @@ -18,6 +18,12 @@ package org.apache.storm; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + import java.math.BigInteger; import java.util.ArrayList; import java.util.Arrays; @@ -30,7 +36,6 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import javax.security.auth.Subject; - import org.apache.storm.blobstore.BlobStore; import org.apache.storm.blobstore.NimbusBlobStore; import org.apache.storm.generated.AuthorizationException; @@ -40,13 +45,12 @@ import org.apache.storm.shade.com.google.common.collect.ImmutableList; import org.apache.storm.shade.com.google.common.collect.ImmutableMap; import org.apache.storm.utils.Utils; -import org.apache.storm.validation.ConfigValidation; import org.apache.storm.validation.ConfigValidation.ImpersonationAclUserEntryValidator; import org.apache.storm.validation.ConfigValidation.IntegerValidator; import org.apache.storm.validation.ConfigValidation.KryoRegValidator; -import org.apache.storm.validation.ConfigValidation.LongValidator; import org.apache.storm.validation.ConfigValidation.ListEntryTypeValidator; import org.apache.storm.validation.ConfigValidation.ListOfListOfStringValidator; +import org.apache.storm.validation.ConfigValidation.LongValidator; import org.apache.storm.validation.ConfigValidation.NoDuplicateInListValidator; import org.apache.storm.validation.ConfigValidation.NotNullValidator; import org.apache.storm.validation.ConfigValidation.PositiveNumberValidator; @@ -54,6 +58,7 @@ import org.apache.storm.validation.ConfigValidation.RasConstraintsTypeValidator; import org.apache.storm.validation.ConfigValidation.StringValidator; import org.apache.storm.validation.ConfigValidation.UserResourcePoolEntryValidator; +import org.apache.storm.validation.ConfigValidation; import org.apache.storm.validation.ConfigValidationAnnotations.IsExactlyOneOf; import org.apache.storm.validation.ConfigValidationAnnotations.IsImplementationOfClass; import org.apache.storm.validation.ConfigValidationAnnotations.IsListEntryCustom; @@ -63,15 +68,8 @@ import org.apache.storm.validation.ConfigValidationAnnotations.IsNoDuplicateInList; import org.apache.storm.validation.ConfigValidationAnnotations.IsString; import org.apache.storm.validation.ConfigValidationAnnotations.NotNull; - import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - public class TestConfigValidate { @Test @@ -94,12 +92,14 @@ public void invalidPacemakerAuthTest() { @Test public void fallbackJavaSerializationFilterPatternTest() { - // A malformed JEP-290 pattern must be rejected at conf validation time, with the key named in the error. + // A malformed JEP-290 pattern must be rejected at conf validation time, with the key named + // in the error. for (String invalid : Arrays.asList("!", "maxbytes=not-a-number")) { Map conf = new HashMap<>(); conf.put(Config.TOPOLOGY_FALL_BACK_ON_JAVA_SERIALIZATION_FILTER, invalid); IllegalArgumentException ex = - assertThrows(IllegalArgumentException.class, () -> ConfigValidation.validateFields(conf)); + assertThrows(IllegalArgumentException.class, () -> ConfigValidation + .validateFields(conf)); // The validator reports the config field name (the convention for all validators here). assertTrue(ex.getMessage().contains("TOPOLOGY_FALL_BACK_ON_JAVA_SERIALIZATION_FILTER"), "message should name the offending field: " + ex.getMessage()); @@ -123,7 +123,8 @@ public void upstreamFeedbackRequiresEwmaTest() { conf.put(Config.TOPOLOGY_UPSTREAM_FEEDBACK_ENABLE, true); conf.put(Config.TOPOLOGY_STATS_EWMA_ENABLE, false); IllegalArgumentException ex = - assertThrows(IllegalArgumentException.class, () -> ConfigValidation.validateFields(conf)); + assertThrows(IllegalArgumentException.class, () -> ConfigValidation + .validateFields(conf)); assertTrue(ex.getMessage().contains(Config.TOPOLOGY_STATS_EWMA_ENABLE), "message should name the required key: " + ex.getMessage()); } @@ -291,7 +292,8 @@ public void testTopologyStatsEwmaSmoothingFactorCustomValidator() { ConfigValidation.validateFields(conf); for (Object notAllowedValue : new Object[]{0.0, -0.1, 1.9, "1.9"}) { conf.put(Config.TOPOLOGY_STATS_EWMA_SMOOTHING_FACTOR, notAllowedValue); - assertThrows(IllegalArgumentException.class, () -> ConfigValidation.validateFields(conf)); + assertThrows(IllegalArgumentException.class, () -> ConfigValidation + .validateFields(conf)); } } @@ -310,7 +312,7 @@ public void testWorkerChildoptsIsStringOrStringList() { Integer[] wrongStuff = { 1, 2, 3 }; failCases.add(Arrays.asList(wrongStuff)); - //worker.childopts validates + // worker.childopts validates for (Object value : passCases) { conf.put(Config.WORKER_CHILDOPTS, value); ConfigValidation.validateFields(conf); @@ -318,10 +320,11 @@ public void testWorkerChildoptsIsStringOrStringList() { for (Object value : failCases) { conf.put(Config.WORKER_CHILDOPTS, value); - assertThrows(IllegalArgumentException.class, () -> ConfigValidation.validateFields(conf)); + assertThrows(IllegalArgumentException.class, () -> ConfigValidation + .validateFields(conf)); } - //topology.worker.childopts validates + // topology.worker.childopts validates conf.clear(); for (Object value : passCases) { conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, value); @@ -330,7 +333,8 @@ public void testWorkerChildoptsIsStringOrStringList() { for (Object value : failCases) { conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, value); - assertThrows(IllegalArgumentException.class, () -> ConfigValidation.validateFields(conf)); + assertThrows(IllegalArgumentException.class, () -> ConfigValidation + .validateFields(conf)); } } @@ -375,20 +379,24 @@ public void testKryoRegValidator() { Object[] failCases = { ImmutableMap.of("f", "g"), ImmutableList.of(1), Collections.singletonList(ImmutableMap.of("a", 1))}; for (Object value : failCases) { - assertThrows(IllegalArgumentException.class, () -> validator.validateField("test", value)); + assertThrows(IllegalArgumentException.class, () -> validator.validateField("test", + value)); } // pass cases - validator.validateField("test", Arrays.asList("a", "b", "c", ImmutableMap.of("d", "e"), ImmutableMap.of("f", "g"))); + validator.validateField("test", Arrays.asList("a", "b", "c", ImmutableMap.of("d", "e"), + ImmutableMap.of("f", "g"))); } @Test public void testPowerOf2Validator() { PowerOf2Validator validator = new PowerOf2Validator(); - Object[] failCases = { 42.42, 42, -33, 23423423423.0, -32, -1, -0.00001, 0, -0, "Forty-two" }; + Object[] failCases = + { 42.42, 42, -33, 23423423423.0, -32, -1, -0.00001, 0, -0, "Forty-two" }; for (Object value : failCases) { - assertThrows(IllegalArgumentException.class, () -> validator.validateField("test", value)); + assertThrows(IllegalArgumentException.class, () -> validator.validateField("test", + value)); } Object[] passCases = { 64, 4294967296.0, 1, null }; @@ -410,7 +418,8 @@ public void testPositiveNumberValidator() { Object[] failCases = { -1.0, -1, -0.01, 0.0, 0, "43", "string" }; for (Object value : failCases) { - assertThrows(IllegalArgumentException.class, () -> validator.validateField("test", value)); + assertThrows(IllegalArgumentException.class, () -> validator.validateField("test", + value)); } Object[] passCasesIncludeZero = { null, 1.0, 0.01, 0, 2147483647, 0.0 }; @@ -422,7 +431,8 @@ public void testPositiveNumberValidator() { Object[] failCasesIncludeZero = { -1.0, -1, -0.01, "43", "string" }; for (Object value : failCasesIncludeZero) { - assertThrows(IllegalArgumentException.class, () -> validator.validateField("test", true, value)); + assertThrows(IllegalArgumentException.class, () -> validator.validateField("test", true, + value)); } } @@ -439,7 +449,8 @@ public void testIntegerValidator() { Object[] failCases = { 1.34, (long) Integer.MAX_VALUE + 1 }; for (Object value : failCases) { - assertThrows(IllegalArgumentException.class, () -> validator.validateField("test", value)); + assertThrows(IllegalArgumentException.class, () -> validator.validateField("test", + value)); } } @@ -453,10 +464,12 @@ public void testLongValidator() { validator.validateField("test", value); } - Object[] failCases = { 1.34, BigInteger.valueOf(Long.MAX_VALUE).add(BigInteger.valueOf(1L))}; + Object[] failCases = { 1.34, BigInteger.valueOf(Long.MAX_VALUE).add(BigInteger + .valueOf(1L))}; for (Object value : failCases) { - assertThrows(IllegalArgumentException.class, () -> validator.validateField("test", value)); + assertThrows(IllegalArgumentException.class, () -> validator.validateField("test", + value)); } } @@ -489,7 +502,8 @@ public void NoDuplicateInListValidator() { failCases.add(Arrays.asList(failCase2)); failCases.add(Arrays.asList(failCase3)); for (Object value : failCases) { - assertThrows(IllegalArgumentException.class, () -> validator.validateField("test", value)); + assertThrows(IllegalArgumentException.class, () -> validator.validateField("test", + value)); } } @@ -509,7 +523,8 @@ public void testListEntryTypeValidator() { } for (Object value : testCases1) { - assertThrows(IllegalArgumentException.class, () -> ListEntryTypeValidator.validateField("test", Number.class, value)); + assertThrows(IllegalArgumentException.class, () -> ListEntryTypeValidator + .validateField("test", Number.class, value)); } Object[] testCase3 = { 1000, 0, 1000 }; @@ -519,7 +534,8 @@ public void testListEntryTypeValidator() { testCases2.add(Arrays.asList(testCase4)); testCases2.add(Arrays.asList(testCase5)); for (Object value : testCases2) { - assertThrows(IllegalArgumentException.class, () -> ListEntryTypeValidator.validateField("test", String.class, value)); + assertThrows(IllegalArgumentException.class, () -> ListEntryTypeValidator + .validateField("test", String.class, value)); } for (Object value : testCases2) { ListEntryTypeValidator.validateField("test", Number.class, value); @@ -530,10 +546,12 @@ public void testListEntryTypeValidator() { testCases3.add(Arrays.asList(testCase6)); testCases3.add(Arrays.asList(testCase7)); for (Object value : testCases3) { - assertThrows(IllegalArgumentException.class, () -> ListEntryTypeValidator.validateField("test", String.class, value)); + assertThrows(IllegalArgumentException.class, () -> ListEntryTypeValidator + .validateField("test", String.class, value)); } for (Object value : testCases1) { - assertThrows(IllegalArgumentException.class, () -> ListEntryTypeValidator.validateField("test", Number.class, value)); + assertThrows(IllegalArgumentException.class, () -> ListEntryTypeValidator + .validateField("test", Number.class, value)); } } @@ -567,7 +585,8 @@ public void testMapEntryTypeAnnotation() { failCases.add(failCase2); for (Object value : failCases) { config.put(TestConfig.TEST_MAP_CONFIG, value); - assertThrows(IllegalArgumentException.class, () -> ConfigValidation.validateFields(config, Collections.singletonList(TestConfig.class))); + assertThrows(IllegalArgumentException.class, () -> ConfigValidation + .validateFields(config, Collections.singletonList(TestConfig.class))); } } @@ -611,7 +630,8 @@ public void testMapEntryCustomAnnotation() { failCases.add(failCase4); for (Object value : failCases) { config.put(TestConfig.TEST_MAP_CONFIG_2, value); - assertThrows(IllegalArgumentException.class, () -> ConfigValidation.validateFields(config, Collections.singletonList(TestConfig.class))); + assertThrows(IllegalArgumentException.class, () -> ConfigValidation + .validateFields(config, Collections.singletonList(TestConfig.class))); } } @@ -736,7 +756,8 @@ public void testExactlyOneOfCustomAnnotation() { for (Object value : failCases) { config.put(TestConfig.TEST_MAP_CONFIG_9, value); - assertThrows(IllegalArgumentException.class, () -> ConfigValidation.validateFields(config, Collections.singletonList(TestConfig.class))); + assertThrows(IllegalArgumentException.class, () -> ConfigValidation + .validateFields(config, Collections.singletonList(TestConfig.class))); } } @@ -764,7 +785,8 @@ public void testListEntryTypeAnnotation() { failCases.add(null); for (Object value : failCases) { config.put(TestConfig.TEST_MAP_CONFIG_3, value); - assertThrows(IllegalArgumentException.class, () -> ConfigValidation.validateFields(config, Collections.singletonList(TestConfig.class))); + assertThrows(IllegalArgumentException.class, () -> ConfigValidation + .validateFields(config, Collections.singletonList(TestConfig.class))); } } @@ -798,7 +820,8 @@ public void testListEntryCustomAnnotation() { failCases.add("b"); for (Object value : failCases) { config.put(TestConfig.TEST_MAP_CONFIG_4, value); - assertThrows(IllegalArgumentException.class, () -> ConfigValidation.validateFields(config, Collections.singletonList(TestConfig.class))); + assertThrows(IllegalArgumentException.class, () -> ConfigValidation + .validateFields(config, Collections.singletonList(TestConfig.class))); } } @@ -814,8 +837,9 @@ public void TestAcceptedStrings() { String[] failCases = { "aa", "bb", "cc", "abc", "a", "b", "c", "" }; for (Object value : failCases) { - config.put(TestConfig.TEST_MAP_CONFIG_5, value); - assertThrows(IllegalArgumentException.class, () -> ConfigValidation.validateFields(config, Collections.singletonList(TestConfig.class))); + config.put(TestConfig.TEST_MAP_CONFIG_5, value); + assertThrows(IllegalArgumentException.class, () -> ConfigValidation + .validateFields(config, Collections.singletonList(TestConfig.class))); } } @@ -858,7 +882,8 @@ public void TestImpersonationAclUserEntryValidator() { for (Object value : failCases) { config.put(TestConfig.TEST_MAP_CONFIG_6, value); - assertThrows(IllegalArgumentException.class, () -> ConfigValidation.validateFields(config, Collections.singletonList(TestConfig.class))); + assertThrows(IllegalArgumentException.class, () -> ConfigValidation + .validateFields(config, Collections.singletonList(TestConfig.class))); } } @@ -891,11 +916,11 @@ public void TestResourceAwareSchedulerUserPool() { failCase1.get("jerry").put("memory", 20148); failCase1.get("bobby").put("cpu", 20000); failCase1.get("bobby").put("memory", 40148); - //this will fail the test since user derek does not have an entry for memory + // this will fail the test since user derek does not have an entry for memory failCase1.get("derek").put("cpu", 30000); Map> failCase2 = new HashMap<>(); - //this will fail since jerry doesn't have either cpu or memory entries + // this will fail since jerry doesn't have either cpu or memory entries failCase2.put("jerry", new HashMap<>()); failCase2.put("bobby", new HashMap<>()); failCase2.put("derek", new HashMap<>()); @@ -909,7 +934,8 @@ public void TestResourceAwareSchedulerUserPool() { for (Object value : failCases) { config.put(TestConfig.TEST_MAP_CONFIG_7, value); - assertThrows(IllegalArgumentException.class, () -> ConfigValidation.validateFields(config, Collections.singletonList(TestConfig.class))); + assertThrows(IllegalArgumentException.class, () -> ConfigValidation + .validateFields(config, Collections.singletonList(TestConfig.class))); } } @@ -925,13 +951,15 @@ public void TestImplementsClassValidator() { config.put(TestConfig.TEST_MAP_CONFIG_8, value); ConfigValidation.validateFields(config, Collections.singletonList(TestConfig.class)); } - //will fail since org.apache.storm.nimbus.NimbusInfo doesn't implement or extend org.apache.storm.networktopography + // will fail since org.apache.storm.nimbus.NimbusInfo doesn't implement or extend + // org.apache.storm.networktopography // .DNSToSwitchMapping failCases.add("org.apache.storm.nimbus.NimbusInfo"); failCases.add(null); for (Object value : failCases) { config.put(TestConfig.TEST_MAP_CONFIG_8, value); - assertThrows(IllegalArgumentException.class, () -> ConfigValidation.validateFields(config, Collections.singletonList(TestConfig.class))); + assertThrows(IllegalArgumentException.class, () -> ConfigValidation + .validateFields(config, Collections.singletonList(TestConfig.class))); } } @@ -956,18 +984,22 @@ public static class TestConfig extends HashMap { @IsString(acceptedValues = { "aaa", "bbb", "ccc" }) public static final String TEST_MAP_CONFIG_5 = "test.map.config.5"; - @IsMapEntryCustom(keyValidatorClasses = { StringValidator.class }, valueValidatorClasses = { ImpersonationAclUserEntryValidator - .class }) + @IsMapEntryCustom(keyValidatorClasses = { StringValidator.class }, + valueValidatorClasses = { ImpersonationAclUserEntryValidator + .class }) public static final String TEST_MAP_CONFIG_6 = "test.map.config.6"; - @IsMapEntryCustom(keyValidatorClasses = { StringValidator.class }, valueValidatorClasses = { UserResourcePoolEntryValidator.class }) + @IsMapEntryCustom(keyValidatorClasses = { StringValidator.class }, + valueValidatorClasses = { UserResourcePoolEntryValidator.class }) public static final String TEST_MAP_CONFIG_7 = "test.map.config.7"; - @IsImplementationOfClass(implementsClass = org.apache.storm.networktopography.DNSToSwitchMapping.class) + @IsImplementationOfClass(implementsClass = + org.apache.storm.networktopography.DNSToSwitchMapping.class) @NotNull public static final String TEST_MAP_CONFIG_8 = "test.map.config.8"; - @IsExactlyOneOf(valueValidatorClasses = {ListOfListOfStringValidator.class, RasConstraintsTypeValidator.class}) + @IsExactlyOneOf(valueValidatorClasses = {ListOfListOfStringValidator.class, + RasConstraintsTypeValidator.class}) @NotNull public static final String TEST_MAP_CONFIG_9 = "test.map.config.9"; } diff --git a/storm-client/test/jvm/org/apache/storm/TestStormSubmitter.java b/storm-client/test/jvm/org/apache/storm/TestStormSubmitter.java index c68dd068f58..e862280c54b 100644 --- a/storm-client/test/jvm/org/apache/storm/TestStormSubmitter.java +++ b/storm-client/test/jvm/org/apache/storm/TestStormSubmitter.java @@ -18,6 +18,8 @@ package org.apache.storm; +import static org.junit.jupiter.api.Assertions.fail; + import java.util.Map; import org.apache.storm.generated.InvalidTopologyException; import org.apache.storm.generated.StormTopology; @@ -27,8 +29,6 @@ import org.apache.storm.topology.TopologyBuilder; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.fail; - public class TestStormSubmitter { @Test @@ -43,11 +43,14 @@ public void invalidTopologyWithoutSpout() { SubmitOptions opts = new SubmitOptions(TopologyInitialStatus.INACTIVE); try { - StormSubmitter.submitTopologyAs("test-topo-without-spout", topoConf, topology, opts, null, "none"); + StormSubmitter.submitTopologyAs("test-topo-without-spout", topoConf, topology, opts, + null, "none"); fail("Topology without spout should fail in submission"); } catch (InvalidTopologyException ex) { if (!ex.getMessage().contains(expectedExceptionMsgFragment)) { - String err = String.format("Topology submit failure should contain string \"%s\", but is \"%s\"", + String err = String + .format("Topology submit failure should contain string \"%s\", but is " + + "\"%s\"", expectedExceptionMsgFragment, ex.getMessage()); fail(err); } diff --git a/storm-client/test/jvm/org/apache/storm/TestStormTimer.java b/storm-client/test/jvm/org/apache/storm/TestStormTimer.java index 6709fdb1f85..99a8e312d7d 100644 --- a/storm-client/test/jvm/org/apache/storm/TestStormTimer.java +++ b/storm-client/test/jvm/org/apache/storm/TestStormTimer.java @@ -1,23 +1,27 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - import java.util.ArrayList; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; public class TestStormTimer { @@ -27,52 +31,59 @@ enum SCHEDULE_TYPE { AFTER_MILLISECONDS, AFTER_MILLISECONDS_WITH_JITTER, RECURRING, RECURRING_MS, RECURRING_WITH_JITTER + }; /** - * Test {@link StormTimer#schedule(int, Runnable)} and {@link StormTimer#schedule(int, Runnable, boolean, int)} + * Test {@link StormTimer#schedule(int, Runnable)} and {@link StormTimer#schedule(int, Runnable, + * boolean, int)} * for scheduling order under multithreaded environment. */ @Test public void testSchedule() { - StormTimer stormTimer = new StormTimer("testSchedule", (x,y) -> {}); + StormTimer stormTimer = new StormTimer("testSchedule", (x, y) -> {}); int threadCnt = 100; Assertions.assertTrue(schedule(stormTimer, threadCnt, SCHEDULE_TYPE.IMMEDIATE)); Assertions.assertTrue(schedule(stormTimer, threadCnt, SCHEDULE_TYPE.AFTER_1_SECOND)); Assertions.assertTrue(schedule(stormTimer, threadCnt, SCHEDULE_TYPE.IMMEDIATE_WITH_JITTER)); - Assertions.assertTrue(schedule(stormTimer, threadCnt, SCHEDULE_TYPE.AFTER_MILLISECONDS_WITH_JITTER)); + Assertions.assertTrue(schedule(stormTimer, threadCnt, + SCHEDULE_TYPE.AFTER_MILLISECONDS_WITH_JITTER)); close(stormTimer); } /** * Test {@link StormTimer#scheduleMs(long, Runnable)} and - * {@link StormTimer#scheduleMs(long, Runnable, boolean, int)} for scheduling order under multithreaded environment. + * {@link StormTimer#scheduleMs(long, Runnable, boolean, int)} for scheduling order under + * multithreaded environment. */ @Test public void testScheduleMs() { - StormTimer stormTimer = new StormTimer("testScheduleMs", (x,y) -> {}); + StormTimer stormTimer = new StormTimer("testScheduleMs", (x, y) -> {}); int threadCnt = 100; Assertions.assertTrue(schedule(stormTimer, threadCnt, SCHEDULE_TYPE.AFTER_MILLISECONDS)); - Assertions.assertTrue(schedule(stormTimer, threadCnt, SCHEDULE_TYPE.AFTER_MILLISECONDS_WITH_JITTER)); + Assertions.assertTrue(schedule(stormTimer, threadCnt, + SCHEDULE_TYPE.AFTER_MILLISECONDS_WITH_JITTER)); close(stormTimer); } /** - * Test {@link StormTimer#scheduleRecurring(int, int, Runnable)} for scheduling order under multithreaded environment. + * Test {@link StormTimer#scheduleRecurring(int, int, Runnable)} for scheduling order under + * multithreaded environment. */ @Test public void scheduleRecurring() { - StormTimer stormTimer = new StormTimer("testScheduleMs", (x,y) -> {}); + StormTimer stormTimer = new StormTimer("testScheduleMs", (x, y) -> {}); int threadCnt = 10; Assertions.assertTrue(schedule(stormTimer, threadCnt, SCHEDULE_TYPE.RECURRING)); close(stormTimer); } /** - * Test {@link StormTimer#scheduleRecurringMs(long, long, Runnable)} for scheduling order under multithreaded environment. + * Test {@link StormTimer#scheduleRecurringMs(long, long, Runnable)} for scheduling order under + * multithreaded environment. */ @Test public void testScheduleRecurringMs() { - StormTimer stormTimer = new StormTimer("testScheduleRecurringMs", (x,y) -> {}); + StormTimer stormTimer = new StormTimer("testScheduleRecurringMs", (x, y) -> {}); int threadCnt = 10; Assertions.assertTrue(schedule(stormTimer, threadCnt, SCHEDULE_TYPE.RECURRING_MS)); close(stormTimer); @@ -84,7 +95,7 @@ public void testScheduleRecurringMs() { */ @Test public void testScheduleRecurringWithJitter() { - StormTimer stormTimer = new StormTimer("testScheduleRecurringWithJitter", (x,y) -> {}); + StormTimer stormTimer = new StormTimer("testScheduleRecurringWithJitter", (x, y) -> {}); int threadCnt = 10; Assertions.assertTrue(schedule(stormTimer, threadCnt, SCHEDULE_TYPE.RECURRING_WITH_JITTER)); close(stormTimer); @@ -100,9 +111,12 @@ private void close(StormTimer stormTimer) { /** * Schedule specified number of threads. The threads are not executed in the order in which - * they call {@link StormTimer#schedule(int, Runnable)}. Just check if all of them are called (a weaker guarantee). - * If the threads are executed in the order they are scheduled, then the counter will match the number in the thread. - * Which they dont. If all the threads are executed, then the total number will match the total jobs scheduled. + * they call {@link StormTimer#schedule(int, Runnable)}. Just check if all of them are called (a + * weaker guarantee). + * If the threads are executed in the order they are scheduled, then the counter will match the + * number in the thread. + * Which they dont. If all the threads are executed, then the total number will match the total + * jobs scheduled. * * @param stormTimer StormTimer stormTimer. * @param threadCnt Number of threads to fire. @@ -131,8 +145,9 @@ public boolean isSuccess() { return counterValueBeforeIncrement == runnableNum; } } + ScheduleRunnable[] runnables = new ScheduleRunnable[threadCnt]; - for (int i = 0; i < threadCnt ; i++) { + for (int i = 0; i < threadCnt; i++) { runnables[i] = new ScheduleRunnable(i); } long sleepMsBeforeCheck = 3000; @@ -144,7 +159,7 @@ public boolean isSuccess() { final int delayMs = 300; final int recurSecs = 3; final int recurMs = 300; - switch(scheduleType) { + switch (scheduleType) { case IMMEDIATE: delaySecs = 0; stormTimer.schedule(delaySecs, runnables[i]); @@ -170,19 +185,23 @@ public boolean isSuccess() { case RECURRING: stormTimer.scheduleRecurring(delaySecs, recurSecs, runnables[i]); sleepMsBeforeCheck = 10000 + delaySecs * 1000; - int sleepSecsBeforeCheck = (int)(sleepMsBeforeCheck / 1000); - expectedCounterAtCheck = ((sleepSecsBeforeCheck - delaySecs) / recurSecs) * threadCnt; + int sleepSecsBeforeCheck = (int) (sleepMsBeforeCheck / 1000); + expectedCounterAtCheck = + ((sleepSecsBeforeCheck - delaySecs) / recurSecs) * threadCnt; break; case RECURRING_MS: stormTimer.scheduleRecurringMs(delayMs, recurMs, runnables[i]); sleepMsBeforeCheck = 10000 + delayMs; - expectedCounterAtCheck = threadCnt * (int)((sleepMsBeforeCheck - delayMs) / recurMs); + expectedCounterAtCheck = + threadCnt * (int) ((sleepMsBeforeCheck - delayMs) / recurMs); break; case RECURRING_WITH_JITTER: - stormTimer.scheduleRecurringWithJitter(delaySecs, recurSecs, jitterMs, runnables[i]); + stormTimer.scheduleRecurringWithJitter(delaySecs, recurSecs, jitterMs, + runnables[i]); sleepMsBeforeCheck = 10000 + delaySecs * 1000; - sleepSecsBeforeCheck = (int)(sleepMsBeforeCheck / 1000); - expectedCounterAtCheck = ((sleepSecsBeforeCheck - delaySecs) / recurSecs) * threadCnt; + sleepSecsBeforeCheck = (int) (sleepMsBeforeCheck / 1000); + expectedCounterAtCheck = + ((sleepSecsBeforeCheck - delaySecs) / recurSecs) * threadCnt; break; default: // do nothing @@ -201,26 +220,31 @@ public boolean isSuccess() { for (int i = 0; i < threadCnt; i++) { ScheduleRunnable runnable = runnables[i]; if (!runnable.isSuccess()) { - errs.add(String.format("Runnable %d was expecting counter value %d but found %d", + errs.add(String + .format("Runnable %d was expecting counter value %d but found %d", runnable.runnableNum, runnable.runnableNum, runnable.counterValueBeforeIncrement)); } } Assertions.assertTrue(errs.isEmpty(), String.join(",\n\t", errs)); return errs.isEmpty(); } else { - // this weaker guarantee of total number of executions should succeed except for recurring schedule + // this weaker guarantee of total number of executions should succeed except for + // recurring schedule int actualCounter = counter.get(); - if (scheduleType == SCHEDULE_TYPE.RECURRING || - scheduleType == SCHEDULE_TYPE.RECURRING_MS || - scheduleType == SCHEDULE_TYPE.RECURRING_WITH_JITTER) { + if (scheduleType == SCHEDULE_TYPE.RECURRING + || scheduleType == SCHEDULE_TYPE.RECURRING_MS + || scheduleType == SCHEDULE_TYPE.RECURRING_WITH_JITTER) { if (expectedCounterAtCheck != counter.get()) { - System.err.printf("Ignoring count mismatch with recurring scheduleType of %s, expected=%d, actual=%d\n", + System.err + .printf("Ignoring count mismatch with recurring scheduleType of %s, " + + "expected=%d, actual=%d\n", scheduleType, expectedCounterAtCheck, actualCounter); } } else { - Assertions.assertEquals(expectedCounterAtCheck, actualCounter, "Number of runnables completed"); + Assertions.assertEquals(expectedCounterAtCheck, actualCounter, + "Number of runnables completed"); } return true; } } -} \ No newline at end of file +} diff --git a/storm-client/test/jvm/org/apache/storm/assignments/LocalAssignmentsBackendTest.java b/storm-client/test/jvm/org/apache/storm/assignments/LocalAssignmentsBackendTest.java index 96f5297e8a9..717e6049681 100644 --- a/storm-client/test/jvm/org/apache/storm/assignments/LocalAssignmentsBackendTest.java +++ b/storm-client/test/jvm/org/apache/storm/assignments/LocalAssignmentsBackendTest.java @@ -18,6 +18,9 @@ package org.apache.storm.assignments; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -30,10 +33,6 @@ import org.apache.storm.utils.ConfigUtils; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; - - public class LocalAssignmentsBackendTest { @Test @@ -43,7 +42,8 @@ public void testLocalAssignment() { Assignment ass1 = mockedAssignment(1); Assignment ass2 = mockedAssignment(2); - ILocalAssignmentsBackend backend = LocalAssignmentsBackendFactory.getBackend(ConfigUtils.readStormConfig()); + ILocalAssignmentsBackend backend = LocalAssignmentsBackendFactory.getBackend(ConfigUtils + .readStormConfig()); assertNull(backend.getAssignment(storm1)); backend.keepOrUpdateAssignment(storm1, ass1); backend.keepOrUpdateAssignment(storm2, ass2); @@ -66,7 +66,8 @@ public void testLocalIdInfo() { String id2 = "id2"; String id3 = "id3"; - ILocalAssignmentsBackend backend = LocalAssignmentsBackendFactory.getBackend(ConfigUtils.readStormConfig()); + ILocalAssignmentsBackend backend = LocalAssignmentsBackendFactory.getBackend(ConfigUtils + .readStormConfig()); assertNull(backend.getStormId(name3)); backend.keepStormId(name1, id1); backend.keepStormId(name2, id2); @@ -90,7 +91,8 @@ private Assignment mockedAssignment(int i) { Map, NodeInfo> executor_node_port = new HashMap<>(); Set nodePorts = new HashSet<>(); nodePorts.add(9723L); - executor_node_port.put(Collections.singletonList(i + 0L), new NodeInfo("node" + i, nodePorts)); + executor_node_port.put(Collections.singletonList(i + 0L), new NodeInfo("node" + i, + nodePorts)); ass.set_executor_node_port(executor_node_port); Map, Long> executor_start_time_secs = new HashMap<>(); executor_start_time_secs.put(Collections.singletonList(1L), 12345L); diff --git a/storm-client/test/jvm/org/apache/storm/blobstore/ClientBlobStoreTest.java b/storm-client/test/jvm/org/apache/storm/blobstore/ClientBlobStoreTest.java index f88e278260d..1e532b5196c 100644 --- a/storm-client/test/jvm/org/apache/storm/blobstore/ClientBlobStoreTest.java +++ b/storm-client/test/jvm/org/apache/storm/blobstore/ClientBlobStoreTest.java @@ -1,17 +1,26 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.blobstore; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + import java.util.HashMap; import java.util.Iterator; import java.util.Map; @@ -26,9 +35,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertThrows; - public class ClientBlobStoreTest { private ClientBlobStore client; @@ -98,7 +104,8 @@ public void testBloblStoreKeyWithUnicodesValidation() { BlobStore.validateKey("msg-kafka-ascii-11-148343436363-stormconf.ser"); } - private void createTestBlob(String testKey, SettableBlobMeta meta) throws AuthorizationException, KeyAlreadyExistsException { + private void createTestBlob(String testKey, + SettableBlobMeta meta) throws AuthorizationException, KeyAlreadyExistsException { AccessControl submitterAcl = BlobStoreAclHandler.parseAccessControl("u:tester:rwa"); meta.add_to_acl(submitterAcl); client.createBlob(testKey, meta); diff --git a/storm-client/test/jvm/org/apache/storm/bolt/TestJoinBolt.java b/storm-client/test/jvm/org/apache/storm/bolt/TestJoinBolt.java index 5af9ddc8130..19aeffcd95e 100644 --- a/storm-client/test/jvm/org/apache/storm/bolt/TestJoinBolt.java +++ b/storm-client/test/jvm/org/apache/storm/bolt/TestJoinBolt.java @@ -1,17 +1,26 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.bolt; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -25,8 +34,6 @@ import org.apache.storm.windowing.TupleWindow; import org.apache.storm.windowing.TupleWindowImpl; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; public class TestJoinBolt { String[] userFields = { "userId", "name", "city" }; @@ -106,20 +113,23 @@ private static TupleWindow makeTupleWindow(ArrayList... streams) { return new TupleWindowImpl(combined, null, null); } - private static ArrayList makeStream(String streamName, String[] fieldNames, Object[][] data, String srcComponentName) { + private static ArrayList makeStream(String streamName, String[] fieldNames, + Object[][] data, String srcComponentName) { ArrayList result = new ArrayList<>(); MockContext mockContext = new MockContext(fieldNames); for (Object[] record : data) { - TupleImpl rec = new TupleImpl(mockContext, Arrays.asList(record), srcComponentName, 0, streamName); + TupleImpl rec = new TupleImpl(mockContext, Arrays.asList(record), srcComponentName, 0, + streamName); result.add(rec); } return result; } - private static ArrayList makeNestedEventsStream(String streamName, String[] fieldNames, Object[][] records - , String srcComponentName) { + private static ArrayList makeNestedEventsStream(String streamName, String[] fieldNames, + Object[][] records, + String srcComponentName) { MockContext mockContext = new MockContext(new String[]{ "outer" }); ArrayList result = new ArrayList<>(records.length); @@ -133,7 +143,8 @@ private static ArrayList makeNestedEventsStream(String streamName, String ArrayList tupleValues = new ArrayList<>(1); tupleValues.add(recordMap); - TupleImpl tuple = new TupleImpl(mockContext, tupleValues, srcComponentName, 0, streamName); + TupleImpl tuple = new TupleImpl(mockContext, tupleValues, srcComponentName, 0, + streamName); result.add(tuple); } @@ -156,7 +167,8 @@ public void testTrivial() throws Exception { @Test public void testNestedKeys() throws Exception { - ArrayList userStream = makeNestedEventsStream("users", userFields, users, "usersSpout"); + ArrayList userStream = makeNestedEventsStream("users", userFields, users, + "usersSpout"); TupleWindow window = makeTupleWindow(userStream); JoinBolt bolt = new JoinBolt(JoinBolt.Selector.STREAM, "users", "outer.userId") .select("outer.name, outer.city"); @@ -278,14 +290,16 @@ public void testThreeStreamLeftJoin_2() throws Exception { JoinBolt bolt = new JoinBolt(JoinBolt.Selector.STREAM, "users", "city") .leftJoin("stores", "city", "users") - .leftJoin("cities", "cityName", "stores") // join against diff stream compared to testThreeStreamLeftJoin_1 + .leftJoin("cities", "cityName", + "stores") // join against diff stream compared to testThreeStreamLeftJoin_1 .select("name,storeName,city,country"); MockCollector collector = new MockCollector(); bolt.prepare(null, null, collector); bolt.execute(window); printResults(collector); - assertEquals(stores.length + 1, collector.actualResults.size()); // stores.length+1 as 2 users in Bengaluru + assertEquals(stores.length + 1, collector.actualResults + .size()); // stores.length+1 as 2 users in Bengaluru } @Test @@ -305,7 +319,8 @@ public void testThreeStreamMixedJoin() throws Exception { bolt.prepare(null, null, collector); bolt.execute(window); printResults(collector); - assertEquals(stores.length + 1, collector.actualResults.size()); // stores.length+1 as 2 users in Bengaluru + assertEquals(stores.length + 1, collector.actualResults + .size()); // stores.length+1 as 2 users in Bengaluru } static class MockCollector extends OutputCollector { diff --git a/storm-client/test/jvm/org/apache/storm/cluster/DaemonTypeTest.java b/storm-client/test/jvm/org/apache/storm/cluster/DaemonTypeTest.java index fc2aabfcf1a..427a35faa07 100644 --- a/storm-client/test/jvm/org/apache/storm/cluster/DaemonTypeTest.java +++ b/storm-client/test/jvm/org/apache/storm/cluster/DaemonTypeTest.java @@ -1,17 +1,26 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.cluster; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -23,9 +32,6 @@ import org.apache.storm.utils.ConfigUtils; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; - public class DaemonTypeTest { @Test @@ -48,8 +54,10 @@ public void getDefaultZkAclsSecureServerConf() { assertNull(DaemonType.UNKNOWN.getDefaultZkAcls(conf)); assertNull(DaemonType.PACEMAKER.getDefaultZkAcls(conf)); - assertEquals(DaemonType.NIMBUS_SUPERVISOR_ZK_ACLS, DaemonType.SUPERVISOR.getDefaultZkAcls(conf)); - assertEquals(DaemonType.NIMBUS_SUPERVISOR_ZK_ACLS, DaemonType.NIMBUS.getDefaultZkAcls(conf)); + assertEquals(DaemonType.NIMBUS_SUPERVISOR_ZK_ACLS, DaemonType.SUPERVISOR + .getDefaultZkAcls(conf)); + assertEquals(DaemonType.NIMBUS_SUPERVISOR_ZK_ACLS, DaemonType.NIMBUS + .getDefaultZkAcls(conf)); assertNull(DaemonType.WORKER.getDefaultZkAcls(conf)); } @@ -70,4 +78,4 @@ public void getDefaultZkAclsSecureWorkerConf() { expected.add(new ACL(ZooDefs.Perms.ALL, new Id("sasl", "nimbus"))); assertEquals(expected, DaemonType.WORKER.getDefaultZkAcls(conf)); } -} \ No newline at end of file +} diff --git a/storm-client/test/jvm/org/apache/storm/cluster/StormClusterStateImplTest.java b/storm-client/test/jvm/org/apache/storm/cluster/StormClusterStateImplTest.java index 0c88a59fea6..ab0861737b9 100644 --- a/storm-client/test/jvm/org/apache/storm/cluster/StormClusterStateImplTest.java +++ b/storm-client/test/jvm/org/apache/storm/cluster/StormClusterStateImplTest.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -43,10 +49,10 @@ public class StormClusterStateImplTest { public void init() throws Exception { storage = Mockito.mock(IStateStorage.class); context = new ClusterStateContext(); - state = new StormClusterStateImpl(storage, LocalAssignmentsBackendFactory.getDefault(), context, false /*solo*/); + state = new StormClusterStateImpl(storage, LocalAssignmentsBackendFactory.getDefault(), + context, false /*solo*/); } - @Test public void registeredCallback() { Mockito.verify(storage).register(ArgumentMatchers.any(ZKStateChangedCallback.class)); diff --git a/storm-client/test/jvm/org/apache/storm/daemon/metrics/ClientMetricsUtilsTest.java b/storm-client/test/jvm/org/apache/storm/daemon/metrics/ClientMetricsUtilsTest.java index 65a4e27f311..fcf99ce68fc 100644 --- a/storm-client/test/jvm/org/apache/storm/daemon/metrics/ClientMetricsUtilsTest.java +++ b/storm-client/test/jvm/org/apache/storm/daemon/metrics/ClientMetricsUtilsTest.java @@ -1,12 +1,17 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -67,9 +72,11 @@ public void getTimeUnitForConfig() { assertNull(ClientMetricsUtils.getTimeUnitForConfig(reporterConf, dummyKey)); reporterConf.put(dummyKey, "SECONDS"); - assertEquals(TimeUnit.SECONDS, ClientMetricsUtils.getTimeUnitForConfig(reporterConf, dummyKey)); + assertEquals(TimeUnit.SECONDS, ClientMetricsUtils.getTimeUnitForConfig(reporterConf, + dummyKey)); reporterConf.put(dummyKey, "MINUTES"); - assertEquals(TimeUnit.MINUTES, ClientMetricsUtils.getTimeUnitForConfig(reporterConf, dummyKey)); + assertEquals(TimeUnit.MINUTES, ClientMetricsUtils.getTimeUnitForConfig(reporterConf, + dummyKey)); } } \ No newline at end of file diff --git a/storm-client/test/jvm/org/apache/storm/daemon/worker/BackPressureTrackerTest.java b/storm-client/test/jvm/org/apache/storm/daemon/worker/BackPressureTrackerTest.java index a3963584108..2e6eea9221d 100644 --- a/storm-client/test/jvm/org/apache/storm/daemon/worker/BackPressureTrackerTest.java +++ b/storm-client/test/jvm/org/apache/storm/daemon/worker/BackPressureTrackerTest.java @@ -24,7 +24,6 @@ import static org.mockito.Mockito.when; import java.util.Collections; - import org.apache.storm.daemon.worker.BackPressureTracker.BackpressureState; import org.apache.storm.messaging.netty.BackPressureStatus; import org.apache.storm.metrics2.StormMetricRegistry; diff --git a/storm-client/test/jvm/org/apache/storm/daemon/worker/LogConfigManagerTest.java b/storm-client/test/jvm/org/apache/storm/daemon/worker/LogConfigManagerTest.java index 53f84cb2c8c..29d7a29f59c 100644 --- a/storm-client/test/jvm/org/apache/storm/daemon/worker/LogConfigManagerTest.java +++ b/storm-client/test/jvm/org/apache/storm/daemon/worker/LogConfigManagerTest.java @@ -1,42 +1,46 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

      - * http://www.apache.org/licenses/LICENSE-2.0 - *

      - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

      http://www.apache.org/licenses/LICENSE-2.0 + * + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.daemon.worker; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; + import java.util.TreeMap; import java.util.concurrent.atomic.AtomicReference; - import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.core.LoggerContext; import org.apache.storm.generated.LogConfig; import org.apache.storm.generated.LogLevel; import org.apache.storm.generated.LogLevelAction; -import org.apache.storm.utils.Time; import org.apache.storm.utils.Time.SimulatedTime; +import org.apache.storm.utils.Time; import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.eq; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.verify; - - public class LogConfigManagerTest { private static final Logger LOG = LoggerFactory.getLogger(LogConfigManagerTest.class); @@ -101,7 +105,8 @@ public void testLogResetResetsDoesNothingForEmptyLogConfig() { LogConfigManager underTest = spy(new LogConfigManagerUnderTest(atomConf)); underTest.resetLogLevels(); assertEquals(new TreeMap<>(), atomConf.get()); - verify(underTest, never()).setLoggerLevel(any(LoggerContext.class), anyString(), anyString()); + verify(underTest, never()).setLoggerLevel(any(LoggerContext.class), anyString(), + anyString()); } @Test @@ -115,7 +120,8 @@ public void testLogResetResetsRootLoggerIfSet() { LogConfigManager underTest = spy(new LogConfigManagerUnderTest(atomConf)); underTest.resetLogLevels(); assertEquals(new TreeMap<>(), atomConf.get()); - verify(underTest).setLoggerLevel(any(LoggerContext.class), eq(LogManager.ROOT_LOGGER_NAME), eq("WARN")); + verify(underTest).setLoggerLevel(any(LoggerContext.class), + eq(LogManager.ROOT_LOGGER_NAME), eq("WARN")); } } @@ -138,25 +144,29 @@ public void testLogResetProperlyResetLogLevelAfterTimeout() { underTest.resetLogLevels(); assertEquals(expected, atomConf.get()); - verify(underTest, never()).setLoggerLevel(any(LoggerContext.class), eq(LogManager.ROOT_LOGGER_NAME), anyString()); + verify(underTest, never()).setLoggerLevel(any(LoggerContext.class), + eq(LogManager.ROOT_LOGGER_NAME), anyString()); // 11 seconds passed by, not timing out Time.advanceTimeSecs(11); underTest.resetLogLevels(); assertEquals(expected, atomConf.get()); - verify(underTest, never()).setLoggerLevel(any(LoggerContext.class), eq(LogManager.ROOT_LOGGER_NAME), anyString()); + verify(underTest, never()).setLoggerLevel(any(LoggerContext.class), + eq(LogManager.ROOT_LOGGER_NAME), anyString()); // 22 seconds passed by, still not timing out Time.advanceTimeSecs(11); underTest.resetLogLevels(); assertEquals(expected, atomConf.get()); - verify(underTest, never()).setLoggerLevel(any(LoggerContext.class), eq(LogManager.ROOT_LOGGER_NAME), anyString()); + verify(underTest, never()).setLoggerLevel(any(LoggerContext.class), + eq(LogManager.ROOT_LOGGER_NAME), anyString()); // 33 seconds passed by, timed out Time.advanceTimeSecs(11); underTest.resetLogLevels(); assertEquals(new TreeMap<>(), atomConf.get()); - verify(underTest).setLoggerLevel(any(LoggerContext.class), eq(LogManager.ROOT_LOGGER_NAME), eq("WARN")); + verify(underTest).setLoggerLevel(any(LoggerContext.class), + eq(LogManager.ROOT_LOGGER_NAME), eq("WARN")); } } @@ -173,9 +183,12 @@ public void testLogResetsNamedLoggersWithPastTimeout() { LogConfigManager underTest = spy(new LogConfigManagerUnderTest(atomConf)); underTest.resetLogLevels(); assertEquals(new TreeMap<>(), atomConf.get()); - verify(underTest).setLoggerLevel(any(LoggerContext.class), eq("my_debug_logger"), eq("INFO")); - verify(underTest).setLoggerLevel(any(LoggerContext.class), eq("my_info_logger"), eq("WARN")); - verify(underTest).setLoggerLevel(any(LoggerContext.class), eq("my_error_logger"), eq("INFO")); + verify(underTest).setLoggerLevel(any(LoggerContext.class), eq("my_debug_logger"), + eq("INFO")); + verify(underTest).setLoggerLevel(any(LoggerContext.class), eq("my_info_logger"), + eq("WARN")); + verify(underTest).setLoggerLevel(any(LoggerContext.class), eq("my_error_logger"), + eq("INFO")); } } @@ -214,7 +227,8 @@ public void testProcessLogConfigChangeThrowsIllegalArgumentExceptionWhenTargetLo logLevel.set_reset_log_level("INFO"); logConfig.put_to_named_logger_level("RESET_LOG", logLevel); - assertThrows(IllegalArgumentException.class, () -> logConfigManager.processLogConfigChange(logConfig)); + assertThrows(IllegalArgumentException.class, () -> logConfigManager + .processLogConfigChange(logConfig)); } @Test @@ -247,20 +261,25 @@ public void testProcessRootLogLevelToDebugSetsLoggerAndTimeout() { underTest.processLogConfigChange(mockConfig); verify(underTest).setLoggerLevel(any(LoggerContext.class), eq(""), eq("DEBUG")); - verify(underTest).setLoggerLevel(any(LoggerContext.class), eq("my_debug_logger"), eq("DEBUG")); - verify(underTest).setLoggerLevel(any(LoggerContext.class), eq("my_info_logger"), eq("INFO")); - verify(underTest).setLoggerLevel(any(LoggerContext.class), eq("my_error_logger"), eq("ERROR")); + verify(underTest).setLoggerLevel(any(LoggerContext.class), eq("my_debug_logger"), + eq("DEBUG")); + verify(underTest).setLoggerLevel(any(LoggerContext.class), eq("my_info_logger"), + eq("INFO")); + verify(underTest).setLoggerLevel(any(LoggerContext.class), eq("my_error_logger"), + eq("ERROR")); } } public static class LogConfigManagerUnderTest extends LogConfigManager { - public LogConfigManagerUnderTest(AtomicReference> latestLogConfig) { + public LogConfigManagerUnderTest(AtomicReference> latestLogConfig) { super(latestLogConfig); } @Override - public void setLoggerLevel(LoggerContext logContext, String loggerName, String newLevelStr) { - //NOOP, we don't actually want to change log levels for tests + public void setLoggerLevel(LoggerContext logContext, String loggerName, + String newLevelStr) { + // NOOP, we don't actually want to change log levels for tests } } } diff --git a/storm-client/test/jvm/org/apache/storm/daemon/worker/TestUtilsForWorkerState.java b/storm-client/test/jvm/org/apache/storm/daemon/worker/TestUtilsForWorkerState.java index 3941f1778e2..89245b2ff01 100644 --- a/storm-client/test/jvm/org/apache/storm/daemon/worker/TestUtilsForWorkerState.java +++ b/storm-client/test/jvm/org/apache/storm/daemon/worker/TestUtilsForWorkerState.java @@ -16,6 +16,15 @@ package org.apache.storm.daemon.worker; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.function.Supplier; import org.apache.storm.Config; import org.apache.storm.cluster.IStateStorage; import org.apache.storm.cluster.IStormClusterState; @@ -30,22 +39,13 @@ import org.apache.storm.thrift.TException; import org.apache.storm.utils.SupervisorIfaceFactory; -import java.io.IOException; -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; -import java.util.function.Supplier; - -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - public class TestUtilsForWorkerState { public static final String RESOURCE_KEY = "resource-key"; public static final String RESOURCE_VALUE = "resource-value"; - public static WorkerState getWorkerState(Map conf, String topologyId) throws IOException, TException { + public static WorkerState getWorkerState(Map conf, + String topologyId) throws IOException, TException { IContext context = mock(IContext.class); String assignmentId = null; @@ -61,12 +61,14 @@ public static WorkerState getWorkerState(Map conf, String topolo String workerId = null; Map topologyConf = new HashMap<>(); - topologyConf.put(Config.STORM_MESSAGING_TRANSPORT, "org.apache.storm.messaging.netty.Context"); + topologyConf.put(Config.STORM_MESSAGING_TRANSPORT, + "org.apache.storm.messaging.netty.Context"); topologyConf.put(Config.STORM_MESSAGING_NETTY_CLIENT_WORKER_THREADS, 1); topologyConf.put(Config.TOPOLOGY_EXECUTOR_RECEIVE_BUFFER_SIZE, 32768); topologyConf.put(Config.TOPOLOGY_PRODUCER_BATCH_SIZE, 1); topologyConf.put(Config.TOPOLOGY_EXECUTOR_OVERFLOW_LIMIT, 0); - topologyConf.put(Config.TOPOLOGY_BACKPRESSURE_WAIT_STRATEGY, "org.apache.storm.policy.WaitStrategyProgressive"); + topologyConf.put(Config.TOPOLOGY_BACKPRESSURE_WAIT_STRATEGY, + "org.apache.storm.policy.WaitStrategyProgressive"); topologyConf.put(Config.TOPOLOGY_BACKPRESSURE_WAIT_PROGRESSIVE_LEVEL1_COUNT, 1); topologyConf.put(Config.TOPOLOGY_BACKPRESSURE_WAIT_PROGRESSIVE_LEVEL2_COUNT, 1000); topologyConf.put(Config.TOPOLOGY_BACKPRESSURE_WAIT_PROGRESSIVE_LEVEL3_SLEEP_MILLIS, 1); @@ -84,7 +86,8 @@ public static WorkerState getWorkerState(Map conf, String topolo StormMetricRegistry metricRegistry = mock(StormMetricRegistry.class); Credentials initialCredentials = null; - WorkerState workerState = new WorkerState(conf, context, topologyId, assignmentId, supervisorIfaceSupplier, port, workerId, + WorkerState workerState = new WorkerState(conf, context, topologyId, assignmentId, + supervisorIfaceSupplier, port, workerId, topologyConf, stateStorage, stormClusterState, autoCreds, metricRegistry, initialCredentials); return workerState; diff --git a/storm-client/test/jvm/org/apache/storm/daemon/worker/WorkerStateTest.java b/storm-client/test/jvm/org/apache/storm/daemon/worker/WorkerStateTest.java index d738f61ef77..4cb3f44c14f 100644 --- a/storm-client/test/jvm/org/apache/storm/daemon/worker/WorkerStateTest.java +++ b/storm-client/test/jvm/org/apache/storm/daemon/worker/WorkerStateTest.java @@ -16,24 +16,6 @@ package org.apache.storm.daemon.worker; -import org.apache.storm.Config; -import org.apache.storm.daemon.supervisor.AdvancedFSOps; -import org.apache.storm.generated.StormTopology; -import org.apache.storm.thrift.TException; -import org.apache.storm.tuple.AddressedTuple; -import org.apache.storm.tuple.Tuple; -import org.apache.storm.utils.ConfigUtils; -import org.apache.storm.utils.Utils; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -44,6 +26,23 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.storm.Config; +import org.apache.storm.daemon.supervisor.AdvancedFSOps; +import org.apache.storm.generated.StormTopology; +import org.apache.storm.thrift.TException; +import org.apache.storm.tuple.AddressedTuple; +import org.apache.storm.tuple.Tuple; +import org.apache.storm.utils.ConfigUtils; +import org.apache.storm.utils.Utils; +import org.junit.jupiter.api.Test; + public class WorkerStateTest { @Test @@ -55,16 +54,19 @@ public void testWorkerHooksLifecycle() throws TException, IOException { Map conf = new HashMap<>(); conf.put(Config.TOPOLOGY_WORKER_SHARED_THREAD_POOL_SIZE, 1); - List workerHookBuffers = Collections.singletonList(ByteBuffer.wrap(Utils.javaSerialize(new TestUtilsForWorkerState.StateTrackingWorkerHook()))); + List workerHookBuffers = Collections.singletonList(ByteBuffer.wrap(Utils + .javaSerialize(new TestUtilsForWorkerState.StateTrackingWorkerHook()))); String topologyId = "1"; StormTopology topology = mock(StormTopology.class); when(topology.deepCopy()).thenReturn(topology); when(topology.is_set_worker_hooks()).thenReturn(true); when(topology.get_worker_hooks()).thenReturn(workerHookBuffers); - when(mockedConfigUtils.readSupervisorTopologyImpl(eq(conf), eq(topologyId), any(AdvancedFSOps.class))).thenReturn(topology); + when(mockedConfigUtils.readSupervisorTopologyImpl(eq(conf), eq(topologyId), + any(AdvancedFSOps.class))).thenReturn(topology); WorkerState workerState = TestUtilsForWorkerState.getWorkerState(conf, topologyId); - TestUtilsForWorkerState.StateTrackingWorkerHook workerHook = workerState.getDeserializedWorkerHooks().stream() + TestUtilsForWorkerState.StateTrackingWorkerHook workerHook = workerState + .getDeserializedWorkerHooks().stream() .filter(iwh -> iwh instanceof TestUtilsForWorkerState.StateTrackingWorkerHook) .map(iwh -> (TestUtilsForWorkerState.StateTrackingWorkerHook) iwh) .findFirst() @@ -91,20 +93,25 @@ public void testVisibilityOfUserResource() throws IOException, TException { Map conf = new HashMap<>(); conf.put(Config.TOPOLOGY_WORKER_SHARED_THREAD_POOL_SIZE, 1); - TestUtilsForWorkerState.ResourceInitializingWorkerHook workerHook = new TestUtilsForWorkerState.ResourceInitializingWorkerHook(); - List workerHookBuffers = Collections.singletonList(ByteBuffer.wrap(Utils.javaSerialize(workerHook))); + TestUtilsForWorkerState.ResourceInitializingWorkerHook workerHook = + new TestUtilsForWorkerState.ResourceInitializingWorkerHook(); + List workerHookBuffers = Collections.singletonList(ByteBuffer.wrap(Utils + .javaSerialize(workerHook))); String topologyId = "1"; StormTopology topology = mock(StormTopology.class); when(topology.deepCopy()).thenReturn(topology); when(topology.is_set_worker_hooks()).thenReturn(true); when(topology.get_worker_hooks()).thenReturn(workerHookBuffers); - when(mockedConfigUtils.readSupervisorTopologyImpl(eq(conf), eq(topologyId), any(AdvancedFSOps.class))).thenReturn(topology); + when(mockedConfigUtils.readSupervisorTopologyImpl(eq(conf), eq(topologyId), + any(AdvancedFSOps.class))).thenReturn(topology); WorkerState workerState = TestUtilsForWorkerState.getWorkerState(conf, topologyId); - assertNull(workerState.getWorkerTopologyContext().getResource(TestUtilsForWorkerState.RESOURCE_KEY)); + assertNull(workerState.getWorkerTopologyContext() + .getResource(TestUtilsForWorkerState.RESOURCE_KEY)); workerState.runWorkerStartHooks(); - assertEquals(TestUtilsForWorkerState.RESOURCE_VALUE, workerState.getWorkerTopologyContext().getResource(TestUtilsForWorkerState.RESOURCE_KEY)); + assertEquals(TestUtilsForWorkerState.RESOURCE_VALUE, workerState + .getWorkerTopologyContext().getResource(TestUtilsForWorkerState.RESOURCE_KEY)); workerState.runWorkerShutdownHooks(); } finally { @@ -124,7 +131,8 @@ public void testTransferLocalBatchDropsTuplesForUnknownTasks() throws TException String topologyId = "1"; StormTopology topology = mock(StormTopology.class); when(topology.deepCopy()).thenReturn(topology); - when(mockedConfigUtils.readSupervisorTopologyImpl(eq(conf), eq(topologyId), any(AdvancedFSOps.class))).thenReturn(topology); + when(mockedConfigUtils.readSupervisorTopologyImpl(eq(conf), eq(topologyId), + any(AdvancedFSOps.class))).thenReturn(topology); WorkerState workerState = TestUtilsForWorkerState.getWorkerState(conf, topologyId); @@ -133,7 +141,8 @@ public void testTransferLocalBatchDropsTuplesForUnknownTasks() throws TException AddressedTuple tupleForUnknownTask = new AddressedTuple(42, mock(Tuple.class)); assertDoesNotThrow(() -> - workerState.transferLocalBatch(new ArrayList<>(Collections.singletonList(tupleForUnknownTask)))); + workerState.transferLocalBatch(new ArrayList<>(Collections + .singletonList(tupleForUnknownTask)))); } finally { ConfigUtils.setInstance(previousConfigUtils); } diff --git a/storm-client/test/jvm/org/apache/storm/daemon/worker/WorkerTest.java b/storm-client/test/jvm/org/apache/storm/daemon/worker/WorkerTest.java index b03f3e7d46d..cdfd8723868 100644 --- a/storm-client/test/jvm/org/apache/storm/daemon/worker/WorkerTest.java +++ b/storm-client/test/jvm/org/apache/storm/daemon/worker/WorkerTest.java @@ -1,24 +1,29 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

      * http://www.apache.org/licenses/LICENSE-2.0 *

      - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.daemon.worker; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + import org.apache.storm.messaging.ConnectionWithStatus; import org.junit.jupiter.api.Test; import org.mockito.Mockito; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - public class WorkerTest { @Test public void testWorkerIsConnectionReady() { diff --git a/storm-client/test/jvm/org/apache/storm/dependency/DependencyPropertiesParserTest.java b/storm-client/test/jvm/org/apache/storm/dependency/DependencyPropertiesParserTest.java index 7b38eca086e..31c0589a3a3 100644 --- a/storm-client/test/jvm/org/apache/storm/dependency/DependencyPropertiesParserTest.java +++ b/storm-client/test/jvm/org/apache/storm/dependency/DependencyPropertiesParserTest.java @@ -1,27 +1,32 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.dependency; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + import java.io.File; import java.util.HashMap; import java.util.List; import java.util.Map; - import org.apache.storm.shade.net.minidev.json.JSONValue; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertThrows; - -import static org.junit.jupiter.api.Assertions.assertEquals; public class DependencyPropertiesParserTest { private final DependencyPropertiesParser sut = new DependencyPropertiesParser(); @@ -50,8 +55,10 @@ public void parsePackagesProperties() { Map parsed = sut.parseArtifactsProperties(testJson); assertEquals(2, parsed.size()); - assertEquals("storm-core-1.0.0.jar", parsed.get("org.apache.storm:storm-core:1.0.0").getName()); - assertEquals("json-simple-1.1.jar", parsed.get("com.googlecode.json-simple:json-simple:1.1").getName()); + assertEquals("storm-core-1.0.0.jar", parsed.get("org.apache.storm:storm-core:1.0.0") + .getName()); + assertEquals("json-simple-1.1.jar", parsed.get("com.googlecode.json-simple:json-simple:1.1") + .getName()); } @Test diff --git a/storm-client/test/jvm/org/apache/storm/dependency/DependencyUploaderTest.java b/storm-client/test/jvm/org/apache/storm/dependency/DependencyUploaderTest.java index e549fc83b5c..14669843f8e 100644 --- a/storm-client/test/jvm/org/apache/storm/dependency/DependencyUploaderTest.java +++ b/storm-client/test/jvm/org/apache/storm/dependency/DependencyUploaderTest.java @@ -18,6 +18,23 @@ package org.apache.storm.dependency; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.contains; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + import java.io.BufferedWriter; import java.io.File; import java.io.FileWriter; @@ -44,23 +61,6 @@ import org.mockito.ArgumentCaptor; import org.mockito.stubbing.Answer; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyInt; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.ArgumentMatchers.contains; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.doNothing; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - public class DependencyUploaderTest { private DependencyUploader sut; @@ -127,12 +127,15 @@ public void uploadFilesWhichOneOfThemIsFailedToBeUploaded() throws Exception { when(mockFile2.toPath()).thenThrow(new RuntimeException("just for test!")); String mockFileFileNameWithoutExtension = Files.getNameWithoutExtension(mockFile.getName()); - String mockFile2FileNameWithoutExtension = Files.getNameWithoutExtension(mockFile2.getName()); + String mockFile2FileNameWithoutExtension = Files.getNameWithoutExtension(mockFile2 + .getName()); // we skip uploading first one since we want to test rollback, not upload - when(mockBlobStore.getBlobMeta(contains(mockFileFileNameWithoutExtension))).thenReturn(new ReadableBlobMeta()); + when(mockBlobStore.getBlobMeta(contains(mockFileFileNameWithoutExtension))) + .thenReturn(new ReadableBlobMeta()); // we try uploading second one and it should be failed throwing RuntimeException - when(mockBlobStore.getBlobMeta(contains(mockFile2FileNameWithoutExtension))).thenThrow(new KeyNotFoundException()); + when(mockBlobStore.getBlobMeta(contains(mockFile2FileNameWithoutExtension))) + .thenThrow(new KeyNotFoundException()); List dependencies = Lists.newArrayList(mockFile, mockFile2); @@ -163,11 +166,13 @@ public void uploadFiles() throws Exception { doAnswer(incrementCounter).when(mockOutputStream).write(anyInt()); doAnswer(incrementCounter).when(mockOutputStream).write(any(byte[].class)); - doAnswer(incrementCounter).when(mockOutputStream).write(any(byte[].class), anyInt(), anyInt()); + doAnswer(incrementCounter).when(mockOutputStream).write(any(byte[].class), anyInt(), + anyInt()); doNothing().when(mockOutputStream).close(); when(mockBlobStore.getBlobMeta(anyString())).thenThrow(new KeyNotFoundException()); - when(mockBlobStore.createBlob(anyString(), any(SettableBlobMeta.class))).thenReturn(mockOutputStream); + when(mockBlobStore.createBlob(anyString(), any(SettableBlobMeta.class))) + .thenReturn(mockOutputStream); File mockFile = createTemporaryDummyFile(); String mockFileFileNameWithoutExtension = Files.getNameWithoutExtension(mockFile.getName()); @@ -180,14 +185,15 @@ public void uploadFiles() throws Exception { assertTrue(counter.get() > 0); verify(mockOutputStream).close(); - ArgumentCaptor blobMetaArgumentCaptor = ArgumentCaptor.forClass(SettableBlobMeta.class); + ArgumentCaptor blobMetaArgumentCaptor = ArgumentCaptor + .forClass(SettableBlobMeta.class); verify(mockBlobStore).createBlob(anyString(), blobMetaArgumentCaptor.capture()); SettableBlobMeta actualBlobMeta = blobMetaArgumentCaptor.getValue(); List actualAcls = actualBlobMeta.get_acl(); assertTrue(actualAcls.contains(new AccessControl(AccessControlType.USER, - BlobStoreAclHandler.READ | BlobStoreAclHandler.WRITE | - BlobStoreAclHandler.ADMIN))); + BlobStoreAclHandler.READ | BlobStoreAclHandler.WRITE + | BlobStoreAclHandler.ADMIN))); assertTrue(actualAcls.contains(new AccessControl(AccessControlType.OTHER, BlobStoreAclHandler.READ))); } @@ -246,9 +252,11 @@ public void uploadArtifactsWhichOneOfThemIsFailedToBeUploaded() throws Exception when(mockFile2.toPath()).thenThrow(new RuntimeException("just for test!")); // we skip uploading first one since we don't test upload for now - when(mockBlobStore.getBlobMeta(contains(expectedBlobKeyForArtifact))).thenReturn(new ReadableBlobMeta()); + when(mockBlobStore.getBlobMeta(contains(expectedBlobKeyForArtifact))) + .thenReturn(new ReadableBlobMeta()); // we try uploading second one and it should be failed throwing RuntimeException - when(mockBlobStore.getBlobMeta(contains(expectedBlobKeyForArtifact2))).thenThrow(new KeyNotFoundException()); + when(mockBlobStore.getBlobMeta(contains(expectedBlobKeyForArtifact2))) + .thenThrow(new KeyNotFoundException()); Map artifacts = new LinkedHashMap<>(); artifacts.put(artifact, mockFile); @@ -264,8 +272,10 @@ public void uploadArtifactsWhichOneOfThemIsFailedToBeUploaded() throws Exception verify(mockBlobStore).getBlobMeta(contains(expectedBlobKeyForArtifact)); verify(mockBlobStore).getBlobMeta(contains(expectedBlobKeyForArtifact2)); - // the artifacts uploaded before the failure are rolled back: their keys are unique to this upload, - // so nothing else can be referring to them and leaving them behind leaks blob store space forever + // the artifacts uploaded before the failure are rolled back: their keys are unique to this + // upload, + // so nothing else can be referring to them and leaving them behind leaks blob store space + // forever verify(mockBlobStore).deleteBlob(contains(expectedBlobKeyForArtifact)); verify(mockBlobStore, never()).deleteBlob(contains(expectedBlobKeyForArtifact2)); } @@ -283,11 +293,13 @@ public void uploadArtifacts() throws Exception { doAnswer(incrementCounter).when(mockOutputStream).write(anyInt()); doAnswer(incrementCounter).when(mockOutputStream).write(any(byte[].class)); - doAnswer(incrementCounter).when(mockOutputStream).write(any(byte[].class), anyInt(), anyInt()); + doAnswer(incrementCounter).when(mockOutputStream).write(any(byte[].class), anyInt(), + anyInt()); doNothing().when(mockOutputStream).close(); when(mockBlobStore.getBlobMeta(anyString())).thenThrow(new KeyNotFoundException()); - when(mockBlobStore.createBlob(anyString(), any(SettableBlobMeta.class))).thenReturn(mockOutputStream); + when(mockBlobStore.createBlob(anyString(), any(SettableBlobMeta.class))) + .thenReturn(mockOutputStream); String artifact = "group:artifact:1.0.0"; String expectedBlobKeyForArtifact = "group-artifact-1.0.0"; @@ -310,7 +322,8 @@ public void uploadArtifactsAssignsUniqueKeyPerUpload() throws Exception { doNothing().when(mockOutputStream).close(); when(mockBlobStore.getBlobMeta(anyString())).thenThrow(new KeyNotFoundException()); - when(mockBlobStore.createBlob(anyString(), any(SettableBlobMeta.class))).thenReturn(mockOutputStream); + when(mockBlobStore.createBlob(anyString(), any(SettableBlobMeta.class))) + .thenReturn(mockOutputStream); String artifact = "group:artifact:1.0.0"; Map artifacts = new LinkedHashMap<>(); @@ -321,7 +334,8 @@ public void uploadArtifactsAssignsUniqueKeyPerUpload() throws Exception { assertEquals(1, firstKeys.size()); assertEquals(1, secondKeys.size()); - // the key must not be derived from the artifact coordinate alone, otherwise a blob left behind + // the key must not be derived from the artifact coordinate alone, otherwise a blob left + // behind // by another submission is picked up instead of the artifact we just resolved assertNotEquals("dep-group-artifact-1.0.0.jar", firstKeys.get(0)); assertNotEquals(firstKeys.get(0), secondKeys.get(0)); @@ -331,7 +345,8 @@ public void uploadArtifactsAssignsUniqueKeyPerUpload() throws Exception { @Test public void uploadArtifactsFailsWhenKeyAlreadyExists() throws Exception { when(mockBlobStore.getBlobMeta(anyString())).thenThrow(new KeyNotFoundException()); - when(mockBlobStore.createBlob(anyString(), any(SettableBlobMeta.class))).thenThrow(new KeyAlreadyExistsException()); + when(mockBlobStore.createBlob(anyString(), any(SettableBlobMeta.class))) + .thenThrow(new KeyAlreadyExistsException()); Map artifacts = new LinkedHashMap<>(); artifacts.put("group:artifact:1.0.0", createTemporaryDummyFile()); diff --git a/storm-client/test/jvm/org/apache/storm/executor/ChildEwmaStatsTest.java b/storm-client/test/jvm/org/apache/storm/executor/ChildEwmaStatsTest.java index 2b4be5de9e7..45355fce826 100644 --- a/storm-client/test/jvm/org/apache/storm/executor/ChildEwmaStatsTest.java +++ b/storm-client/test/jvm/org/apache/storm/executor/ChildEwmaStatsTest.java @@ -1,25 +1,31 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.executor; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + import java.util.HashMap; import java.util.Map; import org.apache.storm.metrics2.TaskMetrics; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - /** * Unit tests for {@link ChildEwmaStats}: the per-source-task aggregation of downstream jitter * reports and the {@link ChildEwmaStats#compareByJitter} ordering used by @@ -71,12 +77,14 @@ public void update_latestReportOverwritesPrevious() { ChildEwmaStats stats = new ChildEwmaStats(true); stats.update(PARENT, CHILD_A, new EwmaFeedbackRecord(1, 1, 1)); stats.update(PARENT, CHILD_A, new EwmaFeedbackRecord(9, 9, 9)); - assertEquals(9.0, stats.getStats(PARENT).get(CHILD_A).get(TaskMetrics.METRIC_NAME_EXECUTE_JITTER)); + assertEquals(9.0, stats.getStats(PARENT).get(CHILD_A) + .get(TaskMetrics.METRIC_NAME_EXECUTE_JITTER)); } @Test public void update_absentMetricNotStored() { - // VOID-valued components are skipped by EwmaFeedbackRecord#forEachMetric, so they never reach the map. + // VOID-valued components are skipped by EwmaFeedbackRecord#forEachMetric, so they never + // reach the map. ChildEwmaStats stats = new ChildEwmaStats(true); stats.update(PARENT, CHILD_A, new EwmaFeedbackRecord(-1, -1, 5.0)); Map child = stats.getStats(PARENT).get(CHILD_A); diff --git a/storm-client/test/jvm/org/apache/storm/executor/EwmaFeedbackRecordTest.java b/storm-client/test/jvm/org/apache/storm/executor/EwmaFeedbackRecordTest.java index 38913d9007b..a9a2748d79c 100644 --- a/storm-client/test/jvm/org/apache/storm/executor/EwmaFeedbackRecordTest.java +++ b/storm-client/test/jvm/org/apache/storm/executor/EwmaFeedbackRecordTest.java @@ -1,17 +1,29 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.executor; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + import com.codahale.metrics.Gauge; import java.util.HashMap; import java.util.LinkedHashMap; @@ -29,12 +41,6 @@ import org.apache.storm.utils.Utils; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertInstanceOf; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - /** * Tests for {@link EwmaFeedbackRecord}, covering three concerns: *

        @@ -48,7 +54,8 @@ * suffixed names (one per {@code sourceComponent:sourceStream}), e.g. * {@code __execute-jitter-splitter:default}, so a bare {@code get("__execute-jitter")} never matches. * These tests assert the signal is actually discovered and aggregated — without it - * {@link org.apache.storm.grouping.JitterAwareStreamGrouping} silently degrades to round-robin. + * {@link org.apache.storm.grouping.JitterAwareStreamGrouping} silently degrades to + * round-robin. *
      */ public class EwmaFeedbackRecordTest { @@ -96,7 +103,8 @@ public void forEachMetric_emitsAllPresentMetricsWithCorrectNames() { @Test public void forEachMetric_skipsAbsentMetrics() { - // -1 is the VOID sentinel: an absent gauge must not be reported as a real (negative) measurement. + // -1 is the VOID sentinel: an absent gauge must not be reported as a real (negative) + // measurement. EwmaFeedbackRecord record = new EwmaFeedbackRecord(-1, 2.0, -1); Map collected = new LinkedHashMap<>(); @@ -108,7 +116,8 @@ public void forEachMetric_skipsAbsentMetrics() { @Test public void forEachMetric_emitsZeroJitter() { - // Zero is a legitimate value (jitter decays to 0 under stable latency) and must be distinct from VOID. + // Zero is a legitimate value (jitter decays to 0 under stable latency) and must be distinct + // from VOID. EwmaFeedbackRecord record = new EwmaFeedbackRecord(0.0, 0.0, 0.0); Map collected = new LinkedHashMap<>(); @@ -135,7 +144,8 @@ public void kryoRoundTrip_preservesRecord() { @Test public void kryoRoundTrip_insideFeedbackTuple() { - // Mirrors Executor#buildUpstreamFeedbackTuple: [TaskInfo, EwmaFeedbackRecord]. Both elements + // Mirrors Executor#buildUpstreamFeedbackTuple: [TaskInfo, EwmaFeedbackRecord]. Both + // elements // must survive the wire so the receiving task can rebuild its child stats. Map conf = conf(); KryoValuesSerializer serializer = new KryoValuesSerializer(conf); @@ -153,7 +163,8 @@ public void kryoRoundTrip_insideFeedbackTuple() { assertInstanceOf(EwmaFeedbackRecord.class, restored.get(1)); assertEquals(7, ((IMetricsConsumer.TaskInfo) restored.get(0)).srcTaskId); assertEquals(feedback, restored.get(1)); - assertTrue(((EwmaFeedbackRecord) restored.get(1)).processJitter() < 0, "VOID sentinel preserved"); + assertTrue(((EwmaFeedbackRecord) restored.get(1)).processJitter() < 0, + "VOID sentinel preserved"); } @Test @@ -161,24 +172,30 @@ public void fromWorkerState_discoversSuffixedExecuteJitterGauge() { StormMetricRegistry registry = new StormMetricRegistry(); TaskMetrics taskMetrics = newTaskMetrics(registry, ewmaConf(true)); - // Real path: registers "__execute-jitter-splitter:default" and feeds the RFC-1889 EWMA estimator. + // Real path: registers "__execute-jitter-splitter:default" and feeds the RFC-1889 EWMA + // estimator. // Varying latencies make the jitter estimate strictly positive. for (long latency : new long[]{10, 60, 15, 90, 20}) { taskMetrics.boltExecuteTuple("splitter", "default", latency); } - EwmaFeedbackRecord record = EwmaFeedbackRecord.fromWorkerState(workerStateFor(registry), TASK_ID); + EwmaFeedbackRecord record = EwmaFeedbackRecord.fromWorkerState(workerStateFor(registry), + TASK_ID); - // Regression guard: a bare get("__execute-jitter") misses the suffixed key and leaves this VOID (-1). + // Regression guard: a bare get("__execute-jitter") misses the suffixed key and leaves this + // VOID (-1). assertTrue(record.executeJitter() > 0, - "execute-jitter must be discovered from the suffixed gauge, got " + record.executeJitter()); + "execute-jitter must be discovered from the suffixed gauge, got " + record + .executeJitter()); - // Metrics that were never driven must stay VOID — proving the prefix filter does not cross-match + // Metrics that were never driven must stay VOID — proving the prefix filter does not + // cross-match // (e.g. __execute-jitter must not be picked up when asking for __process-jitter). assertEquals(VOID, record.processJitter()); assertEquals(VOID, record.completeJitter()); - // forEachMetric must surface the value under the canonical bare name consumed by ChildEwmaStats. + // forEachMetric must surface the value under the canonical bare name consumed by + // ChildEwmaStats. Map emitted = new HashMap<>(); record.forEachMetric(emitted::put); assertEquals(1, emitted.size()); @@ -190,7 +207,8 @@ public void fromWorkerState_aggregatesMaxAcrossMultipleSources() { StormMetricRegistry registry = new StormMetricRegistry(); TaskMetrics taskMetrics = newTaskMetrics(registry, ewmaConf(true)); - // Same task consuming from two upstream sources => two distinct __execute-jitter- gauges. + // Same task consuming from two upstream sources => two distinct __execute-jitter- + // gauges. for (long latency : new long[]{10, 20, 12, 18}) { taskMetrics.boltExecuteTuple("srcA", "default", latency); } @@ -207,7 +225,8 @@ public void fromWorkerState_aggregatesMaxAcrossMultipleSources() { .orElseThrow(() -> new AssertionError("no execute-jitter gauges registered")); assertTrue(expectedMax > 0, "precondition: at least one source produced positive jitter"); - EwmaFeedbackRecord record = EwmaFeedbackRecord.fromWorkerState(workerStateFor(registry), TASK_ID); + EwmaFeedbackRecord record = EwmaFeedbackRecord.fromWorkerState(workerStateFor(registry), + TASK_ID); assertEquals(expectedMax, record.executeJitter(), "fromWorkerState must report the max jitter across all of the task's source gauges"); } @@ -220,7 +239,8 @@ public void fromWorkerState_allVoidWhenNoJitterGaugesRegistered() { taskMetrics.boltExecuteTuple("splitter", "default", 42); - EwmaFeedbackRecord record = EwmaFeedbackRecord.fromWorkerState(workerStateFor(registry), TASK_ID); + EwmaFeedbackRecord record = EwmaFeedbackRecord.fromWorkerState(workerStateFor(registry), + TASK_ID); assertEquals(VOID, record.executeJitter()); assertEquals(VOID, record.processJitter()); assertEquals(VOID, record.completeJitter()); diff --git a/storm-client/test/jvm/org/apache/storm/executor/ExecutorTransferControlLaneTest.java b/storm-client/test/jvm/org/apache/storm/executor/ExecutorTransferControlLaneTest.java index 4b072be0f53..2c7c7a6cf6e 100644 --- a/storm-client/test/jvm/org/apache/storm/executor/ExecutorTransferControlLaneTest.java +++ b/storm-client/test/jvm/org/apache/storm/executor/ExecutorTransferControlLaneTest.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.executor; @@ -44,9 +50,12 @@ import org.junit.jupiter.api.Test; /** - * Verifies the control-lane routing in {@link ExecutorTransfer#tryTransferLocal}: whitelisted control streams - * (see {@link Constants#SYSTEM_CONTROL_STREAM_IDS}) must reach the destination's control lane regardless of - * pendingEmits ordering and data-queue saturation, while data tuples and the lane-disabled path must keep the + * Verifies the control-lane routing in {@link ExecutorTransfer#tryTransferLocal}: whitelisted + * control streams + * (see {@link Constants#SYSTEM_CONTROL_STREAM_IDS}) must reach the destination's control lane + * regardless of + * pendingEmits ordering and data-queue saturation, while data tuples and the lane-disabled path + * must keep the * original semantics. */ public class ExecutorTransferControlLaneTest { @@ -86,11 +95,13 @@ private ExecutorTransfer mkExecutorTransfer(JCQueue localQueue) { private JCQueue mkQueue(String name, int size, int controlQueueSize) { return new JCQueue(name, name, size, 0, 1, new WaitStrategyPark(100), "test", "test", - Collections.singletonList(DEST_TASK_ID), 6701, new StormMetricRegistry(), false, controlQueueSize); + Collections + .singletonList(DEST_TASK_ID), 6701, new StormMetricRegistry(), false, controlQueueSize); } private AddressedTuple mkTuple(String streamId) { - TupleImpl tuple = new TupleImpl(generalTopologyContext, new Values("v"), Constants.SYSTEM_COMPONENT_ID, + TupleImpl tuple = new TupleImpl(generalTopologyContext, new Values("v"), + Constants.SYSTEM_COMPONENT_ID, (int) Constants.SYSTEM_TASK_ID, streamId); return new AddressedTuple(DEST_TASK_ID, tuple); } @@ -116,12 +127,14 @@ public void testControlTupleBypassesPendingEmits() { ExecutorTransfer executorTransfer = mkExecutorTransfer(queue); Queue pendingEmits = new ArrayDeque<>(); - pendingEmits.add(mkTuple("default")); // non-empty: the data path would reject and append here + pendingEmits + .add(mkTuple("default")); // non-empty: the data path would reject and append here AddressedTuple controlTuple = mkTuple(Constants.SYSTEM_FLUSH_STREAM_ID); assertTrue(executorTransfer.tryTransferLocal(controlTuple, queue, pendingEmits), "control tuple must be reported as handled"); - assertEquals(1, pendingEmits.size(), "control tuple must not be queued behind pendingEmits"); + assertEquals(1, pendingEmits.size(), + "control tuple must not be queued behind pendingEmits"); assertEquals(Collections.singletonList(controlTuple), drain(queue)); } @@ -135,7 +148,8 @@ public void testControlTupleDeliveredWhenDataQueueFull() { AddressedTuple controlTuple = mkTuple(Constants.SYSTEM_TICK_STREAM_ID); assertTrue(executorTransfer.tryTransferLocal(controlTuple, queue, null)); - assertEquals(controlTuple, drain(queue).get(0), "control tuple must be drained ahead of the data backlog"); + assertEquals(controlTuple, drain(queue).get(0), + "control tuple must be drained ahead of the data backlog"); } @Test @@ -150,9 +164,11 @@ public void testControlTupleDroppedOnFullLaneStillReportedHandled() { assertTrue(accepted > 0 && accepted < 64); Queue pendingEmits = new ArrayDeque<>(); - assertTrue(executorTransfer.tryTransferLocal(mkTuple(Constants.SYSTEM_FLUSH_STREAM_ID), queue, pendingEmits), + assertTrue(executorTransfer.tryTransferLocal(mkTuple(Constants.SYSTEM_FLUSH_STREAM_ID), + queue, pendingEmits), "a dropped control tuple is self-healing and must be reported as handled"); - assertTrue(pendingEmits.isEmpty(), "a dropped control tuple must not fall back to pendingEmits"); + assertTrue(pendingEmits.isEmpty(), + "a dropped control tuple must not fall back to pendingEmits"); assertEquals(accepted, drain(queue).size()); } @@ -179,7 +195,8 @@ public void testControlTupleFollowsDataPathWhenLaneDisabled() { Queue pendingEmits = new ArrayDeque<>(); pendingEmits.add(mkTuple("default")); - assertFalse(executorTransfer.tryTransferLocal(mkTuple(Constants.SYSTEM_FLUSH_STREAM_ID), queue, pendingEmits), + assertFalse(executorTransfer.tryTransferLocal(mkTuple(Constants.SYSTEM_FLUSH_STREAM_ID), + queue, pendingEmits), "with the lane disabled, control tuples must keep the original data-path semantics"); assertEquals(2, pendingEmits.size()); } diff --git a/storm-client/test/jvm/org/apache/storm/executor/ExecutorTransferMultiThreadingTest.java b/storm-client/test/jvm/org/apache/storm/executor/ExecutorTransferMultiThreadingTest.java index 279864b3f1b..bb22a734bfa 100644 --- a/storm-client/test/jvm/org/apache/storm/executor/ExecutorTransferMultiThreadingTest.java +++ b/storm-client/test/jvm/org/apache/storm/executor/ExecutorTransferMultiThreadingTest.java @@ -1,17 +1,27 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.executor; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + import java.lang.reflect.Field; import java.util.Collections; import java.util.HashMap; @@ -42,18 +52,14 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - /** - * Some topologies might spawn extra threads inside components to perform real processing work and emit processed results. + * Some topologies might spawn extra threads inside components to perform real processing work and + * emit processed results. * This unit test is to mimic these scenarios in the {@link ExecutorTransfer} level * and make sure the tuples sent out in a multi-threading fashion to the workerTransferQueue * is properly handled and received by the remote Worker (consumer) * - * The topology structure this test mimics is + *

      The topology structure this test mimics is * {worker1: taskId=1, component="1"} --> {worker2: taskId=2, component="2"}. */ public class ExecutorTransferMultiThreadingTest { @@ -83,7 +89,7 @@ public void setup() throws NoSuchFieldException, IllegalAccessException { workerState = mock(WorkerState.class); when(workerState.getWorkerTopologyContext()).thenReturn(workerTopologyContext); Map receiveQMap = new HashMap<>(); - //local recvQ is not important in this test; simple mock it + // local recvQ is not important in this test; simple mock it receiveQMap.put(selfTaskId, mock(JCQueue.class)); when(workerState.getLocalReceiveQueues()).thenReturn(receiveQMap); when(workerState.getTopologyId()).thenReturn(topologyId); @@ -91,15 +97,20 @@ public void setup() throws NoSuchFieldException, IllegalAccessException { when(workerState.getMetricRegistry()).thenReturn(new StormMetricRegistry()); when(workerState.tryTransferRemote(any(), any(), any())).thenCallRealMethod(); - //the actual worker transfer queue to be used in this test - //taskId for worker transfer queue should be -1. - //But there is already one worker transfer queue initialized by WorkerTransfer class (taskId=-1). - //However the taskId is only used for metrics and it is not important here. Making it -100 to avoid collision. - transferQueue = new JCQueue("worker-transfer-queue", "worker-transfer-queue", 1024, 0, 1, new WaitStrategyPark(100), - workerState.getTopologyId(), Constants.SYSTEM_COMPONENT_ID, Collections.singletonList(-100), workerState.getPort(), + // the actual worker transfer queue to be used in this test + // taskId for worker transfer queue should be -1. + // But there is already one worker transfer queue initialized by WorkerTransfer class + // (taskId=-1). + // However the taskId is only used for metrics and it is not important here. Making it -100 + // to avoid collision. + transferQueue = new JCQueue("worker-transfer-queue", "worker-transfer-queue", 1024, 0, 1, + new WaitStrategyPark(100), + workerState.getTopologyId(), Constants.SYSTEM_COMPONENT_ID, Collections + .singletonList(-100), workerState.getPort(), workerState.getMetricRegistry()); - //Replace the transferQueue inside WorkerTransfer (inside WorkerState) with the customized transferQueue to be used in this test + // Replace the transferQueue inside WorkerTransfer (inside WorkerState) with the customized + // transferQueue to be used in this test WorkerTransfer workerTransfer = new WorkerTransfer(workerState, topoConf, 2); setPrivateField(workerTransfer, "transferQueue", transferQueue); setPrivateField(workerState, "workerTransfer", workerTransfer); @@ -107,7 +118,8 @@ public void setup() throws NoSuchFieldException, IllegalAccessException { generalTopologyContext = mock(GeneralTopologyContext.class); } - private void setPrivateField(Object target, String fieldName, Object fieldValue) throws NoSuchFieldException, IllegalAccessException { + private void setPrivateField(Object target, String fieldName, + Object fieldValue) throws NoSuchFieldException, IllegalAccessException { Field privateField = target.getClass().getDeclaredField(fieldName); privateField.setAccessible(true); privateField.set(target, fieldValue); @@ -115,13 +127,14 @@ private void setPrivateField(Object target, String fieldName, Object fieldValue) @Test public void testExecutorTransfer() throws InterruptedException { - //There is one ExecutorTransfer per executor + // There is one ExecutorTransfer per executor ExecutorTransfer executorTransfer = new ExecutorTransfer(workerState, topoConf); executorTransfer.initLocalRecvQueues(); ExecutorService executorService = Executors.newFixedThreadPool(5); - //There can be multiple producer threads sending out tuples inside each executor - //This mimics the case of multi-threading components where a component spawns extra threads to emit tuples. + // There can be multiple producer threads sending out tuples inside each executor + // This mimics the case of multi-threading components where a component spawns extra threads + // to emit tuples. int producerTaskNum = 10; Runnable[] producerTasks = new Runnable[producerTaskNum]; for (int i = 0; i < producerTaskNum; i++) { @@ -131,12 +144,13 @@ public void testExecutorTransfer() throws InterruptedException { executorService.submit(task); } - //give producers enough time to insert messages into the queue + // give producers enough time to insert messages into the queue executorService.awaitTermination(1000, TimeUnit.MILLISECONDS); - //consume all the tuples in the queue and deserialize them one by one - //this mimics a remote worker. - KryoTupleDeserializer deserializer = new KryoTupleDeserializer(topoConf, workerState.getWorkerTopologyContext()); + // consume all the tuples in the queue and deserialize them one by one + // this mimics a remote worker. + KryoTupleDeserializer deserializer = new KryoTupleDeserializer(topoConf, workerState + .getWorkerTopologyContext()); SingleThreadedConsumer consumer = new SingleThreadedConsumer(deserializer, producerTaskNum); transferQueue.consume(consumer); consumer.finalCheck(); @@ -145,7 +159,8 @@ public void testExecutorTransfer() throws InterruptedException { private Runnable createProducerTask(ExecutorTransfer executorTransfer) { return new Runnable() { - final Tuple tuple = new TupleImpl(generalTopologyContext, new Values(value1, value2), sourceComp, selfTaskId, "default"); + final Tuple tuple = new TupleImpl(generalTopologyContext, new Values(value1, value2), + sourceComp, selfTaskId, "default"); final AddressedTuple addressedTuple = new AddressedTuple(remoteTaskId, tuple); @Override @@ -158,7 +173,8 @@ public void run() { private StormTopology createStormTopology() { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout(sourceComp, new TestWordSpout(true), 1); - builder.setBolt(destComp, new TestWordCounter(), 1).fieldsGrouping(sourceComp, new Fields("word")); + builder.setBolt(destComp, new TestWordCounter(), 1).fieldsGrouping(sourceComp, + new Fields("word")); return builder.createTopology(); } @@ -175,7 +191,9 @@ public SingleThreadedConsumer(KryoTupleDeserializer deserializer, int numMessage /** * There are multiple producers sending messages to the queue simultaneously. * The consumer receives messages one by one and tries to deserialize them. - * If there is any issues/exceptions during the process, it basically means data corruption is happening. + * If there is any issues/exceptions during the process, it basically means data corruption + * is happening. + * * @param o the received object */ @Override @@ -196,7 +214,7 @@ public void finalCheck() { @Override public void flush() { - //no op + // no op } } -} \ No newline at end of file +} diff --git a/storm-client/test/jvm/org/apache/storm/executor/SpoutExecutorTest.java b/storm-client/test/jvm/org/apache/storm/executor/SpoutExecutorTest.java index b83009f8933..0eb67e5c2db 100644 --- a/storm-client/test/jvm/org/apache/storm/executor/SpoutExecutorTest.java +++ b/storm-client/test/jvm/org/apache/storm/executor/SpoutExecutorTest.java @@ -1,17 +1,31 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.executor; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyString; + +import java.lang.reflect.Field; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import org.apache.storm.Constants; import org.apache.storm.cluster.IStateStorage; import org.apache.storm.daemon.worker.WorkerState; @@ -29,27 +43,18 @@ import org.junit.platform.commons.util.ReflectionUtils; import org.mockito.Mockito; -import java.lang.reflect.Field; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.junit.jupiter.api.Assertions.assertInstanceOf; -import static org.mockito.ArgumentMatchers.anyInt; -import static org.mockito.ArgumentMatchers.anyString; - public class SpoutExecutorTest { - @Test public void testPendingTuplesRotateShouldBeCalledOnlyOnce() throws Exception { RateCounter rateCounter = Mockito.mock(RateCounter.class); StormMetricRegistry stormMetricRegistry = Mockito.mock(StormMetricRegistry.class); - Mockito.when(stormMetricRegistry.rateCounter(anyString(),anyString(),anyInt())).thenReturn(rateCounter); + Mockito.when(stormMetricRegistry.rateCounter(anyString(), anyString(), anyInt())) + .thenReturn(rateCounter); - Map hashmap = Utils.readDefaultConfig(); + Map hashmap = Utils.readDefaultConfig(); IStateStorage stateStorage = Mockito.mock(IStateStorage.class); @@ -58,7 +63,8 @@ public void testPendingTuplesRotateShouldBeCalledOnlyOnce() throws Exception { WorkerTopologyContext workerTopologyContext = Mockito.mock(WorkerTopologyContext.class); Mockito.when(workerTopologyContext.getComponentId(anyInt())).thenReturn("1"); - Mockito.when(workerTopologyContext.getComponentCommon(anyString())).thenReturn(componentCommon); + Mockito.when(workerTopologyContext.getComponentCommon(anyString())) + .thenReturn(componentCommon); WorkerState workerState = Mockito.mock(WorkerState.class); Mockito.when(workerState.getWorkerTopologyContext()).thenReturn(workerTopologyContext); @@ -66,7 +72,8 @@ public void testPendingTuplesRotateShouldBeCalledOnlyOnce() throws Exception { Mockito.when(workerState.getTopologyConf()).thenReturn(hashmap); Mockito.when(workerState.getMetricRegistry()).thenReturn(stormMetricRegistry); - SpoutExecutor spoutExecutor = new SpoutExecutor(workerState,List.of(1L,5L),new HashMap<>()); + SpoutExecutor spoutExecutor = new SpoutExecutor(workerState, List.of(1L, 5L), + new HashMap<>()); TupleImpl tuple = Mockito.mock(TupleImpl.class); Mockito.when(tuple.getSourceStreamId()).thenReturn(Constants.SYSTEM_TICK_STREAM_ID); @@ -84,7 +91,7 @@ public void testPendingTuplesRotateShouldBeCalledOnlyOnce() throws Exception { spoutExecutor.accept(addressedTuple); - Mockito.verify(rotatingMap,Mockito.times(1)).rotate(); + Mockito.verify(rotatingMap, Mockito.times(1)).rotate(); } @Test @@ -93,14 +100,16 @@ public void testTupleTreeIdsComeFromAnUnguessableGenerator() { RateCounter rateCounter = Mockito.mock(RateCounter.class); StormMetricRegistry stormMetricRegistry = Mockito.mock(StormMetricRegistry.class); - Mockito.when(stormMetricRegistry.rateCounter(anyString(),anyString(),anyInt())).thenReturn(rateCounter); + Mockito.when(stormMetricRegistry.rateCounter(anyString(), anyString(), anyInt())) + .thenReturn(rateCounter); ComponentCommon componentCommon = Mockito.mock(ComponentCommon.class); Mockito.when(componentCommon.get_json_conf()).thenReturn(null); WorkerTopologyContext workerTopologyContext = Mockito.mock(WorkerTopologyContext.class); Mockito.when(workerTopologyContext.getComponentId(anyInt())).thenReturn("1"); - Mockito.when(workerTopologyContext.getComponentCommon(anyString())).thenReturn(componentCommon); + Mockito.when(workerTopologyContext.getComponentCommon(anyString())) + .thenReturn(componentCommon); WorkerState workerState = Mockito.mock(WorkerState.class); Mockito.when(workerState.getWorkerTopologyContext()).thenReturn(workerTopologyContext); @@ -108,7 +117,8 @@ public void testTupleTreeIdsComeFromAnUnguessableGenerator() { Mockito.when(workerState.getTopologyConf()).thenReturn(Utils.readDefaultConfig()); Mockito.when(workerState.getMetricRegistry()).thenReturn(stormMetricRegistry); - SpoutExecutor spoutExecutor = new SpoutExecutor(workerState,List.of(1L,5L),new HashMap<>()); + SpoutExecutor spoutExecutor = new SpoutExecutor(workerState, List.of(1L, 5L), + new HashMap<>()); assertInstanceOf(KeyStreamRandom.class, spoutExecutor.rand); } diff --git a/storm-client/test/jvm/org/apache/storm/executor/error/ReportErrorTest.java b/storm-client/test/jvm/org/apache/storm/executor/error/ReportErrorTest.java index f52e6595500..b1757461f6b 100644 --- a/storm-client/test/jvm/org/apache/storm/executor/error/ReportErrorTest.java +++ b/storm-client/test/jvm/org/apache/storm/executor/error/ReportErrorTest.java @@ -1,35 +1,40 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.executor.error; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + import java.util.HashMap; import java.util.Map; import java.util.concurrent.atomic.AtomicLong; import org.apache.storm.Config; import org.apache.storm.cluster.IStormClusterState; import org.apache.storm.task.WorkerTopologyContext; -import org.apache.storm.utils.Time; import org.apache.storm.utils.Time.SimulatedTime; +import org.apache.storm.utils.Time; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.any; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - public class ReportErrorTest { @Test @@ -44,7 +49,8 @@ public void testReport() { IStormClusterState state = mock(IStormClusterState.class); doAnswer((invocation) -> errorCount.incrementAndGet()) - .when(state).reportError(eq(topo), eq(comp), anyString(), eq(port), any(Throwable.class)); + .when(state).reportError(eq(topo), eq(comp), anyString(), eq(port), + any(Throwable.class)); Map conf = new HashMap<>(); conf.put(Config.TOPOLOGY_ERROR_THROTTLE_INTERVAL_SECS, 10); conf.put(Config.TOPOLOGY_MAX_ERROR_REPORT_PER_INTERVAL, 4); @@ -59,7 +65,7 @@ public void testReport() { assertEquals(3, errorCount.get()); report.report(new RuntimeException("ERROR-4")); assertEquals(4, errorCount.get()); - //Too fast not reported + // Too fast not reported report.report(new RuntimeException("ERROR-5")); assertEquals(4, errorCount.get()); Time.advanceTime(9000); diff --git a/storm-client/test/jvm/org/apache/storm/grouping/JitterAwareStreamGroupingTest.java b/storm-client/test/jvm/org/apache/storm/grouping/JitterAwareStreamGroupingTest.java index 558ef0546f7..0d5c0f099e1 100644 --- a/storm-client/test/jvm/org/apache/storm/grouping/JitterAwareStreamGroupingTest.java +++ b/storm-client/test/jvm/org/apache/storm/grouping/JitterAwareStreamGroupingTest.java @@ -1,17 +1,28 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.grouping; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -26,17 +37,15 @@ import org.apache.storm.task.WorkerTopologyContext; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - /** - * Unit tests for {@link JitterAwareStreamGrouping}: power-of-two-choices steering toward the lower-jitter + * Unit tests for {@link JitterAwareStreamGrouping}: power-of-two-choices steering toward the + * lower-jitter * child once {@link ChildEwmaStats} is populated, and delegation to the embedded - * {@link LoadAwareShuffleGrouping} whenever jitter cannot pick a winner (no feedback, or a tied pair). + * {@link LoadAwareShuffleGrouping} whenever jitter cannot pick a winner (no feedback, or a tied + * pair). * - *

      With exactly two targets, P2C deterministically samples both, so the lower-jitter target always wins + *

      With exactly two targets, P2C deterministically samples both, so the lower-jitter target + * always wins * those tests without needing to control the random source. */ public class JitterAwareStreamGroupingTest { @@ -46,7 +55,8 @@ public class JitterAwareStreamGroupingTest { private Map createConf() { Map conf = new HashMap<>(); - conf.put(Config.STORM_NETWORK_TOPOGRAPHY_PLUGIN, "org.apache.storm.networktopography.DefaultRackDNSToSwitchMapping"); + conf.put(Config.STORM_NETWORK_TOPOGRAPHY_PLUGIN, + "org.apache.storm.networktopography.DefaultRackDNSToSwitchMapping"); conf.put(Config.TOPOLOGY_LOCALITYAWARE_HIGHER_BOUND, 0.8); conf.put(Config.TOPOLOGY_LOCALITYAWARE_LOWER_BOUND, 0.2); return conf; @@ -62,7 +72,8 @@ private WorkerTopologyContext mockContext(List availableTaskIds) { when(context.getTaskToNodePort()).thenReturn(new AtomicReference<>(taskNodeToPort)); when(context.getAssignmentId()).thenReturn("node-id"); when(context.getThisWorkerPort()).thenReturn(6700); - when(context.getNodeToHost()).thenReturn(new AtomicReference<>(Collections.singletonMap("node-id", "hostname1"))); + when(context.getNodeToHost()).thenReturn(new AtomicReference<>(Collections + .singletonMap("node-id", "hostname1"))); return context; } @@ -83,7 +94,8 @@ public void chooseTasks_emptyTargetsReturnsEmpty() { @Test public void chooseTasks_singleTargetAlwaysReturnsIt() { - JitterAwareStreamGrouping grouping = prepared(Collections.singletonList(42), new ChildEwmaStats(true)); + JitterAwareStreamGrouping grouping = prepared(Collections.singletonList(42), + new ChildEwmaStats(true)); for (int i = 0; i < 5; i++) { assertEquals(Collections.singletonList(42), grouping.chooseTasks(SOURCE_TASK, VALUES)); } @@ -119,7 +131,8 @@ public void chooseTasks_steersToLowerJitterChild() { @Test public void chooseTasks_prefersReportedOverUnreported() { - // Only 11 has reported; with two targets it is always in the pair and beats the unreported 10. + // Only 11 has reported; with two targets it is always in the pair and beats the unreported + // 10. List targets = Arrays.asList(10, 11); ChildEwmaStats stats = new ChildEwmaStats(true); stats.update(SOURCE_TASK, 11, new EwmaFeedbackRecord(7.0, 7.0, 7.0)); @@ -132,7 +145,8 @@ public void chooseTasks_prefersReportedOverUnreported() { @Test public void chooseTasks_tiedJitter_delegatesToFallback() { - // Equal jitter on both targets => every pair ties => the load-aware fallback decides and spreads. + // Equal jitter on both targets => every pair ties => the load-aware fallback decides and + // spreads. List targets = Arrays.asList(10, 11); ChildEwmaStats stats = new ChildEwmaStats(true); stats.update(SOURCE_TASK, 10, new EwmaFeedbackRecord(5.0, 5.0, 5.0)); @@ -143,7 +157,8 @@ public void chooseTasks_tiedJitter_delegatesToFallback() { @Test public void chooseTasks_isPerSourceTask() { - // Two targets, source 1 only: child 10 is best; the grouping must not consult another source's stats. + // Two targets, source 1 only: child 10 is best; the grouping must not consult another + // source's stats. List targets = Arrays.asList(10, 11); ChildEwmaStats stats = new ChildEwmaStats(true); stats.update(1, 10, new EwmaFeedbackRecord(1, 1, 1.0)); @@ -157,7 +172,8 @@ public void chooseTasks_isPerSourceTask() { @Test public void chooseTasks_p2cAvoidsHerd() { - // Three strictly-ordered targets, real randomness: the best (10) takes the plurality but NOT 100% + // Three strictly-ordered targets, real randomness: the best (10) takes the plurality but + // NOT 100% // (herd avoided), the worst (12) never wins a pair, and 11 takes the remainder. List targets = Arrays.asList(10, 11, 12); ChildEwmaStats stats = new ChildEwmaStats(true); @@ -176,10 +192,13 @@ public void chooseTasks_p2cAvoidsHerd() { int best = hits.getOrDefault(10, 0); int mid = hits.getOrDefault(11, 0); int worst = hits.getOrDefault(12, 0); - assertTrue(best > 0 && best < total, "best target should take a share but not the whole herd: " + best); - assertTrue(best > mid, "best target should outweigh the middle one: " + best + " vs " + mid); + assertTrue(best > 0 && best < total, + "best target should take a share but not the whole herd: " + best); + assertTrue(best > mid, "best target should outweigh the middle one: " + best + " vs " + + mid); assertTrue(mid > 0, "middle target should still receive traffic: " + mid); - // The worst target is never the lower-jitter of any sampled pair, and there are no ties to delegate. + // The worst target is never the lower-jitter of any sampled pair, and there are no ties to + // delegate. assertEquals(0, worst, "worst target should never win a P2C comparison"); } @@ -193,7 +212,8 @@ private void assertDelegatedSpread(JitterAwareStreamGrouping grouping, List 0, "target " + target + " should receive some traffic"); + assertTrue(hits.getOrDefault(target, 0) > 0, "target " + target + + " should receive some traffic"); } } } diff --git a/storm-client/test/jvm/org/apache/storm/grouping/LoadAwareShuffleGroupingTest.java b/storm-client/test/jvm/org/apache/storm/grouping/LoadAwareShuffleGroupingTest.java index 00755f688b1..ffe864c65b5 100644 --- a/storm-client/test/jvm/org/apache/storm/grouping/LoadAwareShuffleGroupingTest.java +++ b/storm-client/test/jvm/org/apache/storm/grouping/LoadAwareShuffleGroupingTest.java @@ -1,17 +1,29 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.grouping; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -41,19 +53,14 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - public class LoadAwareShuffleGroupingTest { public static final double ACCEPTABLE_MARGIN = 0.015; private static final Logger LOG = LoggerFactory.getLogger(LoadAwareShuffleGroupingTest.class); private Map createConf() { Map conf = new HashMap<>(); - conf.put(Config.STORM_NETWORK_TOPOGRAPHY_PLUGIN, "org.apache.storm.networktopography.DefaultRackDNSToSwitchMapping"); + conf.put(Config.STORM_NETWORK_TOPOGRAPHY_PLUGIN, + "org.apache.storm.networktopography.DefaultRackDNSToSwitchMapping"); conf.put(Config.TOPOLOGY_LOCALITYAWARE_HIGHER_BOUND, 0.8); conf.put(Config.TOPOLOGY_LOCALITYAWARE_LOWER_BOUND, 0.2); return conf; @@ -68,7 +75,8 @@ private WorkerTopologyContext mockContext(List availableTaskIds) { when(context.getTaskToNodePort()).thenReturn(new AtomicReference<>(taskNodeToPort)); when(context.getAssignmentId()).thenReturn("node-id"); when(context.getThisWorkerPort()).thenReturn(6700); - AtomicReference> nodeToHost = new AtomicReference<>(Collections.singletonMap("node-id", "hostname1")); + AtomicReference> nodeToHost = new AtomicReference<>(Collections + .singletonMap("node-id", "hostname1")); when(context.getNodeToHost()).thenReturn(nodeToHost); return context; } @@ -86,14 +94,16 @@ public void testUnevenLoadOverTime() { localLoad.put(2, 0.0); LoadMapping lm = new LoadMapping(); lm.setLocal(localLoad); - //First verify that if something has a high load it's distribution will drop over time + // First verify that if something has a high load it's distribution will drop over time for (int i = 9; i >= 0; i--) { grouping.refreshLoad(lm); expectedOneWeight -= 10.0; Map countByType = count(grouping.choices, grouping.rets); LOG.info("contByType = {}", countByType); - double expectedOnePercentage = expectedOneWeight / (expectedOneWeight + expectedTwoWeight); - double expectedTwoPercentage = expectedTwoWeight / (expectedOneWeight + expectedTwoWeight); + double expectedOnePercentage = expectedOneWeight / (expectedOneWeight + + expectedTwoWeight); + double expectedTwoPercentage = expectedTwoWeight / (expectedOneWeight + + expectedTwoWeight); assertEquals(expectedOnePercentage, countByType.getOrDefault(1, 0.0) / grouping.getCapacity(), 0.01, "i = " + i); @@ -102,7 +112,7 @@ public void testUnevenLoadOverTime() { 0.01, "i = " + i); } - //Now verify that when it is switched we can recover + // Now verify that when it is switched we can recover localLoad.put(1, 0.0); localLoad.put(2, 1.0); lm.setLocal(localLoad); @@ -113,11 +123,15 @@ public void testUnevenLoadOverTime() { expectedTwoWeight = Math.max(0.0, expectedTwoWeight - 10.0); Map countByType = count(grouping.choices, grouping.rets); LOG.info("contByType = {}", countByType); - double expectedOnePercentage = expectedOneWeight / (expectedOneWeight + expectedTwoWeight); - double expectedTwoPercentage = expectedTwoWeight / (expectedOneWeight + expectedTwoWeight); - assertEquals(expectedOnePercentage, countByType.getOrDefault(1, 0.0) / grouping.getCapacity(), + double expectedOnePercentage = expectedOneWeight / (expectedOneWeight + + expectedTwoWeight); + double expectedTwoPercentage = expectedTwoWeight / (expectedOneWeight + + expectedTwoWeight); + assertEquals(expectedOnePercentage, countByType.getOrDefault(1, 0.0) / grouping + .getCapacity(), 0.01); - assertEquals(expectedTwoPercentage, countByType.getOrDefault(2, 0.0) / grouping.getCapacity(), + assertEquals(expectedTwoPercentage, countByType.getOrDefault(2, 0.0) / grouping + .getCapacity(), 0.01); } } @@ -158,7 +172,8 @@ private void testLoadAwareShuffleGroupingWithEvenLoad(int numTasks) { int minPrCount = (int) (totalEmits * ((1.0 / numTasks) - ACCEPTABLE_MARGIN)); int maxPrCount = (int) (totalEmits * ((1.0 / numTasks) + ACCEPTABLE_MARGIN)); - int[] taskCounts = runChooseTasksWithVerification(grouper, totalEmits, numTasks, loadMapping); + int[] taskCounts = runChooseTasksWithVerification(grouper, totalEmits, numTasks, + loadMapping); for (int i = 0; i < numTasks; i++) { assertTrue(taskCounts[i] >= minPrCount && taskCounts[i] <= maxPrCount, @@ -215,7 +230,8 @@ private void testLoadAwareShuffleGroupingWithEvenLoadMultiThreaded(int numTasks) int taskId = taskIds.get(0); - assertTrue(taskId >= 0 && taskId < availableTaskIds.size(), "TaskId should exist"); + assertTrue(taskId >= 0 && taskId < availableTaskIds.size(), + "TaskId should exist"); taskCounts[taskId]++; } return taskCounts; @@ -253,7 +269,8 @@ private void testLoadAwareShuffleGroupingWithEvenLoadMultiThreaded(int numTasks) public void testShuffleLoadEven() { // port test-shuffle-load-even LoadAwareCustomStreamGrouping shuffler = GrouperFactory - .mkGrouper(mockContext(Lists.newArrayList(1, 2)), "comp", "stream", null, Grouping.shuffle(new NullStruct()), + .mkGrouper(mockContext(Lists.newArrayList(1, 2)), "comp", "stream", null, Grouping + .shuffle(new NullStruct()), Lists.newArrayList(1, 2), Collections.emptyMap()); int numMessages = 100000; int minPrCount = (int) (numMessages * (0.5 - ACCEPTABLE_MARGIN)); @@ -393,7 +410,8 @@ private void runSimpleBenchmark(LoadAwareCustomStreamGrouping grouper, // periodically calls refreshLoad in 1 sec to simulate worker load update timer ScheduledExecutorService refreshService = MoreExecutors.getExitingScheduledExecutorService( new ScheduledThreadPoolExecutor(1)); - refreshService.scheduleAtFixedRate(() -> grouper.refreshLoad(loadMapping), 1, 1, TimeUnit.SECONDS); + refreshService.scheduleAtFixedRate(() -> grouper.refreshLoad(loadMapping), 1, 1, + TimeUnit.SECONDS); long current = System.currentTimeMillis(); int idx = 0; @@ -433,7 +451,8 @@ private void runMultithreadedBenchmark(LoadAwareCustomStreamGrouping grouper, // periodically calls refreshLoad in 1 sec to simulate worker load update timer ScheduledExecutorService refreshService = MoreExecutors.getExitingScheduledExecutorService( new ScheduledThreadPoolExecutor(1)); - refreshService.scheduleAtFixedRate(() -> grouper.refreshLoad(loadMapping), 1, 1, TimeUnit.SECONDS); + refreshService.scheduleAtFixedRate(() -> grouper.refreshLoad(loadMapping), 1, 1, + TimeUnit.SECONDS); long current = System.currentTimeMillis(); int idx = 0; @@ -491,7 +510,8 @@ public void testLoadSwitching() { WorkerTopologyContext context = createLoadSwitchingContext(); grouping.prepare(context, new GlobalStreamId("a", "default"), Arrays.asList(1, 2, 3)); // startup should default to worker local - assertEquals(LoadAwareShuffleGrouping.LocalityScope.WORKER_LOCAL, grouping.getCurrentScope()); + assertEquals(LoadAwareShuffleGrouping.LocalityScope.WORKER_LOCAL, grouping + .getCurrentScope()); // with high load, switch to host local LoadMapping lm = createLoadMapping(1.0, 1.0, 1.0); @@ -523,7 +543,8 @@ public void testLoadSwitching() { // reduce load on local worker task, should switch to worker local lm = createLoadMapping(0.1, 0.1, 0.1); grouping.refreshLoad(lm); - assertEquals(LoadAwareShuffleGrouping.LocalityScope.WORKER_LOCAL, grouping.getCurrentScope()); + assertEquals(LoadAwareShuffleGrouping.LocalityScope.WORKER_LOCAL, grouping + .getCurrentScope()); } private LoadMapping createLoadMapping(double load1, double load2, double load3) { @@ -561,4 +582,4 @@ private WorkerTopologyContext createLoadSwitchingContext() { return context; } -} \ No newline at end of file +} diff --git a/storm-client/test/jvm/org/apache/storm/grouping/ShuffleGroupingTest.java b/storm-client/test/jvm/org/apache/storm/grouping/ShuffleGroupingTest.java index 5c295331e78..f21557d85c3 100644 --- a/storm-client/test/jvm/org/apache/storm/grouping/ShuffleGroupingTest.java +++ b/storm-client/test/jvm/org/apache/storm/grouping/ShuffleGroupingTest.java @@ -1,17 +1,27 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.grouping; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; + import java.util.List; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; @@ -22,11 +32,6 @@ import org.apache.storm.task.WorkerTopologyContext; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.mock; - public class ShuffleGroupingTest { /** @@ -70,7 +75,8 @@ public void testShuffleGrouping() { } /** - * Tests that we round robbin correctly with multiple threads using ShuffleGrouping implementation. + * Tests that we round robbin correctly with multiple threads using ShuffleGrouping + * implementation. */ @Test public void testShuffleGroupMultiThreaded() throws InterruptedException, ExecutionException { @@ -106,7 +112,8 @@ public void testShuffleGroupMultiThreaded() throws InterruptedException, Executi int taskId = taskIds.get(0); - assertTrue(taskId >= 0 && taskId < availableTaskIds.size(), "TaskId should exist"); + assertTrue(taskId >= 0 && taskId < availableTaskIds.size(), + "TaskId should exist"); taskCounts[taskId]++; } return taskCounts; diff --git a/storm-client/test/jvm/org/apache/storm/grouping/partialKeyGrouping/BalancedTargetSelectorTest.java b/storm-client/test/jvm/org/apache/storm/grouping/partialKeyGrouping/BalancedTargetSelectorTest.java index d0cc0df2dc6..47ff9a8987c 100644 --- a/storm-client/test/jvm/org/apache/storm/grouping/partialKeyGrouping/BalancedTargetSelectorTest.java +++ b/storm-client/test/jvm/org/apache/storm/grouping/partialKeyGrouping/BalancedTargetSelectorTest.java @@ -1,17 +1,25 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.grouping.partialKeyGrouping; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; + import java.util.Arrays; import java.util.stream.Collectors; import org.apache.storm.grouping.PartialKeyGrouping; @@ -19,15 +27,12 @@ import org.hamcrest.Matchers; import org.junit.jupiter.api.Test; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.equalTo; - public class BalancedTargetSelectorTest { - private static final int[] TASK_LIST = { 9, 8, 7, 6 }; - private final PartialKeyGrouping.TargetSelector targetSelector = new PartialKeyGrouping.BalancedTargetSelector(); + private final PartialKeyGrouping.TargetSelector targetSelector = new PartialKeyGrouping + .BalancedTargetSelector(); @Test public void classIsSerializable() { @@ -39,7 +44,8 @@ public void selectorReturnsTasksInAssignment() { // select tasks once more than the number of tasks available for (int i = 0; i < TASK_LIST.length + 1; i++) { int selectedTask = targetSelector.chooseTask(TASK_LIST); - assertThat(selectedTask, Matchers.in(Arrays.stream(TASK_LIST).boxed().collect(Collectors.toList()))); + assertThat(selectedTask, Matchers.in(Arrays.stream(TASK_LIST).boxed().collect(Collectors + .toList()))); } } diff --git a/storm-client/test/jvm/org/apache/storm/grouping/partialKeyGrouping/PartialKeyGroupingTest.java b/storm-client/test/jvm/org/apache/storm/grouping/partialKeyGrouping/PartialKeyGroupingTest.java index 0811985c778..d7e5dab11a8 100644 --- a/storm-client/test/jvm/org/apache/storm/grouping/partialKeyGrouping/PartialKeyGroupingTest.java +++ b/storm-client/test/jvm/org/apache/storm/grouping/partialKeyGrouping/PartialKeyGroupingTest.java @@ -1,17 +1,30 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.grouping.partialKeyGrouping; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + import java.util.List; import org.apache.storm.generated.GlobalStreamId; import org.apache.storm.grouping.PartialKeyGrouping; @@ -22,13 +35,6 @@ import org.apache.storm.utils.Utils; import org.junit.jupiter.api.Test; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.CoreMatchers.not; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.mockito.Mockito.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - public class PartialKeyGroupingTest { @Test @@ -69,7 +75,8 @@ public void testChooseTasksWithoutConsecutiveTaskIds() { public void testChooseTasksFields() { PartialKeyGrouping pkg = new PartialKeyGrouping(new Fields("test")); WorkerTopologyContext context = mock(WorkerTopologyContext.class); - when(context.getComponentOutputFields(any(GlobalStreamId.class))).thenReturn(new Fields("test")); + when(context.getComponentOutputFields(any(GlobalStreamId.class))) + .thenReturn(new Fields("test")); pkg.prepare(context, mock(GlobalStreamId.class), Lists.newArrayList(0, 1, 2, 3, 4, 5)); Values message = new Values("key1"); List choice1 = pkg.chooseTasks(0, message); diff --git a/storm-client/test/jvm/org/apache/storm/grouping/partialKeyGrouping/RandomTwoTaskAssignmentCreatorTest.java b/storm-client/test/jvm/org/apache/storm/grouping/partialKeyGrouping/RandomTwoTaskAssignmentCreatorTest.java index fc556f3599b..26604548c14 100644 --- a/storm-client/test/jvm/org/apache/storm/grouping/partialKeyGrouping/RandomTwoTaskAssignmentCreatorTest.java +++ b/storm-client/test/jvm/org/apache/storm/grouping/partialKeyGrouping/RandomTwoTaskAssignmentCreatorTest.java @@ -1,26 +1,31 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.grouping.partialKeyGrouping; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.not; + import org.apache.storm.grouping.PartialKeyGrouping; import org.apache.storm.shade.com.google.common.collect.Lists; import org.apache.storm.utils.Utils; import org.junit.jupiter.api.Test; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.not; - public class RandomTwoTaskAssignmentCreatorTest { private static final byte[] GROUPING_KEY_ONE = "some_key_one".getBytes(); @@ -28,30 +33,39 @@ public class RandomTwoTaskAssignmentCreatorTest { @Test public void classIsSerializable() throws Exception { - PartialKeyGrouping.AssignmentCreator assignmentCreator = new PartialKeyGrouping.RandomTwoTaskAssignmentCreator(); + PartialKeyGrouping.AssignmentCreator assignmentCreator = new PartialKeyGrouping + .RandomTwoTaskAssignmentCreator(); Utils.javaSerialize(assignmentCreator); } @Test public void returnsAssignmentOfExpectedSize() { - PartialKeyGrouping.AssignmentCreator assignmentCreator = new PartialKeyGrouping.RandomTwoTaskAssignmentCreator(); - int[] assignedTasks = assignmentCreator.createAssignment(Lists.newArrayList(9, 8, 7, 6), GROUPING_KEY_ONE); + PartialKeyGrouping.AssignmentCreator assignmentCreator = new PartialKeyGrouping + .RandomTwoTaskAssignmentCreator(); + int[] assignedTasks = assignmentCreator.createAssignment(Lists.newArrayList(9, 8, 7, 6), + GROUPING_KEY_ONE); assertThat(assignedTasks.length, equalTo(2)); } @Test public void returnsDifferentAssignmentForDifferentKeys() { - PartialKeyGrouping.AssignmentCreator assignmentCreator = new PartialKeyGrouping.RandomTwoTaskAssignmentCreator(); - int[] assignmentOne = assignmentCreator.createAssignment(Lists.newArrayList(9, 8, 7, 6), GROUPING_KEY_ONE); - int[] assignmentTwo = assignmentCreator.createAssignment(Lists.newArrayList(9, 8, 7, 6), GROUPING_KEY_TWO); + PartialKeyGrouping.AssignmentCreator assignmentCreator = new PartialKeyGrouping + .RandomTwoTaskAssignmentCreator(); + int[] assignmentOne = assignmentCreator.createAssignment(Lists.newArrayList(9, 8, 7, 6), + GROUPING_KEY_ONE); + int[] assignmentTwo = assignmentCreator.createAssignment(Lists.newArrayList(9, 8, 7, 6), + GROUPING_KEY_TWO); assertThat(assignmentOne, not(equalTo(assignmentTwo))); } @Test public void returnsSameAssignmentForSameKey() { - PartialKeyGrouping.AssignmentCreator assignmentCreator = new PartialKeyGrouping.RandomTwoTaskAssignmentCreator(); - int[] assignmentOne = assignmentCreator.createAssignment(Lists.newArrayList(9, 8, 7, 6), GROUPING_KEY_ONE); - int[] assignmentOneAgain = assignmentCreator.createAssignment(Lists.newArrayList(9, 8, 7, 6), GROUPING_KEY_ONE); + PartialKeyGrouping.AssignmentCreator assignmentCreator = new PartialKeyGrouping + .RandomTwoTaskAssignmentCreator(); + int[] assignmentOne = assignmentCreator.createAssignment(Lists.newArrayList(9, 8, 7, 6), + GROUPING_KEY_ONE); + int[] assignmentOneAgain = assignmentCreator.createAssignment(Lists.newArrayList(9, 8, 7, + 6), GROUPING_KEY_ONE); assertThat(assignmentOne, equalTo(assignmentOneAgain)); } } diff --git a/storm-client/test/jvm/org/apache/storm/messaging/DeserializingConnectionCallbackTest.java b/storm-client/test/jvm/org/apache/storm/messaging/DeserializingConnectionCallbackTest.java index 2a982b3a3b6..111b2309919 100644 --- a/storm-client/test/jvm/org/apache/storm/messaging/DeserializingConnectionCallbackTest.java +++ b/storm-client/test/jvm/org/apache/storm/messaging/DeserializingConnectionCallbackTest.java @@ -1,17 +1,33 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.messaging; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + import com.esotericsoftware.kryo.KryoException; import com.esotericsoftware.kryo.io.Output; import java.io.IOException; @@ -41,16 +57,6 @@ import org.junit.jupiter.api.Test; import org.mockito.ArgumentCaptor; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - public class DeserializingConnectionCallbackTest { private static final byte[] messageBytes = new byte[3]; private static TaskMessage message; @@ -72,13 +78,13 @@ public void setUp() throws Exception { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout(SOURCE_COMPONENT, new TestWordSpout(true), 1); - builder.setBolt(DEST_COMPONENT, new TestWordCounter(), 1).fieldsGrouping(SOURCE_COMPONENT, new Fields("word")); + builder.setBolt(DEST_COMPONENT, new TestWordCounter(), 1).fieldsGrouping(SOURCE_COMPONENT, + new Fields("word")); context = mock(GeneralTopologyContext.class); when(context.getRawTopology()).thenReturn(builder.createTopology()); when(context.getComponentId(SOURCE_TASK_ID)).thenReturn(SOURCE_COMPONENT); } - @Test public void testUpdateMetricsConfigOff() { Map config = new HashMap<>(); @@ -106,7 +112,7 @@ public void testUpdateMetricsConfigOn() { // Starting empty Object metrics = withMetrics.getValueAndReset(); assertTrue(metrics instanceof Map); - assertTrue(((Map) metrics).isEmpty()); + assertTrue(((Map) metrics).isEmpty()); // Add messages withMetrics.updateMetrics(123, message); @@ -121,10 +127,12 @@ public void testUpdateMetricsConfigOn() { @Test public void testTruncatedKryoPayloadDroppedAndBatchContinues() { Map conf = baseConf(); - byte[] full = serializedTuple(conf, new Values("a-string-long-enough-to-survive-truncation", 7)); + byte[] full = serializedTuple(conf, new Values("a-string-long-enough-to-survive-truncation", + 7)); byte[] truncated = Arrays.copyOf(full, full.length - 10); - assertThrows(KryoException.class, () -> new KryoTupleDeserializer(conf, context).deserialize(truncated)); + assertThrows(KryoException.class, () -> new KryoTupleDeserializer(conf, context) + .deserialize(truncated)); assertBatchDeliversOnlyValidMessages(conf, truncated); } @@ -137,7 +145,8 @@ public void testUnknownSourceTaskDroppedAndBatchContinues() { out.writeInt(1, true); // default stream id byte[] unknownTask = out.toBytes(); - assertThrows(IllegalArgumentException.class, () -> new KryoTupleDeserializer(conf, context).deserialize(unknownTask)); + assertThrows(IllegalArgumentException.class, () -> new KryoTupleDeserializer(conf, context) + .deserialize(unknownTask)); assertBatchDeliversOnlyValidMessages(conf, unknownTask); } @@ -150,7 +159,8 @@ public void testJavaFallbackMissingClassDroppedAndBatchContinues() { byte[] missingClass = replaceAll(bytes, "JavaSerializedValue", "JavaSerializedValuf"); RuntimeException thrown = assertThrows(RuntimeException.class, - () -> new KryoTupleDeserializer(conf, context).deserialize(missingClass)); + () -> new KryoTupleDeserializer(conf, context) + .deserialize(missingClass)); assertTrue(Utils.exceptionCauseIsInstanceOf(ClassNotFoundException.class, thrown), "expected a ClassNotFoundException in the cause chain but was: " + thrown); @@ -163,15 +173,18 @@ public void testJavaFallbackNegativeLengthDroppedAndBatchContinues() { conf.put(Config.TOPOLOGY_FALL_BACK_ON_JAVA_SERIALIZATION, true); byte[] bytes = serializedTuple(conf, Collections.singletonList(new JavaSerializedValue())); - // SerializableSerializer writes the java-serialization byte count right before the stream header; - // an all-bits-set count reads back as a negative length, which the serializer rejects before allocation. + // SerializableSerializer writes the java-serialization byte count right before the stream + // header; + // an all-bits-set count reads back as a negative length, which the serializer rejects + // before allocation. int headerIdx = indexOf(bytes, JAVA_STREAM_HEADER, 0); assertTrue(headerIdx >= 4, "java serialization header not found in tuple payload"); for (int i = 1; i <= 4; i++) { bytes[headerIdx - i] = (byte) 0xFF; } - assertThrows(KryoException.class, () -> new KryoTupleDeserializer(conf, context).deserialize(bytes)); + assertThrows(KryoException.class, () -> new KryoTupleDeserializer(conf, context) + .deserialize(bytes)); assertBatchDeliversOnlyValidMessages(conf, bytes); } @@ -180,10 +193,12 @@ public void testJavaFallbackNegativeLengthDroppedAndBatchContinues() { public void testIoExceptionFailureDroppedAndBatchContinues() { Map conf = baseConf(); conf.put(Config.TOPOLOGY_TUPLE_COMPRESSION_ENABLE, true); - byte[] fakeZstd = {(byte) 0x28, (byte) 0xB5, (byte) 0x2F, (byte) 0xFD, 0x00, 0x01, 0x02, 0x03}; + byte[] fakeZstd = + {(byte) 0x28, (byte) 0xB5, (byte) 0x2F, (byte) 0xFD, 0x00, 0x01, 0x02, 0x03}; RuntimeException thrown = assertThrows(RuntimeException.class, - () -> new KryoTupleDeserializer(conf, context).deserialize(fakeZstd.clone())); + () -> new KryoTupleDeserializer(conf, context) + .deserialize(fakeZstd.clone())); assertTrue(Utils.exceptionCauseIsInstanceOf(IOException.class, thrown), "expected an IOException in the cause chain but was: " + thrown); @@ -194,8 +209,10 @@ public void testIoExceptionFailureDroppedAndBatchContinues() { public void testFailuresCountedSeparatelyFromSizeMetrics() { Map conf = baseConf(); conf.put(Config.TOPOLOGY_SERIALIZED_MESSAGE_SIZE_METRICS, Boolean.TRUE); - WorkerState.ILocalTransferCallback transfer = mock(WorkerState.ILocalTransferCallback.class); - DeserializingConnectionCallback callback = new DeserializingConnectionCallback(conf, context, transfer); + WorkerState.ILocalTransferCallback transfer = + mock(WorkerState.ILocalTransferCallback.class); + DeserializingConnectionCallback callback = new DeserializingConnectionCallback(conf, + context, transfer); callback.recv(Arrays.asList( taskMessage(serializedTuple(conf, new Values("nathan", 1))), @@ -212,10 +229,13 @@ public void testFailuresCountedSeparatelyFromSizeMetrics() { @Test public void testNonToleratedExceptionPropagates() throws Exception { - WorkerState.ILocalTransferCallback transfer = mock(WorkerState.ILocalTransferCallback.class); - DeserializingConnectionCallback callback = new DeserializingConnectionCallback(baseConf(), context, transfer); + WorkerState.ILocalTransferCallback transfer = + mock(WorkerState.ILocalTransferCallback.class); + DeserializingConnectionCallback callback = new DeserializingConnectionCallback(baseConf(), + context, transfer); KryoTupleDeserializer failing = mock(KryoTupleDeserializer.class); - when(failing.deserialize(any(byte[].class))).thenThrow(new IllegalStateException("injected")); + when(failing.deserialize(any(byte[].class))) + .thenThrow(new IllegalStateException("injected")); callback.setDeserializer(failing); assertThrows(IllegalStateException.class, @@ -228,10 +248,13 @@ public void testNonToleratedExceptionPropagates() throws Exception { @Test public void testPostDecodeFailurePropagatesAndIsNotCounted() { - WorkerState.ILocalTransferCallback transfer = mock(WorkerState.ILocalTransferCallback.class); + WorkerState.ILocalTransferCallback transfer = + mock(WorkerState.ILocalTransferCallback.class); // IllegalArgumentException is a tolerated deserialization-failure type; throwing it from - // updateMetrics, which runs after a successful decode, proves the try scope covers decoding only. - DeserializingConnectionCallback callback = new DeserializingConnectionCallback(baseConf(), context, transfer) { + // updateMetrics, which runs after a successful decode, proves the try scope covers decoding + // only. + DeserializingConnectionCallback callback = new DeserializingConnectionCallback(baseConf(), + context, transfer) { @Override protected void updateMetrics(int sourceTaskId, TaskMessage message) { throw new IllegalArgumentException("injected after decode"); @@ -247,8 +270,10 @@ protected void updateMetrics(int sourceTaskId, TaskMessage message) { } private void assertBatchDeliversOnlyValidMessages(Map conf, byte[] badPayload) { - WorkerState.ILocalTransferCallback transfer = mock(WorkerState.ILocalTransferCallback.class); - DeserializingConnectionCallback callback = new DeserializingConnectionCallback(conf, context, transfer); + WorkerState.ILocalTransferCallback transfer = + mock(WorkerState.ILocalTransferCallback.class); + DeserializingConnectionCallback callback = new DeserializingConnectionCallback(conf, + context, transfer); callback.recv(Arrays.asList( taskMessage(serializedTuple(conf, new Values("nathan", 1))), diff --git a/storm-client/test/jvm/org/apache/storm/messaging/netty/MessageDecoderTest.java b/storm-client/test/jvm/org/apache/storm/messaging/netty/MessageDecoderTest.java index 96915882302..8621f1a561e 100644 --- a/storm-client/test/jvm/org/apache/storm/messaging/netty/MessageDecoderTest.java +++ b/storm-client/test/jvm/org/apache/storm/messaging/netty/MessageDecoderTest.java @@ -1,25 +1,23 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.messaging.netty; -import java.util.List; -import org.apache.storm.messaging.TaskMessage; -import org.apache.storm.serialization.KryoValuesDeserializer; -import org.apache.storm.shade.io.netty.buffer.ByteBuf; -import org.apache.storm.shade.io.netty.buffer.Unpooled; -import org.apache.storm.shade.io.netty.channel.embedded.EmbeddedChannel; -import org.junit.jupiter.api.Test; - import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -31,6 +29,14 @@ import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; +import java.util.List; +import org.apache.storm.messaging.TaskMessage; +import org.apache.storm.serialization.KryoValuesDeserializer; +import org.apache.storm.shade.io.netty.buffer.ByteBuf; +import org.apache.storm.shade.io.netty.buffer.Unpooled; +import org.apache.storm.shade.io.netty.channel.embedded.EmbeddedChannel; +import org.junit.jupiter.api.Test; + public class MessageDecoderTest { private static final short TASK_ID = 1; diff --git a/storm-client/test/jvm/org/apache/storm/messaging/netty/ServerTest.java b/storm-client/test/jvm/org/apache/storm/messaging/netty/ServerTest.java index f1f7ca6fe68..cd2a9db4df7 100644 --- a/storm-client/test/jvm/org/apache/storm/messaging/netty/ServerTest.java +++ b/storm-client/test/jvm/org/apache/storm/messaging/netty/ServerTest.java @@ -1,27 +1,33 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software is distributed on an "AS + * IS" + * BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.messaging.netty; -import java.util.Map; -import org.apache.storm.messaging.DeserializingConnectionCallback; -import org.apache.storm.utils.Utils; -import org.junit.jupiter.api.Test; - import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import java.util.Map; +import org.apache.storm.messaging.DeserializingConnectionCallback; +import org.apache.storm.utils.Utils; +import org.junit.jupiter.api.Test; + public class ServerTest { @Test diff --git a/storm-client/test/jvm/org/apache/storm/messaging/netty/StormClientPipelineFactoryTest.java b/storm-client/test/jvm/org/apache/storm/messaging/netty/StormClientPipelineFactoryTest.java index c3889ca18df..50b13dc24d8 100644 --- a/storm-client/test/jvm/org/apache/storm/messaging/netty/StormClientPipelineFactoryTest.java +++ b/storm-client/test/jvm/org/apache/storm/messaging/netty/StormClientPipelineFactoryTest.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -35,10 +41,12 @@ private SSLEngine initSslEngine(Map conf) throws Exception { return initSslEngine(conf, null); } - private SSLEngine initSslEngine(Map conf, SslProvider sslProvider) throws Exception { + private SSLEngine initSslEngine(Map conf, + SslProvider sslProvider) throws Exception { SslContext sslContext = SslContextBuilder.forClient().sslProvider(sslProvider).build(); StormClientPipelineFactory factory = - new StormClientPipelineFactory(null, new AtomicBoolean[]{ new AtomicBoolean(false) }, conf, sslContext, + new StormClientPipelineFactory(null, new AtomicBoolean[]{ new AtomicBoolean(false) }, + conf, sslContext, DST_HOST, DST_PORT); EmbeddedChannel channel = new EmbeddedChannel(factory); try { diff --git a/storm-client/test/jvm/org/apache/storm/metric/FileBasedEventLoggerTest.java b/storm-client/test/jvm/org/apache/storm/metric/FileBasedEventLoggerTest.java index 1db39b4ac5b..6e021f6248d 100644 --- a/storm-client/test/jvm/org/apache/storm/metric/FileBasedEventLoggerTest.java +++ b/storm-client/test/jvm/org/apache/storm/metric/FileBasedEventLoggerTest.java @@ -18,6 +18,10 @@ package org.apache.storm.metric; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + import java.io.File; import java.io.IOException; import java.nio.file.Files; @@ -31,9 +35,6 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; public class FileBasedEventLoggerTest { diff --git a/storm-client/test/jvm/org/apache/storm/metric/filter/FilterByMetricNameTest.java b/storm-client/test/jvm/org/apache/storm/metric/filter/FilterByMetricNameTest.java index 888422049dc..7ae2056952e 100644 --- a/storm-client/test/jvm/org/apache/storm/metric/filter/FilterByMetricNameTest.java +++ b/storm-client/test/jvm/org/apache/storm/metric/filter/FilterByMetricNameTest.java @@ -1,17 +1,26 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.metric.filter; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + import java.util.List; import java.util.Map; import org.apache.storm.metric.api.IMetricsConsumer; @@ -20,9 +29,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; - public class FilterByMetricNameTest { @BeforeEach public void setUp() throws Exception { @@ -62,7 +68,8 @@ public void testBlacklist() { @Test public void testBothWhitelistAndBlacklistAreSpecified() { assertThrows(IllegalArgumentException.class, () -> { - List whitelistPattern = Lists.newArrayList("^metric\\.", "test\\.hello\\.[0-9]+"); + List whitelistPattern = Lists.newArrayList("^metric\\.", + "test\\.hello\\.[0-9]+"); List blacklistPattern = Lists.newArrayList("^__", "test\\."); new FilterByMetricName(whitelistPattern, blacklistPattern); }); @@ -82,9 +89,11 @@ public void testNoneIsSpecified() { assertTests(sut, testMetricNamesAndExpected); } - private void assertTests(FilterByMetricName sut, Map testMetricNamesAndExpected) { + private void assertTests(FilterByMetricName sut, Map testMetricNamesAndExpected) { for (Map.Entry testEntry : testMetricNamesAndExpected.entrySet()) { - assertEquals(testEntry.getValue(), sut.apply(new IMetricsConsumer.DataPoint(testEntry.getKey(), 1)), + assertEquals(testEntry.getValue(), sut.apply(new IMetricsConsumer.DataPoint(testEntry + .getKey(), 1)), "actual filter result is not same: " + testEntry.getKey()); } } diff --git a/storm-client/test/jvm/org/apache/storm/metric/internal/CountStatTest.java b/storm-client/test/jvm/org/apache/storm/metric/internal/CountStatTest.java index d5fc82e692c..3fb39919dcb 100644 --- a/storm-client/test/jvm/org/apache/storm/metric/internal/CountStatTest.java +++ b/storm-client/test/jvm/org/apache/storm/metric/internal/CountStatTest.java @@ -1,25 +1,30 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

      * http://www.apache.org/licenses/LICENSE-2.0 *

      - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.metric.internal; +import static org.junit.jupiter.api.Assertions.assertEquals; + import java.util.HashMap; import java.util.Map; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; - /** - * Unit test for CountStat + * Unit test for CountStat. */ public class CountStatTest { final long TEN_MIN = 10 * 60 * 1000; @@ -32,7 +37,7 @@ public void testBasic() { long time = 0L; CountStat count = new CountStat(10, time); while (time < TEN_MIN) { - //For this part of the test we interleave the different rotation types. + // For this part of the test we interleave the different rotation types. count.incBy(50); time += THIRTY_SEC / 2; count.rotateSched(time); diff --git a/storm-client/test/jvm/org/apache/storm/metric/internal/LatencyStatTest.java b/storm-client/test/jvm/org/apache/storm/metric/internal/LatencyStatTest.java index 29a6906d43f..5dfcf90855c 100644 --- a/storm-client/test/jvm/org/apache/storm/metric/internal/LatencyStatTest.java +++ b/storm-client/test/jvm/org/apache/storm/metric/internal/LatencyStatTest.java @@ -1,24 +1,29 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

      * http://www.apache.org/licenses/LICENSE-2.0 *

      - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.metric.internal; +import static org.junit.jupiter.api.Assertions.assertEquals; + import java.util.Map; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; - /** - * Unit test for LatencyStat + * Unit test for LatencyStat. */ public class LatencyStatTest { final long TEN_MIN_IN_MS = 10 * 60 * 1000; @@ -49,11 +54,12 @@ public void testBasic() { assertEquals(200.0, (Double) lat.getValueAndReset(time), 0.01); } - double expected = ((100.0 * TEN_MIN_IN_MS / THIRTY_SEC_IN_MS) + (200.0 * (THREE_HOUR_IN_MS - TEN_MIN_IN_MS) / THIRTY_SEC_IN_MS)) / - (THREE_HOUR_IN_MS / THIRTY_SEC_IN_MS); + double expected = ((100.0 * TEN_MIN_IN_MS / THIRTY_SEC_IN_MS) + + (200.0 * (THREE_HOUR_IN_MS - TEN_MIN_IN_MS) / THIRTY_SEC_IN_MS)) + / (THREE_HOUR_IN_MS / THIRTY_SEC_IN_MS); found = lat.getTimeLatAvg(time); assertEquals(4, found.size()); - assertEquals(200.0, found.get("600"), 0.01); //flushed the buffers completely + assertEquals(200.0, found.get("600"), 0.01); // flushed the buffers completely assertEquals(expected, found.get("10800"), 0.01); assertEquals(expected, found.get("86400"), 0.01); assertEquals(expected, found.get(":all-time"), 0.01); @@ -64,12 +70,13 @@ public void testBasic() { assertEquals(300.0, (Double) lat.getValueAndReset(time), 0.01); } - expected = ((100.0 * TEN_MIN_IN_MS / THIRTY_SEC_IN_MS) + (200.0 * (THREE_HOUR_IN_MS - TEN_MIN_IN_MS) / THIRTY_SEC_IN_MS) + - (300.0 * (ONE_DAY_IN_MS - THREE_HOUR_IN_MS) / THIRTY_SEC_IN_MS)) / - (ONE_DAY_IN_MS / THIRTY_SEC_IN_MS); + expected = ((100.0 * TEN_MIN_IN_MS / THIRTY_SEC_IN_MS) + + (200.0 * (THREE_HOUR_IN_MS - TEN_MIN_IN_MS) / THIRTY_SEC_IN_MS) + + (300.0 * (ONE_DAY_IN_MS - THREE_HOUR_IN_MS) / THIRTY_SEC_IN_MS)) + / (ONE_DAY_IN_MS / THIRTY_SEC_IN_MS); found = lat.getTimeLatAvg(time); assertEquals(4, found.size()); - assertEquals(300.0, found.get("600"), 0.01); //flushed the buffers completely + assertEquals(300.0, found.get("600"), 0.01); // flushed the buffers completely assertEquals(300.0, found.get("10800"), 0.01); assertEquals(expected, found.get("86400"), 0.01); assertEquals(expected, found.get(":all-time"), 0.01); diff --git a/storm-client/test/jvm/org/apache/storm/metric/internal/RateTrackerTest.java b/storm-client/test/jvm/org/apache/storm/metric/internal/RateTrackerTest.java index 7760d36fc6d..a0147b12147 100644 --- a/storm-client/test/jvm/org/apache/storm/metric/internal/RateTrackerTest.java +++ b/storm-client/test/jvm/org/apache/storm/metric/internal/RateTrackerTest.java @@ -1,34 +1,40 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

      * http://www.apache.org/licenses/LICENSE-2.0 *

      - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.metric.internal; -import org.junit.jupiter.api.Test; - import static org.junit.jupiter.api.Assertions.assertEquals; +import org.junit.jupiter.api.Test; + /** - * Unit test for RateTracker + * Unit test for RateTracker. */ public class RateTrackerTest { @Test public void testExactRate() { - //This test is in two phases. The first phase fills up the 10 buckets with 10 tuples each + // This test is in two phases. The first phase fills up the 10 buckets with 10 tuples each // We purposely simulate a 1 second bucket size so the rate will always be 10 per second. final long interval = 1000L; long time = 0L; RateTracker rt = new RateTracker(10000, 10, time); - double[] expected = new double[]{ 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0 }; + double[] expected = + new double[]{ 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0 }; for (int i = 0; i < expected.length; i++) { double exp = expected[i]; rt.notify(10); @@ -37,8 +43,9 @@ public void testExactRate() { rt.forceRotate(1, interval); assertEquals(exp, actual, 0.00001, "Expected rate on iteration " + i + " is wrong."); } - //In the second part of the test the rate doubles to 20 per second but the rate tracker - // increases its result slowly as we push the 10 tuples per second buckets out and replace them + // In the second part of the test the rate doubles to 20 per second but the rate tracker + // increases its result slowly as we push the 10 tuples per second buckets out and replace + // them // with 20 tuples per second. expected = new double[]{ 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0 }; for (int i = 0; i < expected.length; i++) { diff --git a/storm-client/test/jvm/org/apache/storm/metric/util/DataPointExpanderTest.java b/storm-client/test/jvm/org/apache/storm/metric/util/DataPointExpanderTest.java index 53ca205a6b6..db9ef490e9a 100644 --- a/storm-client/test/jvm/org/apache/storm/metric/util/DataPointExpanderTest.java +++ b/storm-client/test/jvm/org/apache/storm/metric/util/DataPointExpanderTest.java @@ -1,17 +1,26 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.metric.util; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -20,9 +29,6 @@ import org.apache.storm.shade.com.google.common.collect.Lists; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - public class DataPointExpanderTest { @Test @@ -31,7 +37,8 @@ public void testExpandDataPointWithExpandDisabled() { Map value = getDummyMetricMapValue(); IMetricsConsumer.DataPoint point = new IMetricsConsumer.DataPoint("test", value); - Collection expandedDataPoints = populator.expandDataPoint(point); + Collection expandedDataPoints = populator + .expandDataPoint(point); assertEquals(1, expandedDataPoints.size()); assertEquals(point, expandedDataPoints.iterator().next()); } @@ -52,7 +59,8 @@ public void testExpandDataPointsWithExpandDisabled() { public void testExpandDataPointWithVariousKindOfMetrics() { DataPointExpander populator = new DataPointExpander(true, "."); - IMetricsConsumer.DataPoint point = new IMetricsConsumer.DataPoint("point", getDummyMetricMapValue()); + IMetricsConsumer.DataPoint point = new IMetricsConsumer.DataPoint("point", + getDummyMetricMapValue()); Collection expandedDataPoints = populator.expandDataPoints(Collections.singletonList(point)); @@ -68,7 +76,8 @@ public void testExpandDataPointsWithVariousKindOfMetrics() { DataPointExpander populator = new DataPointExpander(true, ":"); IMetricsConsumer.DataPoint point1 = new IMetricsConsumer.DataPoint("point1", 2.5); - IMetricsConsumer.DataPoint point2 = new IMetricsConsumer.DataPoint("point2", getDummyMetricMapValue()); + IMetricsConsumer.DataPoint point2 = new IMetricsConsumer.DataPoint("point2", + getDummyMetricMapValue()); Collection expandedDataPoints = populator.expandDataPoints(Lists.newArrayList(point1, point2)); @@ -78,7 +87,8 @@ public void testExpandDataPointsWithVariousKindOfMetrics() { assertTrue(expandedDataPoints.contains(new IMetricsConsumer.DataPoint("point2:a", 1.0))); assertTrue(expandedDataPoints.contains(new IMetricsConsumer.DataPoint("point2:b", 2.5))); assertTrue(expandedDataPoints.contains(new IMetricsConsumer.DataPoint("point2:c", null))); - assertTrue(expandedDataPoints.contains(new IMetricsConsumer.DataPoint("point2:d", "hello"))); + assertTrue(expandedDataPoints.contains(new IMetricsConsumer.DataPoint("point2:d", + "hello"))); } @Test @@ -86,7 +96,8 @@ public void testExpandDataPointWithNullValueMetric() { DataPointExpander populator = new DataPointExpander(true, "."); IMetricsConsumer.DataPoint point = new IMetricsConsumer.DataPoint("point", null); - Collection expandedDataPoints = populator.expandDataPoint(point); + Collection expandedDataPoints = populator + .expandDataPoint(point); assertEquals(0, expandedDataPoints.size()); } diff --git a/storm-client/test/jvm/org/apache/storm/metrics2/EwmaGaugeTest.java b/storm-client/test/jvm/org/apache/storm/metrics2/EwmaGaugeTest.java index ed1b5009ceb..42e0a2803e7 100644 --- a/storm-client/test/jvm/org/apache/storm/metrics2/EwmaGaugeTest.java +++ b/storm-client/test/jvm/org/apache/storm/metrics2/EwmaGaugeTest.java @@ -1,32 +1,37 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.metrics2; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Nested; -import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; class EwmaGaugeTest { @@ -49,8 +54,8 @@ void defaultAlpha() { @DisplayName("Invalid alpha values throw IllegalArgumentException") void invalidAlphaThrows() { double[] invalidAlphas = { - 0.0, 1.0, -0.1, 1.1, - Double.NaN, Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY + 0.0, 1.0, -0.1, 1.1, + Double.NaN, Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY }; for (double alpha : invalidAlphas) { assertThrows(IllegalArgumentException.class, @@ -70,7 +75,6 @@ void validAlphaAccepted() { } } - @Nested @DisplayName("Cold-start semantics") class ColdStartTest { @@ -145,7 +149,6 @@ void zeroDeviationDecays() { } - @Nested @DisplayName("Negative value guard") class NegativeValueTest { @@ -170,7 +173,6 @@ void negativeAfterSeedIgnored() { } } - @Nested @DisplayName("getValue() preserves EWMA across calls") class GetValueIdempotentTest { diff --git a/storm-client/test/jvm/org/apache/storm/metrics2/RollingAverageGaugeTest.java b/storm-client/test/jvm/org/apache/storm/metrics2/RollingAverageGaugeTest.java index 38b3fef64ea..f8283547512 100644 --- a/storm-client/test/jvm/org/apache/storm/metrics2/RollingAverageGaugeTest.java +++ b/storm-client/test/jvm/org/apache/storm/metrics2/RollingAverageGaugeTest.java @@ -1,21 +1,26 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

      * http://www.apache.org/licenses/LICENSE-2.0 *

      - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.metrics2; -import org.junit.jupiter.api.Test; - import static org.junit.jupiter.api.Assertions.assertEquals; +import org.junit.jupiter.api.Test; + public class RollingAverageGaugeTest { @Test @@ -33,4 +38,4 @@ public void testAverage() { gauge.addValue(0); assertEquals(40.0, gauge.getValue(), 0.001); } -} \ No newline at end of file +} diff --git a/storm-client/test/jvm/org/apache/storm/metrics2/TaskMetricsTest.java b/storm-client/test/jvm/org/apache/storm/metrics2/TaskMetricsTest.java index c812994280c..bd2fe832139 100644 --- a/storm-client/test/jvm/org/apache/storm/metrics2/TaskMetricsTest.java +++ b/storm-client/test/jvm/org/apache/storm/metrics2/TaskMetricsTest.java @@ -1,18 +1,44 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.metrics2; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.contains; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + import com.codahale.metrics.Gauge; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import org.apache.storm.task.WorkerTopologyContext; import org.apache.storm.utils.ConfigUtils; import org.junit.jupiter.api.BeforeEach; @@ -23,18 +49,6 @@ import org.mockito.MockedStatic; import org.mockito.junit.jupiter.MockitoExtension; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.*; -import static org.mockito.Mockito.*; - @ExtendWith(MockitoExtension.class) class TaskMetricsTest { @@ -360,7 +374,10 @@ void concurrentEmittedTuple_registersRateCounterExactlyOnce() throws Interrupted for (int i = 0; i < threadCount; i++) { pool.submit(() -> { ready.countDown(); - try { start.await(); } catch (InterruptedException ignored) {} + try { + start.await(); + } catch (InterruptedException ignored) { + } tm.emittedTuple(STREAM_ID); done.countDown(); }); @@ -390,7 +407,10 @@ void concurrentSpoutAckedTuple_registersGaugeExactlyOnce() throws InterruptedExc for (int i = 0; i < threadCount; i++) { pool.submit(() -> { ready.countDown(); - try { start.await(); } catch (InterruptedException ignored) {} + try { + start.await(); + } catch (InterruptedException ignored) { + } tm.spoutAckedTuple(STREAM_ID, 100L); done.countDown(); }); diff --git a/storm-client/test/jvm/org/apache/storm/nimbus/NimbusInfoTest.java b/storm-client/test/jvm/org/apache/storm/nimbus/NimbusInfoTest.java index 95080e9ce8f..e20a6c53a4b 100644 --- a/storm-client/test/jvm/org/apache/storm/nimbus/NimbusInfoTest.java +++ b/storm-client/test/jvm/org/apache/storm/nimbus/NimbusInfoTest.java @@ -1,23 +1,28 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.nimbus; -import org.junit.jupiter.api.Test; - import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import org.junit.jupiter.api.Test; + class NimbusInfoTest { @Test @@ -55,4 +60,4 @@ void parseInvalidTlsPort() { actualMessage = exception.getMessage(); assertTrue(actualMessage.contains(expectedMessage)); } -} \ No newline at end of file +} diff --git a/storm-client/test/jvm/org/apache/storm/pacemaker/codec/ThriftDecoderTest.java b/storm-client/test/jvm/org/apache/storm/pacemaker/codec/ThriftDecoderTest.java index 3461037a462..11993cc22c7 100644 --- a/storm-client/test/jvm/org/apache/storm/pacemaker/codec/ThriftDecoderTest.java +++ b/storm-client/test/jvm/org/apache/storm/pacemaker/codec/ThriftDecoderTest.java @@ -1,17 +1,31 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software is distributed on an "AS + * IS" + * BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.pacemaker.codec; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + import org.apache.storm.generated.HBMessage; import org.apache.storm.generated.HBMessageData; import org.apache.storm.generated.HBServerMessageType; @@ -24,14 +38,6 @@ import org.apache.storm.utils.Utils; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertArrayEquals; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertSame; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; - @SuppressWarnings("deprecation") public class ThriftDecoderTest { @@ -50,7 +56,8 @@ static ByteBuf controlFrame(ControlMessage controlMessage) { controlMessage.write(blob); byte[] bytes = new byte[blob.readableBytes()]; blob.readBytes(bytes); - return frame(new HBMessage(HBServerMessageType.CONTROL_MESSAGE, HBMessageData.message_blob(bytes))); + return frame(new HBMessage(HBServerMessageType.CONTROL_MESSAGE, HBMessageData + .message_blob(bytes))); } private static EmbeddedChannel serverChannel() { @@ -66,14 +73,16 @@ static ByteBuf saslTokenFrame(short identifier, int declaredPayloadLen, byte[] p } byte[] bytes = new byte[blob.readableBytes()]; blob.readBytes(bytes); - return frame(new HBMessage(HBServerMessageType.SASL_MESSAGE_TOKEN, HBMessageData.message_blob(bytes))); + return frame(new HBMessage(HBServerMessageType.SASL_MESSAGE_TOKEN, HBMessageData + .message_blob(bytes))); } @Test public void serverDropsSaslTokenWithOversizedPayloadLength() { EmbeddedChannel channel = serverChannel(); - // A tiny frame that claims a ~2GB payload must not be allocated; it is dropped and the connection closed. + // A tiny frame that claims a ~2GB payload must not be allocated; it is dropped and the + // connection closed. channel.writeInbound(saslTokenFrame(SaslMessageToken.IDENTIFIER, Integer.MAX_VALUE, null)); assertNull(channel.readInbound()); @@ -123,7 +132,8 @@ public void serverAcceptsSaslMessageToken() { byte[] bytes = new byte[blob.readableBytes()]; blob.readBytes(bytes); - channel.writeInbound(frame(new HBMessage(HBServerMessageType.SASL_MESSAGE_TOKEN, HBMessageData.message_blob(bytes)))); + channel.writeInbound(frame(new HBMessage(HBServerMessageType.SASL_MESSAGE_TOKEN, + HBMessageData.message_blob(bytes)))); SaslMessageToken decoded = channel.readInbound(); assertArrayEquals(token, decoded.getSaslToken()); @@ -133,7 +143,8 @@ public void serverAcceptsSaslMessageToken() { @Test public void serverPassesHeartbeatMessages() { EmbeddedChannel channel = serverChannel(); - HBMessage message = new HBMessage(HBServerMessageType.CREATE_PATH, HBMessageData.path("/path")); + HBMessage message = new HBMessage(HBServerMessageType.CREATE_PATH, HBMessageData + .path("/path")); channel.writeInbound(frame(message)); diff --git a/storm-client/test/jvm/org/apache/storm/security/auth/AuthUtilsTestMock.java b/storm-client/test/jvm/org/apache/storm/security/auth/AuthUtilsTestMock.java index 7c85892483f..b9a5bcefee1 100644 --- a/storm-client/test/jvm/org/apache/storm/security/auth/AuthUtilsTestMock.java +++ b/storm-client/test/jvm/org/apache/storm/security/auth/AuthUtilsTestMock.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -46,7 +52,8 @@ public Set getGroups(String user) throws IOException { // ICredentialsRenewer @Override - public void renew(Map credentials, Map topologyConf, String ownerPrincipal) {} + public void renew(Map credentials, Map topologyConf, + String ownerPrincipal) {} // IAutoCredentials @Override @@ -62,7 +69,8 @@ public void populateCredentials(Map creds) {} // INimbusCredentialPlugin @Override - public void populateCredentials(Map credentials, Map topoConf) {} + public void populateCredentials(Map credentials, Map topoConf) {} // Shutdownable via INimbusCredentailPlugin @Override diff --git a/storm-client/test/jvm/org/apache/storm/security/auth/AutoSSLTest.java b/storm-client/test/jvm/org/apache/storm/security/auth/AutoSSLTest.java index d5035490862..f17c3e959ee 100644 --- a/storm-client/test/jvm/org/apache/storm/security/auth/AutoSSLTest.java +++ b/storm-client/test/jvm/org/apache/storm/security/auth/AutoSSLTest.java @@ -1,17 +1,28 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.security.auth; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + import java.io.File; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -30,13 +41,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; - public class AutoSSLTest { - final static Logger LOG = LoggerFactory.getLogger(AutoSSLTest.class); + static final Logger LOG = LoggerFactory.getLogger(AutoSSLTest.class); @Test public void testgetSSLFilesFromConf() { @@ -101,7 +107,8 @@ public void testpopulateCredentials() throws Exception { // compare contents of files if (outputFiles.length > 0) { - List linesWritten = FileUtils.readLines(new File(baseDir, outputFiles[0]), StandardCharsets.UTF_8); + List linesWritten = FileUtils.readLines(new File(baseDir, outputFiles[0]), + StandardCharsets.UTF_8); for (String l : linesWritten) { assertTrue(lines.contains(l)); } diff --git a/storm-client/test/jvm/org/apache/storm/security/auth/ClientAuthUtilsTest.java b/storm-client/test/jvm/org/apache/storm/security/auth/ClientAuthUtilsTest.java index 6854ecbd35a..de873fe4e87 100644 --- a/storm-client/test/jvm/org/apache/storm/security/auth/ClientAuthUtilsTest.java +++ b/storm-client/test/jvm/org/apache/storm/security/auth/ClientAuthUtilsTest.java @@ -18,6 +18,13 @@ package org.apache.storm.security.auth; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + import java.io.File; import java.io.IOException; import java.util.Arrays; @@ -33,13 +40,6 @@ import org.junit.jupiter.api.io.TempDir; import org.mockito.Mockito; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertSame; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.assertThrows; - public class ClientAuthUtilsTest { // JUnit ensures that the temporary folder is removed after @@ -82,7 +82,8 @@ public void getFirstValueForValidKeyTest() throws IOException { AppConfigurationEntry badEntry = Mockito.mock(AppConfigurationEntry.class); AppConfigurationEntry goodEntry = Mockito.mock(AppConfigurationEntry.class); - Mockito.>when(emptyEntry.getOptions()).thenReturn(new HashMap()); + Mockito.>when(emptyEntry.getOptions()).thenReturn(new HashMap()); Mockito.>when(badEntry.getOptions()).thenReturn(badOptionMap); Mockito.>when(goodEntry.getOptions()).thenReturn(optionMap); @@ -148,7 +149,8 @@ public void populateSubjectTest() { public void invalidConfigResultsInIOException() throws RuntimeException { HashMap conf = new HashMap<>(); conf.put("java.security.auth.login.config", "__FAKE_FILE__"); - assertThrows(RuntimeException.class, () -> assertNotNull(ClientAuthUtils.getConfiguration(conf))); + assertThrows(RuntimeException.class, () -> assertNotNull(ClientAuthUtils + .getConfiguration(conf))); } @Test @@ -167,7 +169,8 @@ public void updateSubjectWithNullThrowsTest() { @Test public void updateSubjectWithNullAutosThrowsTest() { - assertThrows(RuntimeException.class, () -> ClientAuthUtils.updateSubject(new Subject(), null, null)); + assertThrows(RuntimeException.class, () -> ClientAuthUtils.updateSubject(new Subject(), + null, null)); } @Test @@ -189,6 +192,7 @@ public void pluginCreationTest() { assertTrue( ClientAuthUtils.getPrincipalToLocalPlugin(conf).getClass() == AuthUtilsTestMock.class); - assertSame(ClientAuthUtils.getGroupMappingServiceProviderPlugin(conf).getClass(), AuthUtilsTestMock.class); + assertSame(ClientAuthUtils.getGroupMappingServiceProviderPlugin(conf).getClass(), + AuthUtilsTestMock.class); } } diff --git a/storm-client/test/jvm/org/apache/storm/security/auth/MultiThriftServerTest.java b/storm-client/test/jvm/org/apache/storm/security/auth/MultiThriftServerTest.java index c96e532e633..c396140171d 100644 --- a/storm-client/test/jvm/org/apache/storm/security/auth/MultiThriftServerTest.java +++ b/storm-client/test/jvm/org/apache/storm/security/auth/MultiThriftServerTest.java @@ -1,30 +1,39 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

      - * http://www.apache.org/licenses/LICENSE-2.0 - *

      - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

      http://www.apache.org/licenses/LICENSE-2.0 + * + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.security.auth; -import java.util.*; -import org.apache.storm.*; -import org.apache.storm.generated.Nimbus; -import org.junit.jupiter.api.*; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.Mockito.*; +import static org.mockito.Mockito.mock; + +import java.util.HashMap; +import java.util.Map; +import org.apache.storm.Config; +import org.apache.storm.generated.Nimbus; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; public class MultiThriftServerTest { - static private final Map conf = new HashMap<>(); + private static final Map conf = new HashMap<>(); static ThriftServer serverNonTls; static ThriftServer serverTls; - static private MultiThriftServer multiThriftServer; + private static MultiThriftServer multiThriftServer; @BeforeAll public static void setUp() { @@ -40,16 +49,19 @@ public static void setUp() { new Nimbus.Processor<>(handler), ThriftConnectionType.NIMBUS_TLS); } + @Test public void testAddThriftServer() { multiThriftServer.add(serverNonTls); assertEquals(serverNonTls, multiThriftServer.get(ThriftConnectionType.NIMBUS)); } + @Test public void testAddThriftServerTls() { multiThriftServer.add(serverTls); assertEquals(serverTls, multiThriftServer.get(ThriftConnectionType.NIMBUS_TLS)); } + @Test public void testAddThriftServerBoth() { multiThriftServer.add(serverTls); diff --git a/storm-client/test/jvm/org/apache/storm/security/auth/ReqContextTest.java b/storm-client/test/jvm/org/apache/storm/security/auth/ReqContextTest.java index bdb97af86f9..b4aece1fb31 100644 --- a/storm-client/test/jvm/org/apache/storm/security/auth/ReqContextTest.java +++ b/storm-client/test/jvm/org/apache/storm/security/auth/ReqContextTest.java @@ -1,17 +1,28 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.security.auth; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + import java.net.InetAddress; import java.net.UnknownHostException; import java.security.Principal; @@ -22,11 +33,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; - public class ReqContextTest { private ReqContext rc; @@ -56,7 +62,7 @@ public void testRemoteAddress() throws UnknownHostException { } /** - * If subject has no principals, request context should return null principal + * If subject has no principals, request context should return null principal. */ @Test public void testPrincipalReturnsNullWhenNoSubject() { diff --git a/storm-client/test/jvm/org/apache/storm/security/auth/SaslTransportPluginTest.java b/storm-client/test/jvm/org/apache/storm/security/auth/SaslTransportPluginTest.java index 8f8039b2943..636bb7e0246 100644 --- a/storm-client/test/jvm/org/apache/storm/security/auth/SaslTransportPluginTest.java +++ b/storm-client/test/jvm/org/apache/storm/security/auth/SaslTransportPluginTest.java @@ -1,24 +1,29 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.security.auth; -import org.apache.storm.security.auth.sasl.SaslTransportPlugin; -import org.junit.jupiter.api.Test; - import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; +import org.apache.storm.security.auth.sasl.SaslTransportPlugin; +import org.junit.jupiter.api.Test; + public class SaslTransportPluginTest { @Test diff --git a/storm-client/test/jvm/org/apache/storm/security/auth/ShellBasedGroupsMappingTest.java b/storm-client/test/jvm/org/apache/storm/security/auth/ShellBasedGroupsMappingTest.java index 60920b0a958..704db8f438a 100644 --- a/storm-client/test/jvm/org/apache/storm/security/auth/ShellBasedGroupsMappingTest.java +++ b/storm-client/test/jvm/org/apache/storm/security/auth/ShellBasedGroupsMappingTest.java @@ -16,27 +16,27 @@ package org.apache.storm.security.auth; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + import java.util.HashMap; import java.util.Map; import java.util.Set; import org.apache.storm.Config; import org.apache.storm.utils.ShellCommandRunner; import org.apache.storm.utils.ShellUtils; -import org.apache.storm.utils.Time; import org.apache.storm.utils.Time.SimulatedTime; +import org.apache.storm.utils.Time; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.condition.DisabledOnOs; import org.junit.jupiter.api.condition.OS; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.contains; -import static org.hamcrest.Matchers.containsInAnyOrder; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.not; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - public class ShellBasedGroupsMappingTest { private static final String TEST_TWO_GROUPS = "group1 group2"; @@ -63,7 +63,8 @@ public void setUp() { public void testCanGetGroups() throws Exception { try (SimulatedTime ignored = new SimulatedTime()) { groupsMapping.prepare(topoConf); - when(mockShell.execCommand(ShellUtils.getGroupsForUserCommand(TEST_USER_1))).thenReturn(TEST_TWO_GROUPS); + when(mockShell.execCommand(ShellUtils.getGroupsForUserCommand(TEST_USER_1))) + .thenReturn(TEST_TWO_GROUPS); Set groups = groupsMapping.getGroups(TEST_USER_1); @@ -76,7 +77,8 @@ public void testCanGetGroups() throws Exception { public void testWillCacheGroups() throws Exception { try (SimulatedTime ignored = new SimulatedTime()) { groupsMapping.prepare(topoConf); - when(mockShell.execCommand(ShellUtils.getGroupsForUserCommand(TEST_USER_1))).thenReturn(TEST_TWO_GROUPS, TEST_NO_GROUPS); + when(mockShell.execCommand(ShellUtils.getGroupsForUserCommand(TEST_USER_1))) + .thenReturn(TEST_TWO_GROUPS, TEST_NO_GROUPS); Set firstGroups = groupsMapping.getGroups(TEST_USER_1); Set secondGroups = groupsMapping.getGroups(TEST_USER_1); @@ -90,7 +92,8 @@ public void testWillCacheGroups() throws Exception { public void testWillExpireCache() throws Exception { try (SimulatedTime ignored = new SimulatedTime()) { groupsMapping.prepare(topoConf); - when(mockShell.execCommand(ShellUtils.getGroupsForUserCommand(TEST_USER_1))).thenReturn(TEST_TWO_GROUPS, TEST_NO_GROUPS); + when(mockShell.execCommand(ShellUtils.getGroupsForUserCommand(TEST_USER_1))) + .thenReturn(TEST_TWO_GROUPS, TEST_NO_GROUPS); Set firstGroups = groupsMapping.getGroups(TEST_USER_1); Time.advanceTimeSecs(CACHE_EXPIRATION_SECS * 2); diff --git a/storm-client/test/jvm/org/apache/storm/security/auth/SubjectCompatTest.java b/storm-client/test/jvm/org/apache/storm/security/auth/SubjectCompatTest.java index f938eab4d09..d52536e53c8 100644 --- a/storm-client/test/jvm/org/apache/storm/security/auth/SubjectCompatTest.java +++ b/storm-client/test/jvm/org/apache/storm/security/auth/SubjectCompatTest.java @@ -1,26 +1,32 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.security.auth; -import java.io.IOException; -import javax.security.auth.Subject; -import org.junit.jupiter.api.Test; - import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; +import java.io.IOException; +import javax.security.auth.Subject; +import org.junit.jupiter.api.Test; + class SubjectCompatTest { @Test diff --git a/storm-client/test/jvm/org/apache/storm/security/auth/TestingITransportPlugin.java b/storm-client/test/jvm/org/apache/storm/security/auth/TestingITransportPlugin.java index 399d1462acd..eaff34ff601 100644 --- a/storm-client/test/jvm/org/apache/storm/security/auth/TestingITransportPlugin.java +++ b/storm-client/test/jvm/org/apache/storm/security/auth/TestingITransportPlugin.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

      - * http://www.apache.org/licenses/LICENSE-2.0 - *

      - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

      http://www.apache.org/licenses/LICENSE-2.0 + * + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -31,7 +37,8 @@ public TServer getServer(TProcessor processor) throws IOException, TTransportExc } @Override - public TTransport connect(TTransport transport, String serverHost, String asUser) throws IOException, TTransportException { + public TTransport connect(TTransport transport, String serverHost, + String asUser) throws IOException, TTransportException { return null; } diff --git a/storm-client/test/jvm/org/apache/storm/security/auth/ThriftClientTest.java b/storm-client/test/jvm/org/apache/storm/security/auth/ThriftClientTest.java index 7c386ac6c46..8da3a31ba0c 100644 --- a/storm-client/test/jvm/org/apache/storm/security/auth/ThriftClientTest.java +++ b/storm-client/test/jvm/org/apache/storm/security/auth/ThriftClientTest.java @@ -1,17 +1,26 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.security.auth; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + import java.util.Map; import org.apache.storm.Config; import org.apache.storm.thrift.transport.TTransportException; @@ -19,9 +28,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; - public class ThriftClientTest { private final int NIMBUS_TIMEOUT = 3 * 1000; @@ -36,13 +42,15 @@ public void setup() { @Test public void testConstructorThrowsIfPortNegative() { assertThrows(IllegalArgumentException.class, - () -> new ThriftClient(conf, ThriftConnectionType.DRPC, "bogushost", -1, NIMBUS_TIMEOUT)); + () -> new ThriftClient(conf, ThriftConnectionType.DRPC, "bogushost", -1, + NIMBUS_TIMEOUT)); } @Test public void testConstructorThrowsIfPortZero() { assertThrows(IllegalArgumentException.class, - () -> new ThriftClient(conf, ThriftConnectionType.DRPC, "bogushost", 0, NIMBUS_TIMEOUT)); + () -> new ThriftClient(conf, ThriftConnectionType.DRPC, "bogushost", 0, + NIMBUS_TIMEOUT)); } @Test @@ -56,6 +64,7 @@ public void testConstructorThrowsIfHostEmpty() { Exception e = assertThrows(RuntimeException.class, () -> new ThriftClient(conf, ThriftConnectionType.DRPC, "", 4242, NIMBUS_TIMEOUT)); // Now the cause of the thrown exception must be TTransportException - assertTrue(e.getCause().getCause() instanceof TTransportException, e.getCause().getMessage()); + assertTrue(e.getCause().getCause() instanceof TTransportException, e.getCause() + .getMessage()); } } diff --git a/storm-client/test/jvm/org/apache/storm/security/auth/TlsTransportPluginTest.java b/storm-client/test/jvm/org/apache/storm/security/auth/TlsTransportPluginTest.java index 5e66be3f643..e2d47c815a8 100644 --- a/storm-client/test/jvm/org/apache/storm/security/auth/TlsTransportPluginTest.java +++ b/storm-client/test/jvm/org/apache/storm/security/auth/TlsTransportPluginTest.java @@ -1,17 +1,28 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

      - * http://www.apache.org/licenses/LICENSE-2.0 - *

      - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

      http://www.apache.org/licenses/LICENSE-2.0 + * + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.security.auth; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; +import static org.mockito.Mockito.mock; import java.io.IOException; import java.io.InputStream; @@ -34,16 +45,11 @@ import org.apache.storm.security.auth.tls.ReloadableTsslTransportFactory; import org.apache.storm.security.auth.tls.TlsTransportPlugin; import org.apache.storm.thrift.transport.TServerSocket; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import static org.mockito.Mockito.mock; class TlsTransportPluginTest { - static private final Map conf = new HashMap<>(); + private static final Map conf = new HashMap<>(); Nimbus.Iface handler; private TlsTransportPlugin tlsTransportPlugin; private final ThriftConnectionType type = ThriftConnectionType.NIMBUS_TLS; @@ -56,6 +62,7 @@ void setUp() { conf.put(Config.STORM_THRIFT_TRANSPORT_PLUGIN, TlsTransportPlugin.class.getName()); handler = mock(Nimbus.Iface.class); } + @Test void testNonTlsConnection() { assertThrows(IllegalArgumentException.class, () -> { @@ -70,7 +77,8 @@ void testValidTlsSetup() { conf.put(Config.STORM_THRIFT_TLS_SOCKET_TIMEOUT_MS, 60); conf.put(Config.NIMBUS_THRIFT_TLS_SERVER_KEYSTORE_PATH, testDataPath + "testKeyStore.jks"); conf.put(Config.NIMBUS_THRIFT_TLS_SERVER_KEYSTORE_PASSWORD, "testpass"); - conf.put(Config.NIMBUS_THRIFT_TLS_SERVER_TRUSTSTORE_PATH, testDataPath + "testTrustStore.jks"); + conf.put(Config.NIMBUS_THRIFT_TLS_SERVER_TRUSTSTORE_PATH, testDataPath + + "testTrustStore.jks"); conf.put(Config.NIMBUS_THRIFT_TLS_SERVER_TRUSTSTORE_PASSWORD, "testpass"); conf.put(Config.NIMBUS_THRIFT_TLS_THREADS, 1); conf.put(Config.NIMBUS_QUEUE_SIZE, 1); @@ -96,7 +104,8 @@ void testClientAuthRequiredRejectsUnauthenticatedClient() throws Exception { conf.put(Config.STORM_THRIFT_TLS_SOCKET_TIMEOUT_MS, 5000); conf.put(Config.NIMBUS_THRIFT_TLS_SERVER_KEYSTORE_PATH, testDataPath + "testKeyStore.jks"); conf.put(Config.NIMBUS_THRIFT_TLS_SERVER_KEYSTORE_PASSWORD, "testpass"); - conf.put(Config.NIMBUS_THRIFT_TLS_SERVER_TRUSTSTORE_PATH, testDataPath + "testTrustStore.jks"); + conf.put(Config.NIMBUS_THRIFT_TLS_SERVER_TRUSTSTORE_PATH, testDataPath + + "testTrustStore.jks"); conf.put(Config.NIMBUS_THRIFT_TLS_SERVER_TRUSTSTORE_PASSWORD, "testpass"); conf.put(Config.NIMBUS_THRIFT_TLS_CLIENT_AUTH_REQUIRED, true); @@ -122,10 +131,12 @@ void testClientAuthRequiredRejectsUnauthenticatedClient() throws Exception { // Trust-only client context: no client keystore, so the client cannot present a cert. KeyStore ts = KeyStore.getInstance("JKS"); - try (InputStream in = Files.newInputStream(Paths.get(testDataPath + "testTrustStore.jks"))) { + try (InputStream in = Files.newInputStream(Paths.get(testDataPath + + "testTrustStore.jks"))) { ts.load(in, "testpass".toCharArray()); } - TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory + .getDefaultAlgorithm()); tmf.init(ts); SSLContext sslContext = SSLContext.getInstance("TLSv1.2"); sslContext.init(null, tmf.getTrustManagers(), new SecureRandom()); @@ -146,4 +157,4 @@ void testClientAuthRequiredRejectsUnauthenticatedClient() throws Exception { serverTransport.close(); } } -} \ No newline at end of file +} diff --git a/storm-client/test/jvm/org/apache/storm/security/auth/X509CertPrincipalToLocalTest.java b/storm-client/test/jvm/org/apache/storm/security/auth/X509CertPrincipalToLocalTest.java index 8ee550cf854..04d70214a45 100644 --- a/storm-client/test/jvm/org/apache/storm/security/auth/X509CertPrincipalToLocalTest.java +++ b/storm-client/test/jvm/org/apache/storm/security/auth/X509CertPrincipalToLocalTest.java @@ -1,41 +1,47 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.security.auth; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; + import java.security.AccessControlException; import java.util.HashMap; - import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.fail; - public class X509CertPrincipalToLocalTest { private static X509CertPrincipalToLocal x509CertPrincipalToLocal; + @BeforeAll - static public void setup() { + public static void setup() { x509CertPrincipalToLocal = new X509CertPrincipalToLocal(); x509CertPrincipalToLocal.prepare(new HashMap() {{ - put(X509CertPrincipalToLocal.X509_CERT_PRINCIPAL_TO_LOCAL_REGEX, "test:test_role\\.uid\\.(.+)"); - }}); + put(X509CertPrincipalToLocal.X509_CERT_PRINCIPAL_TO_LOCAL_REGEX, + "test:test_role\\.uid\\.(.+)"); + }}); } @Test public void toLocalTest() { assertEquals("xyz", x509CertPrincipalToLocal.toLocal("CN=test:test_role.uid.xyz")); } + @Test public void toLocalTestNegative() { try { diff --git a/storm-client/test/jvm/org/apache/storm/security/auth/authorizer/DRPCSimpleACLAuthorizerTest.java b/storm-client/test/jvm/org/apache/storm/security/auth/authorizer/DRPCSimpleACLAuthorizerTest.java index f439fd5cb86..102c8e45e64 100644 --- a/storm-client/test/jvm/org/apache/storm/security/auth/authorizer/DRPCSimpleACLAuthorizerTest.java +++ b/storm-client/test/jvm/org/apache/storm/security/auth/authorizer/DRPCSimpleACLAuthorizerTest.java @@ -1,17 +1,28 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

      - * http://www.apache.org/licenses/LICENSE-2.0 - *

      - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

      http://www.apache.org/licenses/LICENSE-2.0 + * + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.security.auth.authorizer; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + import java.io.BufferedWriter; import java.io.File; import java.io.FileWriter; @@ -29,11 +40,6 @@ import org.junit.jupiter.api.Test; import org.mockito.Mockito; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; - public class DRPCSimpleACLAuthorizerTest { private static final String function = "jump"; @@ -51,12 +57,14 @@ public class DRPCSimpleACLAuthorizerTest { public static void setup() { strictHandler = new DRPCSimpleACLAuthorizer(); strictHandler.prepare(ImmutableMap - .of(Config.DRPC_AUTHORIZER_ACL_STRICT, true, Config.DRPC_AUTHORIZER_ACL_FILENAME, aclFile, + .of(Config.DRPC_AUTHORIZER_ACL_STRICT, true, + Config.DRPC_AUTHORIZER_ACL_FILENAME, aclFile, Config.STORM_PRINCIPAL_TO_LOCAL_PLUGIN, KerberosPrincipalToLocal.class.getName())); permissiveHandler = new DRPCSimpleACLAuthorizer(); permissiveHandler.prepare(ImmutableMap - .of(Config.DRPC_AUTHORIZER_ACL_STRICT, false, Config.DRPC_AUTHORIZER_ACL_FILENAME, aclFile, + .of(Config.DRPC_AUTHORIZER_ACL_STRICT, false, + Config.DRPC_AUTHORIZER_ACL_FILENAME, aclFile, Config.STORM_PRINCIPAL_TO_LOCAL_PLUGIN, KerberosPrincipalToLocal.class.getName())); } @@ -152,24 +160,29 @@ public void test_deny_when_no_function_given() { @Test public void test_deny_when_invalid_user_given() { - assertFalse(isPermitted(strictHandler, Mockito.mock(ReqContext.class), "execute", function)); + assertFalse(isPermitted(strictHandler, Mockito.mock(ReqContext.class), "execute", + function)); assertFalse(isPermitted(strictHandler, null, "execute", function)); - assertFalse(isPermitted(permissiveHandler, Mockito.mock(ReqContext.class), "execute", function)); + assertFalse(isPermitted(permissiveHandler, Mockito.mock(ReqContext.class), "execute", + function)); assertFalse(isPermitted(permissiveHandler, null, "execute", function)); } - private boolean isPermitted(IAuthorizer authorizer, ReqContext context, String operation, String function) { + private boolean isPermitted(IAuthorizer authorizer, ReqContext context, String operation, + String function) { Map config = new HashMap<>(); config.put(DRPCSimpleACLAuthorizer.FUNCTION_KEY, function); return authorizer.permit(context, operation, config); } /** - * {@link DRPCSimpleACLAuthorizer} should still work even if {@link Config#DRPC_AUTHORIZER_ACL} has no values. + * {@link DRPCSimpleACLAuthorizer} should still work even if {@link Config#DRPC_AUTHORIZER_ACL} + * has no values. + * * @throws IOException if there is any issue with creating or writing the temp file. */ @Test @@ -183,8 +196,10 @@ public void test_read_acl_no_values() throws IOException { writer.close(); authorizer.prepare(ImmutableMap - .of(Config.DRPC_AUTHORIZER_ACL_STRICT, true, Config.DRPC_AUTHORIZER_ACL_FILENAME, tempFile.toString(), - Config.STORM_PRINCIPAL_TO_LOCAL_PLUGIN, KerberosPrincipalToLocal.class.getName())); + .of(Config.DRPC_AUTHORIZER_ACL_STRICT, true, Config.DRPC_AUTHORIZER_ACL_FILENAME, + tempFile.toString(), + Config.STORM_PRINCIPAL_TO_LOCAL_PLUGIN, KerberosPrincipalToLocal.class + .getName())); Map acl = authorizer.readAclFromConfig(); assertEquals(0, acl.size()); @@ -192,6 +207,7 @@ public void test_read_acl_no_values() throws IOException { /** * The file of {@link Config#DRPC_AUTHORIZER_ACL_FILENAME} can not be empty. + * * @throws IOException if there is any issue with creating the temp file. */ @Test @@ -202,8 +218,10 @@ public void test_read_acl_empty_file() throws IOException { tempFile.deleteOnExit(); authorizer.prepare(ImmutableMap - .of(Config.DRPC_AUTHORIZER_ACL_STRICT, true, Config.DRPC_AUTHORIZER_ACL_FILENAME, tempFile.toString(), - Config.STORM_PRINCIPAL_TO_LOCAL_PLUGIN, KerberosPrincipalToLocal.class.getName())); + .of(Config.DRPC_AUTHORIZER_ACL_STRICT, true, Config.DRPC_AUTHORIZER_ACL_FILENAME, + tempFile.toString(), + Config.STORM_PRINCIPAL_TO_LOCAL_PLUGIN, KerberosPrincipalToLocal.class + .getName())); Exception exception = assertThrows(RuntimeException.class, authorizer::readAclFromConfig); assertTrue(exception.getMessage().contains("doesn't have any valid storm configs")); diff --git a/storm-client/test/jvm/org/apache/storm/security/auth/authorizer/SimpleACLAuthorizerTest.java b/storm-client/test/jvm/org/apache/storm/security/auth/authorizer/SimpleACLAuthorizerTest.java index 7d67b034c77..9c565773b21 100644 --- a/storm-client/test/jvm/org/apache/storm/security/auth/authorizer/SimpleACLAuthorizerTest.java +++ b/storm-client/test/jvm/org/apache/storm/security/auth/authorizer/SimpleACLAuthorizerTest.java @@ -1,27 +1,25 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.security.auth.authorizer; -import org.apache.storm.Config; -import org.apache.storm.security.auth.IAuthorizer; -import org.apache.storm.security.auth.IGroupMappingServiceProvider; -import org.apache.storm.security.auth.ReqContext; -import org.apache.storm.utils.ConfigUtils; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.condition.DisabledOnOs; -import org.junit.jupiter.api.condition.OS; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; -import javax.security.auth.Subject; import java.security.Principal; import java.util.Collection; import java.util.Collections; @@ -29,9 +27,15 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; - -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; +import javax.security.auth.Subject; +import org.apache.storm.Config; +import org.apache.storm.security.auth.IAuthorizer; +import org.apache.storm.security.auth.IGroupMappingServiceProvider; +import org.apache.storm.security.auth.ReqContext; +import org.apache.storm.utils.ConfigUtils; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.DisabledOnOs; +import org.junit.jupiter.api.condition.OS; public class SimpleACLAuthorizerTest { @@ -40,7 +44,8 @@ public class SimpleACLAuthorizerTest { public void SimpleACLUserAuthTest() { Map clusterConf = ConfigUtils.readStormConfig(); Collection adminUserSet = new HashSet<>(Collections.singletonList("admin")); - Collection supervisorUserSet = new HashSet<>(Collections.singletonList("supervisor")); + Collection supervisorUserSet = new HashSet<>(Collections + .singletonList("supervisor")); clusterConf.put(Config.NIMBUS_ADMINS, adminUserSet); clusterConf.put(Config.NIMBUS_SUPERVISOR_USERS, supervisorUserSet); @@ -54,47 +59,63 @@ public void SimpleACLUserAuthTest() { authorizer.prepare(clusterConf); assertTrue(authorizer.permit(new ReqContext(adminUser), "submitTopology", new HashMap<>())); - assertFalse(authorizer.permit(new ReqContext(supervisorUser), "submitTopology", new HashMap<>())); + assertFalse(authorizer.permit(new ReqContext(supervisorUser), "submitTopology", + new HashMap<>())); assertTrue(authorizer.permit(new ReqContext(userA), "submitTopology", new HashMap<>())); assertTrue(authorizer.permit(new ReqContext(userB), "submitTopology", new HashMap<>())); assertTrue(authorizer.permit(new ReqContext(adminUser), "fileUpload", new HashMap<>())); - assertFalse(authorizer.permit(new ReqContext(supervisorUser), "fileUpload", new HashMap<>())); + assertFalse(authorizer.permit(new ReqContext(supervisorUser), "fileUpload", + new HashMap<>())); assertTrue(authorizer.permit(new ReqContext(userA), "fileUpload", new HashMap<>())); assertTrue(authorizer.permit(new ReqContext(userB), "fileUpload", new HashMap<>())); - assertTrue(authorizer.permit(new ReqContext(adminUser), "createStateInZookeeper", new HashMap<>())); - assertFalse(authorizer.permit(new ReqContext(supervisorUser), "createStateInZookeeper", new HashMap<>())); - assertTrue(authorizer.permit(new ReqContext(userA), "createStateInZookeeper", new HashMap<>())); - assertTrue(authorizer.permit(new ReqContext(userB), "createStateInZookeeper", new HashMap<>())); + assertTrue(authorizer.permit(new ReqContext(adminUser), "createStateInZookeeper", + new HashMap<>())); + assertFalse(authorizer.permit(new ReqContext(supervisorUser), "createStateInZookeeper", + new HashMap<>())); + assertTrue(authorizer.permit(new ReqContext(userA), "createStateInZookeeper", + new HashMap<>())); + assertTrue(authorizer.permit(new ReqContext(userB), "createStateInZookeeper", + new HashMap<>())); assertTrue(authorizer.permit(new ReqContext(adminUser), "getNimbusConf", new HashMap<>())); - assertFalse(authorizer.permit(new ReqContext(supervisorUser), "getNimbusConf", new HashMap<>())); + assertFalse(authorizer.permit(new ReqContext(supervisorUser), "getNimbusConf", + new HashMap<>())); assertTrue(authorizer.permit(new ReqContext(userA), "getNimbusConf", new HashMap<>())); assertTrue(authorizer.permit(new ReqContext(userB), "getNimbusConf", new HashMap<>())); assertTrue(authorizer.permit(new ReqContext(adminUser), "listBlobs", new HashMap<>())); - assertFalse(authorizer.permit(new ReqContext(supervisorUser), "listBlobs", new HashMap<>())); + assertFalse(authorizer.permit(new ReqContext(supervisorUser), "listBlobs", + new HashMap<>())); assertTrue(authorizer.permit(new ReqContext(userA), "listBlobs", new HashMap<>())); assertTrue(authorizer.permit(new ReqContext(userB), "listBlobs", new HashMap<>())); assertTrue(authorizer.permit(new ReqContext(adminUser), "getClusterInfo", new HashMap<>())); - assertFalse(authorizer.permit(new ReqContext(supervisorUser), "getClusterInfo", new HashMap<>())); + assertFalse(authorizer.permit(new ReqContext(supervisorUser), "getClusterInfo", + new HashMap<>())); assertTrue(authorizer.permit(new ReqContext(userA), "getClusterInfo", new HashMap<>())); assertTrue(authorizer.permit(new ReqContext(userB), "getClusterInfo", new HashMap<>())); - assertTrue(authorizer.permit(new ReqContext(adminUser), "getTopologyHistory", new HashMap<>())); - assertFalse(authorizer.permit(new ReqContext(supervisorUser), "getTopologyHistory", new HashMap<>())); + assertTrue(authorizer.permit(new ReqContext(adminUser), "getTopologyHistory", + new HashMap<>())); + assertFalse(authorizer.permit(new ReqContext(supervisorUser), "getTopologyHistory", + new HashMap<>())); assertTrue(authorizer.permit(new ReqContext(userA), "getTopologyHistory", new HashMap<>())); assertTrue(authorizer.permit(new ReqContext(userB), "getTopologyHistory", new HashMap<>())); - assertTrue(authorizer.permit(new ReqContext(adminUser), "getSupervisorPageInfo", new HashMap<>())); - assertFalse(authorizer.permit(new ReqContext(supervisorUser), "getSupervisorPageInfo", new HashMap<>())); - assertTrue(authorizer.permit(new ReqContext(userA), "getSupervisorPageInfo", new HashMap<>())); - assertTrue(authorizer.permit(new ReqContext(userB), "getSupervisorPageInfo", new HashMap<>())); + assertTrue(authorizer.permit(new ReqContext(adminUser), "getSupervisorPageInfo", + new HashMap<>())); + assertFalse(authorizer.permit(new ReqContext(supervisorUser), "getSupervisorPageInfo", + new HashMap<>())); + assertTrue(authorizer.permit(new ReqContext(userA), "getSupervisorPageInfo", + new HashMap<>())); + assertTrue(authorizer.permit(new ReqContext(userB), "getSupervisorPageInfo", + new HashMap<>())); assertTrue(authorizer.permit(new ReqContext(adminUser), "fileDownload", new HashMap<>())); - assertTrue(authorizer.permit(new ReqContext(supervisorUser), "fileDownload", new HashMap<>())); + assertTrue(authorizer.permit(new ReqContext(supervisorUser), "fileDownload", + new HashMap<>())); assertFalse(authorizer.permit(new ReqContext(userA), "fileDownload", new HashMap<>())); assertFalse(authorizer.permit(new ReqContext(userB), "fileDownload", new HashMap<>())); @@ -143,17 +164,20 @@ public void SimpleACLUserAuthTest() { assertFalse(authorizer.permit(new ReqContext(userB), "getTopologyInfo", topoConf)); assertTrue(authorizer.permit(new ReqContext(adminUser), "getTopologyPageInfo", topoConf)); - assertFalse(authorizer.permit(new ReqContext(supervisorUser), "getTopologyPageInfo", topoConf)); + assertFalse(authorizer.permit(new ReqContext(supervisorUser), "getTopologyPageInfo", + topoConf)); assertTrue(authorizer.permit(new ReqContext(userA), "getTopologyPageInfo", topoConf)); assertFalse(authorizer.permit(new ReqContext(userB), "getTopologyPageInfo", topoConf)); assertTrue(authorizer.permit(new ReqContext(adminUser), "getComponentPageInfo", topoConf)); - assertFalse(authorizer.permit(new ReqContext(supervisorUser), "getComponentPageInfo", topoConf)); + assertFalse(authorizer.permit(new ReqContext(supervisorUser), "getComponentPageInfo", + topoConf)); assertTrue(authorizer.permit(new ReqContext(userA), "getComponentPageInfo", topoConf)); assertFalse(authorizer.permit(new ReqContext(userB), "getComponentPageInfo", topoConf)); assertTrue(authorizer.permit(new ReqContext(adminUser), "uploadNewCredentials", topoConf)); - assertFalse(authorizer.permit(new ReqContext(supervisorUser), "uploadNewCredentials", topoConf)); + assertFalse(authorizer.permit(new ReqContext(supervisorUser), "uploadNewCredentials", + topoConf)); assertTrue(authorizer.permit(new ReqContext(userA), "uploadNewCredentials", topoConf)); assertFalse(authorizer.permit(new ReqContext(userB), "uploadNewCredentials", topoConf)); @@ -163,19 +187,28 @@ public void SimpleACLUserAuthTest() { assertFalse(authorizer.permit(new ReqContext(userB), "setLogConfig", topoConf)); assertTrue(authorizer.permit(new ReqContext(adminUser), "setWorkerProfiler", topoConf)); - assertFalse(authorizer.permit(new ReqContext(supervisorUser), "setWorkerProfiler", topoConf)); + assertFalse(authorizer.permit(new ReqContext(supervisorUser), "setWorkerProfiler", + topoConf)); assertTrue(authorizer.permit(new ReqContext(userA), "setWorkerProfiler", topoConf)); assertFalse(authorizer.permit(new ReqContext(userB), "setWorkerProfiler", topoConf)); - assertTrue(authorizer.permit(new ReqContext(adminUser), "getWorkerProfileActionExpiry", topoConf)); - assertFalse(authorizer.permit(new ReqContext(supervisorUser), "getWorkerProfileActionExpiry", topoConf)); - assertTrue(authorizer.permit(new ReqContext(userA), "getWorkerProfileActionExpiry", topoConf)); - assertFalse(authorizer.permit(new ReqContext(userB), "getWorkerProfileActionExpiry", topoConf)); - - assertTrue(authorizer.permit(new ReqContext(adminUser), "getComponentPendingProfileActions", topoConf)); - assertFalse(authorizer.permit(new ReqContext(supervisorUser), "getComponentPendingProfileActions", topoConf)); - assertTrue(authorizer.permit(new ReqContext(userA), "getComponentPendingProfileActions", topoConf)); - assertFalse(authorizer.permit(new ReqContext(userB), "getComponentPendingProfileActions", topoConf)); + assertTrue(authorizer.permit(new ReqContext(adminUser), "getWorkerProfileActionExpiry", + topoConf)); + assertFalse(authorizer.permit(new ReqContext(supervisorUser), + "getWorkerProfileActionExpiry", topoConf)); + assertTrue(authorizer.permit(new ReqContext(userA), "getWorkerProfileActionExpiry", + topoConf)); + assertFalse(authorizer.permit(new ReqContext(userB), "getWorkerProfileActionExpiry", + topoConf)); + + assertTrue(authorizer.permit(new ReqContext(adminUser), "getComponentPendingProfileActions", + topoConf)); + assertFalse(authorizer.permit(new ReqContext(supervisorUser), + "getComponentPendingProfileActions", topoConf)); + assertTrue(authorizer.permit(new ReqContext(userA), "getComponentPendingProfileActions", + topoConf)); + assertFalse(authorizer.permit(new ReqContext(userB), "getComponentPendingProfileActions", + topoConf)); assertTrue(authorizer.permit(new ReqContext(adminUser), "startProfiling", topoConf)); assertFalse(authorizer.permit(new ReqContext(supervisorUser), "startProfiling", topoConf)); @@ -218,7 +251,8 @@ public void SimpleACLUserAuthTest() { public void SimpleACLNimbusUserAuthTest() { Map clusterConf = ConfigUtils.readStormConfig(); Collection adminUserSet = new HashSet<>(Collections.singletonList("admin")); - Collection supervisorUserSet = new HashSet<>(Collections.singletonList("supervisor")); + Collection supervisorUserSet = new HashSet<>(Collections + .singletonList("supervisor")); Collection nimbusUserSet = new HashSet<>(Collections.singletonList("user-a")); clusterConf.put(Config.NIMBUS_ADMINS, adminUserSet); @@ -237,7 +271,8 @@ public void SimpleACLNimbusUserAuthTest() { assertTrue(authorizer.permit(new ReqContext(userA), "submitTopology", new HashMap<>())); assertFalse(authorizer.permit(new ReqContext(userB), "submitTopology", new HashMap<>())); assertTrue(authorizer.permit(new ReqContext(adminUser), "fileUpload", new HashMap<>())); - assertTrue(authorizer.permit(new ReqContext(supervisorUser), "fileDownload", new HashMap<>())); + assertTrue(authorizer.permit(new ReqContext(supervisorUser), "fileDownload", + new HashMap<>())); } @Test @@ -250,30 +285,36 @@ public void SimpleACLNimbusGroupAuthTest() { // neither nimbus.users nor nimbus.groups is set, so there is no restriction IAuthorizer authorizer = prepareNimbusAuthorizer(null, null); assertTrue(authorizer.permit(new ReqContext(userA), "submitTopology", new HashMap<>())); - assertTrue(authorizer.permit(new ReqContext(userInGroup), "submitTopology", new HashMap<>())); + assertTrue(authorizer.permit(new ReqContext(userInGroup), "submitTopology", + new HashMap<>())); assertTrue(authorizer.permit(new ReqContext(userB), "getClusterInfo", new HashMap<>())); // only nimbus.users is set authorizer = prepareNimbusAuthorizer(Collections.singletonList("user-a"), null); assertTrue(authorizer.permit(new ReqContext(userA), "submitTopology", new HashMap<>())); - assertFalse(authorizer.permit(new ReqContext(userInGroup), "submitTopology", new HashMap<>())); + assertFalse(authorizer.permit(new ReqContext(userInGroup), "submitTopology", + new HashMap<>())); assertFalse(authorizer.permit(new ReqContext(userB), "getClusterInfo", new HashMap<>())); // only nimbus.groups is set authorizer = prepareNimbusAuthorizer(null, Collections.singletonList("group-readonly")); - assertTrue(authorizer.permit(new ReqContext(userInGroup), "submitTopology", new HashMap<>())); + assertTrue(authorizer.permit(new ReqContext(userInGroup), "submitTopology", + new HashMap<>())); assertFalse(authorizer.permit(new ReqContext(userA), "submitTopology", new HashMap<>())); assertFalse(authorizer.permit(new ReqContext(userB), "fileUpload", new HashMap<>())); assertFalse(authorizer.permit(new ReqContext(userB), "getClusterInfo", new HashMap<>())); // both nimbus.users and nimbus.groups are set - authorizer = prepareNimbusAuthorizer(Collections.singletonList("user-a"), Collections.singletonList("group-readonly")); + authorizer = prepareNimbusAuthorizer(Collections.singletonList("user-a"), Collections + .singletonList("group-readonly")); assertTrue(authorizer.permit(new ReqContext(userA), "submitTopology", new HashMap<>())); - assertTrue(authorizer.permit(new ReqContext(userInGroup), "submitTopology", new HashMap<>())); + assertTrue(authorizer.permit(new ReqContext(userInGroup), "submitTopology", + new HashMap<>())); assertFalse(authorizer.permit(new ReqContext(userB), "submitTopology", new HashMap<>())); } - private IAuthorizer prepareNimbusAuthorizer(Collection nimbusUsers, Collection nimbusGroups) { + private IAuthorizer prepareNimbusAuthorizer(Collection nimbusUsers, + Collection nimbusGroups) { Map clusterConf = ConfigUtils.readStormConfig(); clusterConf.put(Config.STORM_GROUP_MAPPING_SERVICE_PROVIDER_PLUGIN, SimpleACLTopologyReadOnlyGroupAuthTestMock.class.getName()); @@ -300,7 +341,8 @@ public void SimpleACLTopologyReadOnlyUserAuthTest() { Collection topologyUserSet = new HashSet<>(Collections.singletonList("user-a")); topoConf.put(Config.TOPOLOGY_USERS, topologyUserSet); - Collection topologyReadOnlyUserSet = new HashSet<>(Collections.singletonList("user-readonly")); + Collection topologyReadOnlyUserSet = new HashSet<>(Collections + .singletonList("user-readonly")); topoConf.put(Config.TOPOLOGY_READONLY_USERS, topologyReadOnlyUserSet); Subject userA = createSubject("user-a"); @@ -342,15 +384,18 @@ public void SimpleACLTopologyReadOnlyUserAuthTest() { assertTrue(authorizer.permit(new ReqContext(userA), "getTopologyInfo", topoConf)); assertFalse(authorizer.permit(new ReqContext(userB), "getTopologyInfo", topoConf)); - assertTrue(authorizer.permit(new ReqContext(readOnlyUser), "getTopologyPageInfo", topoConf)); + assertTrue(authorizer.permit(new ReqContext(readOnlyUser), "getTopologyPageInfo", + topoConf)); assertTrue(authorizer.permit(new ReqContext(userA), "getTopologyPageInfo", topoConf)); assertFalse(authorizer.permit(new ReqContext(userB), "getTopologyPageInfo", topoConf)); - assertTrue(authorizer.permit(new ReqContext(readOnlyUser), "getComponentPageInfo", topoConf)); + assertTrue(authorizer.permit(new ReqContext(readOnlyUser), "getComponentPageInfo", + topoConf)); assertTrue(authorizer.permit(new ReqContext(userA), "getComponentPageInfo", topoConf)); assertFalse(authorizer.permit(new ReqContext(userB), "getComponentPageInfo", topoConf)); - assertFalse(authorizer.permit(new ReqContext(readOnlyUser), "uploadNewCredentials", topoConf)); + assertFalse(authorizer.permit(new ReqContext(readOnlyUser), "uploadNewCredentials", + topoConf)); assertTrue(authorizer.permit(new ReqContext(userA), "uploadNewCredentials", topoConf)); assertFalse(authorizer.permit(new ReqContext(userB), "uploadNewCredentials", topoConf)); @@ -362,13 +407,19 @@ public void SimpleACLTopologyReadOnlyUserAuthTest() { assertTrue(authorizer.permit(new ReqContext(userA), "setWorkerProfiler", topoConf)); assertFalse(authorizer.permit(new ReqContext(userB), "setWorkerProfiler", topoConf)); - assertTrue(authorizer.permit(new ReqContext(readOnlyUser), "getWorkerProfileActionExpiry", topoConf)); - assertTrue(authorizer.permit(new ReqContext(userA), "getWorkerProfileActionExpiry", topoConf)); - assertFalse(authorizer.permit(new ReqContext(userB), "getWorkerProfileActionExpiry", topoConf)); + assertTrue(authorizer.permit(new ReqContext(readOnlyUser), "getWorkerProfileActionExpiry", + topoConf)); + assertTrue(authorizer.permit(new ReqContext(userA), "getWorkerProfileActionExpiry", + topoConf)); + assertFalse(authorizer.permit(new ReqContext(userB), "getWorkerProfileActionExpiry", + topoConf)); - assertTrue(authorizer.permit(new ReqContext(readOnlyUser), "getComponentPendingProfileActions", topoConf)); - assertTrue(authorizer.permit(new ReqContext(userA), "getComponentPendingProfileActions", topoConf)); - assertFalse(authorizer.permit(new ReqContext(userB), "getComponentPendingProfileActions", topoConf)); + assertTrue(authorizer.permit(new ReqContext(readOnlyUser), + "getComponentPendingProfileActions", topoConf)); + assertTrue(authorizer.permit(new ReqContext(userA), "getComponentPendingProfileActions", + topoConf)); + assertFalse(authorizer.permit(new ReqContext(userB), "getComponentPendingProfileActions", + topoConf)); assertFalse(authorizer.permit(new ReqContext(readOnlyUser), "startProfiling", topoConf)); assertTrue(authorizer.permit(new ReqContext(userA), "startProfiling", topoConf)); @@ -406,7 +457,8 @@ public void SimpleACLTopologyReadOnlyGroupAuthTest() { SimpleACLTopologyReadOnlyGroupAuthTestMock.class.getName()); Map topoConf = new HashMap<>(); - Collection topologyReadOnlyGroupSet = new HashSet<>(Collections.singletonList("group-readonly")); + Collection topologyReadOnlyGroupSet = new HashSet<>(Collections + .singletonList("group-readonly")); topoConf.put(Config.TOPOLOGY_READONLY_GROUPS, topologyReadOnlyGroupSet); Subject userInReadOnlyGroup = createSubject("user-in-readonly-group"); @@ -415,10 +467,12 @@ public void SimpleACLTopologyReadOnlyGroupAuthTest() { IAuthorizer authorizer = new SimpleACLAuthorizer(); authorizer.prepare(clusterConf); - assertFalse(authorizer.permit(new ReqContext(userInReadOnlyGroup), "killTopology", topoConf)); + assertFalse(authorizer.permit(new ReqContext(userInReadOnlyGroup), "killTopology", + topoConf)); assertFalse(authorizer.permit(new ReqContext(userB), "killTopology", topoConf)); - assertTrue(authorizer.permit(new ReqContext(userInReadOnlyGroup), "getTopologyInfo", topoConf)); + assertTrue(authorizer.permit(new ReqContext(userInReadOnlyGroup), "getTopologyInfo", + topoConf)); assertFalse(authorizer.permit(new ReqContext(userB), "getTopologyInfo", topoConf)); } @@ -432,7 +486,8 @@ public void requestWithoutPrincipalIsDeniedWhenNimbusUsersAreConfigured() { authorizer.prepare(clusterConf); assertFalse(authorizer.permit(contextWithoutPrincipal(), "getNimbusConf", new HashMap<>())); - assertTrue(authorizer.permit(new ReqContext(createSubject("user-a")), "getNimbusConf", new HashMap<>())); + assertTrue(authorizer.permit(new ReqContext(createSubject("user-a")), "getNimbusConf", + new HashMap<>())); } @Test @@ -474,7 +529,7 @@ public static class SimpleACLTopologyReadOnlyGroupAuthTestMock implements IGroup @Override public void prepare(Map conf) { - //Ignored + // Ignored } @Override @@ -486,4 +541,4 @@ public Set getGroups(String user) { } } } -} \ No newline at end of file +} diff --git a/storm-client/test/jvm/org/apache/storm/security/auth/authorizer/SupervisorSimpleACLAuthorizerTest.java b/storm-client/test/jvm/org/apache/storm/security/auth/authorizer/SupervisorSimpleACLAuthorizerTest.java index 058727b44fa..809d53de096 100644 --- a/storm-client/test/jvm/org/apache/storm/security/auth/authorizer/SupervisorSimpleACLAuthorizerTest.java +++ b/storm-client/test/jvm/org/apache/storm/security/auth/authorizer/SupervisorSimpleACLAuthorizerTest.java @@ -1,17 +1,25 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.security.auth.authorizer; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + import java.security.Principal; import java.util.Collections; import java.util.HashMap; @@ -27,9 +35,6 @@ import org.junit.jupiter.api.condition.DisabledOnOs; import org.junit.jupiter.api.condition.OS; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - public class SupervisorSimpleACLAuthorizerTest { @Test @@ -41,14 +46,16 @@ public void requestWithoutPrincipalIsDenied() { Map topoConf = new HashMap<>(); topoConf.put(Config.TOPOLOGY_USERS, new HashSet<>(Collections.singletonList("user-a"))); - assertFalse(authorizer.permit(new ReqContext(new Subject()), "getLocalAssignmentForStorm", topoConf)); + assertFalse(authorizer.permit(new ReqContext(new Subject()), "getLocalAssignmentForStorm", + topoConf)); } @Test @DisabledOnOs(OS.WINDOWS) public void nimbusUserIsStillAllowedItsCommands() { Map clusterConf = ConfigUtils.readStormConfig(); - clusterConf.put(Config.NIMBUS_DAEMON_USERS, new HashSet<>(Collections.singletonList("nimbus-daemon"))); + clusterConf.put(Config.NIMBUS_DAEMON_USERS, new HashSet<>(Collections + .singletonList("nimbus-daemon"))); IAuthorizer authorizer = new SupervisorSimpleACLAuthorizer(); authorizer.prepare(clusterConf); diff --git a/storm-client/test/jvm/org/apache/storm/security/auth/kerberos/AutoLoginModuleTest.java b/storm-client/test/jvm/org/apache/storm/security/auth/kerberos/AutoLoginModuleTest.java index 1f570a378ec..7a4319618eb 100644 --- a/storm-client/test/jvm/org/apache/storm/security/auth/kerberos/AutoLoginModuleTest.java +++ b/storm-client/test/jvm/org/apache/storm/security/auth/kerberos/AutoLoginModuleTest.java @@ -16,25 +16,24 @@ package org.apache.storm.security.auth.kerberos; -import org.apache.storm.security.auth.kerberos.AutoTGTKrb5LoginModule; -import org.apache.storm.security.auth.kerberos.AutoTGTKrb5LoginModuleTest; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; -import javax.security.auth.Subject; -import javax.security.auth.kerberos.KerberosPrincipal; -import javax.security.auth.kerberos.KerberosTicket; -import javax.security.auth.login.LoginException; import java.net.InetAddress; import java.security.Principal; import java.text.SimpleDateFormat; import java.util.Arrays; import java.util.Collections; import java.util.Date; - -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; +import javax.security.auth.Subject; +import javax.security.auth.kerberos.KerberosPrincipal; +import javax.security.auth.kerberos.KerberosTicket; +import javax.security.auth.login.LoginException; +import org.apache.storm.security.auth.kerberos.AutoTGTKrb5LoginModule; +import org.apache.storm.security.auth.kerberos.AutoTGTKrb5LoginModuleTest; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; public class AutoLoginModuleTest { @@ -51,7 +50,8 @@ public void loginModuleNoSubjNoTgtTest() throws Exception { @Test public void loginModuleReadonlySubjNoTgtTest() throws Exception { // Behavior is correct when there is a read-only Subject and no TGT - Subject readonlySubject = new Subject(true, Collections.emptySet(), Collections.emptySet(), Collections.emptySet()); + Subject readonlySubject = new Subject(true, Collections.emptySet(), Collections.emptySet(), + Collections.emptySet()); AutoTGTKrb5LoginModule loginModule = new AutoTGTKrb5LoginModule(); loginModule.initialize(readonlySubject, null, null, null); assertFalse(loginModule.commit()); @@ -84,7 +84,8 @@ public void loginModuleNoSubjWithTgtTest() throws Exception { @Test public void loginModuleReadonlySubjWithTgtTest() throws Exception { // Behavior is correct when there is a read-only Subject and a TGT - Subject readonlySubject = new Subject(true, Collections.emptySet(), Collections.emptySet(), Collections.emptySet()); + Subject readonlySubject = new Subject(true, Collections.emptySet(), Collections.emptySet(), + Collections.emptySet()); AutoTGTKrb5LoginModuleTest loginModule = new AutoTGTKrb5LoginModuleTest(); loginModule.initialize(readonlySubject, null, null, null); loginModule.setKerbTicket(Mockito.mock(KerberosTicket.class)); @@ -102,9 +103,9 @@ public void loginModuleWithSubjAndTgt() throws Exception { loginModule.client = Mockito.mock(Principal.class); Date endTime = new SimpleDateFormat("ddMMyyyy").parse("31122030"); byte[] asn1Enc = new byte[10]; - Arrays.fill(asn1Enc, (byte)122); + Arrays.fill(asn1Enc, (byte) 122); byte[] sessionKey = new byte[10]; - Arrays.fill(sessionKey, (byte)123); + Arrays.fill(sessionKey, (byte) 123); KerberosTicket ticket = new KerberosTicket( asn1Enc, new KerberosPrincipal("client/localhost@local.com"), diff --git a/storm-client/test/jvm/org/apache/storm/security/serialization/BlowfishTupleSerializerTest.java b/storm-client/test/jvm/org/apache/storm/security/serialization/BlowfishTupleSerializerTest.java index 8ab530f0d98..e49f7a6475e 100644 --- a/storm-client/test/jvm/org/apache/storm/security/serialization/BlowfishTupleSerializerTest.java +++ b/storm-client/test/jvm/org/apache/storm/security/serialization/BlowfishTupleSerializerTest.java @@ -1,17 +1,26 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.security.serialization; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + import com.esotericsoftware.kryo.Kryo; import com.esotericsoftware.kryo.io.Input; import com.esotericsoftware.kryo.io.Output; @@ -24,9 +33,6 @@ import org.apache.storm.utils.ListDelegate; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; - public class BlowfishTupleSerializerTest { /** @@ -34,7 +40,8 @@ public class BlowfishTupleSerializerTest { */ @Test public void testConstructorThrowsOnNullKey() { - assertThrows(RuntimeException.class, () -> new BlowfishTupleSerializer(null, new HashMap<>())); + assertThrows(RuntimeException.class, () -> new BlowfishTupleSerializer(null, + new HashMap<>())); } /** @@ -44,43 +51,56 @@ public void testConstructorThrowsOnNullKey() { public void testConstructorThrowsOnInvalidKey() { // The encryption key must be hexadecimal. assertThrows(RuntimeException.class, - () -> new BlowfishTupleSerializer(null, ImmutableMap.of(BlowfishTupleSerializer.SECRET_KEY, "0123456789abcdefg"))); + () -> new BlowfishTupleSerializer(null, ImmutableMap + .of(BlowfishTupleSerializer.SECRET_KEY, "0123456789abcdefg"))); } /** - * Test using {@link org.apache.storm.security.serialization.BlowfishTupleSerializer#SECRET_KEY}. + * Test using {@link + * org.apache.storm.security.serialization.BlowfishTupleSerializer#SECRET_KEY}. */ @Test public void testUseBlowfishKey() { String arbitraryKey = "7dd6fb3203878381b08f9c89d25ed105"; - Map topoConf = ImmutableMap.of(BlowfishTupleSerializer.SECRET_KEY, arbitraryKey); + Map topoConf = ImmutableMap.of(BlowfishTupleSerializer.SECRET_KEY, + arbitraryKey); testEncryptsAndDecryptsMessage(topoConf); } /** * Test using {@link org.apache.storm.Config#STORM_ZOOKEEPER_TOPOLOGY_AUTH_PAYLOAD} - * when {@link org.apache.storm.security.serialization.BlowfishTupleSerializer#SECRET_KEY} is not present. + * when {@link org.apache.storm.security.serialization.BlowfishTupleSerializer#SECRET_KEY} is + * not present. */ @Test public void testUseZookeeperSecret() { - Map topoConf = ImmutableMap.of(Config.STORM_ZOOKEEPER_TOPOLOGY_AUTH_PAYLOAD, "user:password"); + Map topoConf = ImmutableMap.of(Config.STORM_ZOOKEEPER_TOPOLOGY_AUTH_PAYLOAD, + "user:password"); testEncryptsAndDecryptsMessage(topoConf); } /** - * Reads a string encrypted by another instance with a shared key + * Reads a string encrypted by another instance with a shared key. */ private void testEncryptsAndDecryptsMessage(Map topoConf) { - String testText = "Tetraodontidae is a family of primarily marine and estuarine fish of the order" + - " Tetraodontiformes. The family includes many familiar species, which are" + - " variously called pufferfish, puffers, balloonfish, blowfish, bubblefish," + - " globefish, swellfish, toadfish, toadies, honey toads, sugar toads, and sea" + - " squab.[1] They are morphologically similar to the closely related" + - " porcupinefish, which have large external spines (unlike the thinner, hidden" + - " spines of Tetraodontidae, which are only visible when the fish has puffed up)." + - " The scientific name refers to the four large teeth, fused into an upper and" + - " lower plate, which are used for crushing the shells of crustaceans and" + - " mollusks, their natural prey."; + String testText = + "Tetraodontidae is a family of primarily marine and estuarine fish of the order" + + " Tetraodontiformes. The family includes many familiar species, which " + + "are" + + " variously called pufferfish, puffers, balloonfish, blowfish, " + + "bubblefish," + + " globefish, swellfish, toadfish, toadies, honey toads, sugar toads, " + + "and sea" + + " squab.[1] They are morphologically similar to the closely related" + + " porcupinefish, which have large external spines (unlike the thinner, " + + "hidden" + + " spines of Tetraodontidae, which are only visible when the fish has " + + "puffed up)." + + " The scientific name refers to the four large teeth, fused into an " + + "upper and" + + " lower plate, which are used for crushing the shells of crustaceans " + + "and" + + " mollusks, their natural prey."; Kryo kryo = new Kryo(); kryo.setRegistrationRequired(false); BlowfishTupleSerializer writerBTS = new BlowfishTupleSerializer(null, topoConf); diff --git a/storm-client/test/jvm/org/apache/storm/serialization/GzipBridgeThriftSerializationDelegateTest.java b/storm-client/test/jvm/org/apache/storm/serialization/GzipBridgeThriftSerializationDelegateTest.java index 47c9147d54c..bb6b47bc8b7 100644 --- a/storm-client/test/jvm/org/apache/storm/serialization/GzipBridgeThriftSerializationDelegateTest.java +++ b/storm-client/test/jvm/org/apache/storm/serialization/GzipBridgeThriftSerializationDelegateTest.java @@ -1,25 +1,30 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.serialization; +import static org.junit.jupiter.api.Assertions.assertEquals; + import java.util.Collections; import org.apache.storm.generated.GlobalStreamId; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; - - public class GzipBridgeThriftSerializationDelegateTest { SerializationDelegate testDelegate; diff --git a/storm-client/test/jvm/org/apache/storm/serialization/KryoTupleSerializerDeserializerTest.java b/storm-client/test/jvm/org/apache/storm/serialization/KryoTupleSerializerDeserializerTest.java index bbd03697075..87bfa0b3639 100644 --- a/storm-client/test/jvm/org/apache/storm/serialization/KryoTupleSerializerDeserializerTest.java +++ b/storm-client/test/jvm/org/apache/storm/serialization/KryoTupleSerializerDeserializerTest.java @@ -1,17 +1,33 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.serialization; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.when; + import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -33,19 +49,11 @@ import org.junit.jupiter.api.Test; import org.mockito.MockedStatic; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.anyInt; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.mockStatic; -import static org.mockito.Mockito.when; - /** - * Unit tests for {@link KryoTupleSerializer} and {@link KryoTupleDeserializer}, covering the compressed and - * uncompressed code paths, round-trip fidelity, mixing both encodings through a single (de)serializer instance, + * Unit tests for {@link KryoTupleSerializer} and {@link KryoTupleDeserializer}, covering the + * compressed and + * uncompressed code paths, round-trip fidelity, mixing both encodings through a single + * (de)serializer instance, * and the negative/error paths. */ public class KryoTupleSerializerDeserializerTest { @@ -68,7 +76,8 @@ public void setup() { private StormTopology createStormTopology() { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout(SOURCE_COMPONENT, new TestWordSpout(true), 1); - builder.setBolt(DEST_COMPONENT, new TestWordCounter(), 1).fieldsGrouping(SOURCE_COMPONENT, new Fields("word")); + builder.setBolt(DEST_COMPONENT, new TestWordCounter(), 1).fieldsGrouping(SOURCE_COMPONENT, + new Fields("word")); return builder.createTopology(); } @@ -111,10 +120,12 @@ public void testRoundTripUncompressedWhenCompressionDisabled() { KryoTupleSerializer serializer = new KryoTupleSerializer(conf, context); KryoTupleDeserializer deserializer = new KryoTupleDeserializer(conf, context); - TupleImpl original = tuple(new Values("hello", 42, bigString(8192)), MessageId.makeRootId(7L, 99L)); + TupleImpl original = tuple(new Values("hello", 42, bigString(8192)), MessageId + .makeRootId(7L, 99L)); byte[] bytes = serializer.serialize(original); - assertFalse(Utils.ZstdUtils.isZstd(bytes), "compression disabled must never emit a zstd frame"); + assertFalse(Utils.ZstdUtils.isZstd(bytes), + "compression disabled must never emit a zstd frame"); assertSameTuple(original, deserializer.deserialize(bytes)); } @@ -127,7 +138,8 @@ public void testRoundTripUncompressedWhenBelowThreshold() { TupleImpl original = tuple(new Values("small", 1), MessageId.makeUnanchored()); byte[] bytes = serializer.serialize(original); - assertFalse(Utils.ZstdUtils.isZstd(bytes), "payload below threshold must not be compressed"); + assertFalse(Utils.ZstdUtils.isZstd(bytes), + "payload below threshold must not be compressed"); assertSameTuple(original, deserializer.deserialize(bytes)); } @@ -153,7 +165,8 @@ public void testCompressionAtExactThreshold() { TupleImpl original = tuple(new Values("x"), MessageId.makeUnanchored()); byte[] bytes = serializer.serialize(original); - assertTrue(Utils.ZstdUtils.isZstd(bytes), "threshold 0 must compress every non-empty payload"); + assertTrue(Utils.ZstdUtils.isZstd(bytes), + "threshold 0 must compress every non-empty payload"); assertSameTuple(original, deserializer.deserialize(bytes)); } @@ -164,7 +177,8 @@ public void testMixedCompressedAndUncompressedSameInstances() { KryoTupleDeserializer deserializer = new KryoTupleDeserializer(conf, context); TupleImpl small = tuple(new Values("tiny", 1), MessageId.makeRootId(1L, 2L)); - TupleImpl large = tuple(new Values(bigString(32 * 1024), "tail"), MessageId.makeRootId(3L, 4L)); + TupleImpl large = tuple(new Values(bigString(32 * 1024), "tail"), MessageId.makeRootId(3L, + 4L)); // Buffer reuse across compressed and uncompressed payloads. byte[] smallBytes1 = serializer.serialize(small); @@ -187,7 +201,8 @@ public void testMixedCompressedAndUncompressedSameInstances() { @Test public void testComponentLevelCompressionEnablesDecompressPath() { enableComponentLevelCompression(SOURCE_COMPONENT); - KryoTupleSerializer serializer = new KryoTupleSerializer(compressionEnabledConf(0), context); + KryoTupleSerializer serializer = new KryoTupleSerializer(compressionEnabledConf(0), + context); KryoTupleDeserializer deserializer = new KryoTupleDeserializer(baseConf(), context); TupleImpl original = tuple(new Values(bigString(16 * 1024)), MessageId.makeRootId(8L, 9L)); @@ -199,56 +214,72 @@ public void testComponentLevelCompressionEnablesDecompressPath() { @Test public void testDecompressPathGatedPerTopology() { - // Two distinct topologies sharing the exact same compressed frame on the wire. The gating decision is - // per-topology (workers are per-topology), so the same bytes must be decompressed by one and not the other. + // Two distinct topologies sharing the exact same compressed frame on the wire. The gating + // decision is + // per-topology (workers are per-topology), so the same bytes must be decompressed by one + // and not the other. GeneralTopologyContext compressingTopology = newContext(); - enableComponentLevelCompression(compressingTopology, SOURCE_COMPONENT); // at least one component compresses - GeneralTopologyContext plainTopology = newContext(); // no component enables compression + enableComponentLevelCompression(compressingTopology, + SOURCE_COMPONENT); // at least one component compresses + GeneralTopologyContext plainTopology = + newContext(); // no component enables compression - KryoTupleSerializer serializer = new KryoTupleSerializer(compressionEnabledConf(0), compressingTopology); + KryoTupleSerializer serializer = new KryoTupleSerializer(compressionEnabledConf(0), + compressingTopology); TupleImpl original = tuple(new Values(bigString(16 * 1024)), MessageId.makeRootId(8L, 9L)); byte[] bytes = serializer.serialize(original); - assertTrue(Utils.ZstdUtils.isZstd(bytes), "precondition: serializer produced a compressed frame"); + assertTrue(Utils.ZstdUtils.isZstd(bytes), + "precondition: serializer produced a compressed frame"); - KryoTupleDeserializer compressingDeser = new KryoTupleDeserializer(baseConf(), compressingTopology); + KryoTupleDeserializer compressingDeser = new KryoTupleDeserializer(baseConf(), + compressingTopology); KryoTupleDeserializer plainDeser = new KryoTupleDeserializer(baseConf(), plainTopology); // Compressing topology: decompress path is taken, the frame round-trips. assertSameTuple(original, compressingDeser.deserialize(bytes)); - // Plain topology: decompress path is skipped, the frame is treated as a raw kryo tuple and fails to parse. + // Plain topology: decompress path is skipped, the frame is treated as a raw kryo tuple and + // fails to parse. assertThrows(RuntimeException.class, () -> plainDeser.deserialize(bytes)); } @Test public void testNoComponentCompressionSkipsDecompressPath() { - KryoTupleSerializer serializer = new KryoTupleSerializer(compressionEnabledConf(0), context); + KryoTupleSerializer serializer = new KryoTupleSerializer(compressionEnabledConf(0), + context); KryoTupleDeserializer deserializer = new KryoTupleDeserializer(baseConf(), context); - byte[] bytes = serializer.serialize(tuple(new Values(bigString(16 * 1024)), MessageId.makeRootId(8L, 9L))); - assertTrue(Utils.ZstdUtils.isZstd(bytes), "precondition: serializer produced a compressed frame"); + byte[] bytes = serializer.serialize(tuple(new Values(bigString(16 * 1024)), MessageId + .makeRootId(8L, 9L))); + assertTrue(Utils.ZstdUtils.isZstd(bytes), + "precondition: serializer produced a compressed frame"); assertThrows(RuntimeException.class, () -> deserializer.deserialize(bytes)); } @Test public void testDecompressFailureFallsBackToRawTupleParsing() { - // Exercises the false-positive fallback in KryoTupleDeserializer#deserialize: compression is enabled and - // isZstd() reports a match, but the decompress path surfaces a "Failed to deserialize tuple" error. + // Exercises the false-positive fallback in KryoTupleDeserializer#deserialize: compression + // is enabled and + // isZstd() reports a match, but the decompress path surfaces a "Failed to deserialize + // tuple" error. enableComponentLevelCompression(SOURCE_COMPONENT); // anyTupleCompressionEnabled == true KryoTupleSerializer serializer = new KryoTupleSerializer(baseConf(), context); KryoTupleDeserializer deserializer = new KryoTupleDeserializer(baseConf(), context); TupleImpl original = tuple(new Values("hello", 42), MessageId.makeRootId(5L, 11L)); byte[] raw = serializer.serialize(original); - assertFalse(Utils.ZstdUtils.isZstd(raw), "precondition: serializer produced a raw, uncompressed tuple"); + assertFalse(Utils.ZstdUtils.isZstd(raw), + "precondition: serializer produced a raw, uncompressed tuple"); try (MockedStatic mocked = mockStatic(Utils.ZstdUtils.class)) { - // Force entry into the decompress branch, then make decompression report a tuple-deserialization failure. + // Force entry into the decompress branch, then make decompression report a + // tuple-deserialization failure. mocked.when(() -> Utils.ZstdUtils.isZstd(raw)).thenReturn(true); mocked.when(() -> Utils.ZstdUtils.decompress(eq(raw), anyInt())) .thenThrow(new RuntimeException(KryoTupleDeserializer.FAILED_TO_DESERIALIZE_TUPLE)); TupleImpl result = deserializer.deserialize(raw); - assertSameTuple(original, result); // fallback re-parsed the raw bytes and recovered the tuple + assertSameTuple(original, + result); // fallback re-parsed the raw bytes and recovered the tuple } } @@ -312,26 +343,33 @@ public void testDeserializeEmptyBytesThrows() { public void testDeserializeZstdMagicButInvalidFrameThrows() { enableComponentLevelCompression(SOURCE_COMPONENT); KryoTupleDeserializer deserializer = new KryoTupleDeserializer(baseConf(), context); - // First 4 bytes are the little-endian zstd magic 0xFD2FB528 so isZstd() is true, but the rest is - // not a valid zstd frame. decompress() throws, the false-positive fallback re-parses the raw bytes, + // First 4 bytes are the little-endian zstd magic 0xFD2FB528 so isZstd() is true, but the + // rest is + // not a valid zstd frame. decompress() throws, the false-positive fallback re-parses the + // raw bytes, // which are also not a valid kryo tuple -> RuntimeException. - byte[] fakeZstd = new byte[]{(byte) 0x28, (byte) 0xB5, (byte) 0x2F, (byte) 0xFD, 0x00, 0x01, 0x02, 0x03}; + byte[] fakeZstd = + new byte[]{(byte) 0x28, (byte) 0xB5, (byte) 0x2F, (byte) 0xFD, 0x00, 0x01, 0x02, 0x03}; assertTrue(Utils.ZstdUtils.isZstd(fakeZstd)); assertThrows(RuntimeException.class, () -> deserializer.deserialize(fakeZstd)); } @Test public void testDeserializeCompressedExceedingMaxDecompressedBytesThrows() { - // A genuinely compressed tuple, but the deserializer is configured with a tiny decompression cap. - // decompress() throws ("threshold exceeded"), the fallback re-parses the still-compressed raw bytes, + // A genuinely compressed tuple, but the deserializer is configured with a tiny + // decompression cap. + // decompress() throws ("threshold exceeded"), the fallback re-parses the still-compressed + // raw bytes, // which are not a valid kryo tuple -> RuntimeException. enableComponentLevelCompression(SOURCE_COMPONENT); - KryoTupleSerializer serializer = new KryoTupleSerializer(compressionEnabledConf(0), context); + KryoTupleSerializer serializer = new KryoTupleSerializer(compressionEnabledConf(0), + context); Map deserConf = baseConf(); deserConf.put(Config.TOPOLOGY_TUPLE_COMPRESSION_MAX_DECOMPRESSED_BYTES, 1); KryoTupleDeserializer deserializer = new KryoTupleDeserializer(deserConf, context); - byte[] bytes = serializer.serialize(tuple(new Values(bigString(8192)), MessageId.makeUnanchored())); + byte[] bytes = serializer.serialize(tuple(new Values(bigString(8192)), MessageId + .makeUnanchored())); assertTrue(Utils.ZstdUtils.isZstd(bytes)); assertThrows(RuntimeException.class, () -> deserializer.deserialize(bytes)); } @@ -352,7 +390,8 @@ private static class UnregisteredType { } private TupleImpl tuple(List values, MessageId id) { - return new TupleImpl(context, values, SOURCE_COMPONENT, SOURCE_TASK_ID, Utils.DEFAULT_STREAM_ID, id); + return new TupleImpl(context, values, SOURCE_COMPONENT, SOURCE_TASK_ID, + Utils.DEFAULT_STREAM_ID, id); } private void assertSameTuple(TupleImpl expected, TupleImpl actual) { diff --git a/storm-client/test/jvm/org/apache/storm/serialization/SerializableSerializerFilterTest.java b/storm-client/test/jvm/org/apache/storm/serialization/SerializableSerializerFilterTest.java index 4c4732ea65a..e0be4a05964 100644 --- a/storm-client/test/jvm/org/apache/storm/serialization/SerializableSerializerFilterTest.java +++ b/storm-client/test/jvm/org/apache/storm/serialization/SerializableSerializerFilterTest.java @@ -1,17 +1,29 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.serialization; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertIterableEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + import com.esotericsoftware.kryo.KryoException; import com.esotericsoftware.kryo.io.Input; import com.esotericsoftware.kryo.io.Output; @@ -39,17 +51,15 @@ import org.apache.storm.utils.Utils; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertIterableEquals; -import static org.junit.jupiter.api.Assertions.assertSame; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; - /** - * Tests for the JEP-290 serial filter ({@link Config#TOPOLOGY_FALL_BACK_ON_JAVA_SERIALIZATION_FILTER}) protecting the - * java-serialization fallback bridge. Round-trip cases exercise an actual pass through the bridge via KryoValuesSerializer and - * KryoValuesDeserializer end to end, using JDK classes only (no fixtures under third-party package names). Filter semantics - * that a round-trip cannot express (merging, limit tightening) are asserted through checkInput with synthetic FilterInfos + * Tests for the JEP-290 serial filter ({@link + * Config#TOPOLOGY_FALL_BACK_ON_JAVA_SERIALIZATION_FILTER}) protecting the + * java-serialization fallback bridge. Round-trip cases exercise an actual pass through the bridge + * via KryoValuesSerializer and + * KryoValuesDeserializer end to end, using JDK classes only (no fixtures under third-party package + * names). Filter semantics + * that a round-trip cannot express (merging, limit tightening) are asserted through checkInput with + * synthetic FilterInfos * (rejections only). */ public class SerializableSerializerFilterTest { @@ -58,11 +68,23 @@ public class SerializableSerializerFilterTest { private static final long SAMPLE_MAX_BYTES = 10485760L; /** - * The sample filter pattern documented in docs/SECURITY.md: a deny-list of well-known gadget namespaces plus - * depth/reference/array/byte limits. Only entries whose classes exist on a plain JDK classpath are asserted against real - * Class objects; the rest are covered by the parse (createFilter) and by the doc-sync test below. + * The sample filter pattern documented in docs/SECURITY.md: a deny-list of well-known gadget + * namespaces plus + * depth/reference/array/byte limits. Only entries whose classes exist on a plain JDK classpath + * are asserted against real + * Class objects; the rest are covered by the parse (createFilter) and by the doc-sync test + * below. */ - private static final String SAMPLE_PATTERN = "!org.apache.commons.collections.functors.*;!org.apache.commons.collections.comparators.*;!org.apache.commons.collections4.functors.*;!org.apache.commons.collections4.comparators.*;!org.apache.commons.beanutils.*;!org.apache.xalan.xsltc.trax.*;!com.sun.org.apache.xalan.internal.**;!com.sun.rowset.*;!com.sun.org.apache.rowset.internal.*;!com.mchange.v2.c3p0.**;!org.codehaus.groovy.runtime.ConvertedClosure;!org.codehaus.groovy.runtime.MethodClosure;!javax.management.BadAttributeValueExpException;!sun.reflect.annotation.AnnotationInvocationHandler;!com.sun.jndi.**;!java.rmi.**;!clojure.**;!org.apache.commons.fileupload.**;!bsh.**;!org.python.**;!org.jboss.**;maxdepth=64;maxrefs=2097152;maxarray=1048576;maxbytes=10485760"; + private static final String SAMPLE_PATTERN = + "!org.apache.commons.collections.functors.*;!org.apache.commons.collections.comparator" + + "s.*;!org.apache.commons.collections4.functors.*;!org.apache.commons.collections4.co" + + "mparators.*;!org.apache.commons.beanutils.*;!org.apache.xalan.xsltc.trax.*;!com.sun" + + ".org.apache.xalan.internal.**;!com.sun.rowset.*;!com.sun.org.apache.rowset.internal" + + ".*;!com.mchange.v2.c3p0.**;!org.codehaus.groovy.runtime.ConvertedClosure;!org.codeh" + + "aus.groovy.runtime.MethodClosure;!javax.management.BadAttributeValueExpException;!s" + + "un.reflect.annotation.AnnotationInvocationHandler;!com.sun.jndi.**;!java.rmi.**;!cl" + + "ojure.**;!org.apache.commons.fileupload.**;!bsh.**;!org.python.**;!org.jboss.**;max" + + "depth=64;maxrefs=2097152;maxarray=1048576;maxbytes=10485760"; /** * Minimal conf that routes unregistered classes through the java-serialization fallback bridge. {@code filterSpec == null} @@ -83,16 +105,19 @@ private Map bridgeConf(String filterSpec) { private Object roundTrip(Map conf, Object value) { KryoValuesSerializer serializer = new KryoValuesSerializer(conf); KryoValuesDeserializer deserializer = new KryoValuesDeserializer(conf); - return deserializer.deserialize(serializer.serialize(Collections.singletonList(value))).get(0); + return deserializer.deserialize(serializer.serialize(Collections.singletonList(value))) + .get(0); } /** Serializes {@code value} and asserts that reading it back fails with a JEP-290 rejection in the cause chain. */ private void assertRejectedOnRead(Map conf, Object value) { KryoValuesSerializer serializer = new KryoValuesSerializer(conf); KryoValuesDeserializer deserializer = new KryoValuesDeserializer(conf); - // Writing is plain java serialization (filters apply to deserialization only), so this must succeed. + // Writing is plain java serialization (filters apply to deserialization only), so this must + // succeed. byte[] bytes = serializer.serialize(Collections.singletonList(value)); - RuntimeException ex = assertThrows(RuntimeException.class, () -> deserializer.deserialize(bytes)); + RuntimeException ex = assertThrows(RuntimeException.class, () -> deserializer + .deserialize(bytes)); assertTrue(Utils.exceptionCauseIsInstanceOf(InvalidClassException.class, ex), "expected the JEP-290 filter rejection in the cause chain, got: " + ex); } @@ -148,26 +173,31 @@ public void testFilterAllowsNonDeniedClassesRoundTrip() { HashMap hashMap = new HashMap<>(Collections.singletonMap("one", 1)); assertEquals(hashMap, roundTrip(conf, hashMap)); - // ArrayDeque is unregistered and Serializable, so it travels through the java-serialization bridge itself. + // ArrayDeque is unregistered and Serializable, so it travels through the java-serialization + // bridge itself. ArrayDeque deque = new ArrayDeque<>(Arrays.asList("a", "b", "c")); assertIterableEquals(deque, (Iterable) roundTrip(conf, deque)); } @Test public void testUnsetFilterKeyKeepsUnfilteredBehavior() { - // No filter key in the conf at all: PriorityQueue must round-trip like it did before the filter existed. + // No filter key in the conf at all: PriorityQueue must round-trip like it did before the + // filter existed. PriorityQueue original = new PriorityQueue<>(Arrays.asList(5, 4, 6)); assertIterableEquals(original, (Iterable) roundTrip(bridgeConf(null), original)); } @Test public void testWildcardDepthCoversDirectMembersAndSubpackages() { - // '.*' denies direct package members only: PriorityQueue (member of java.util) is rejected... + // '.*' denies direct package members only: PriorityQueue (member of java.util) is + // rejected... Map shallow = bridgeConf("!java.util.*"); assertRejectedOnRead(shallow, new PriorityQueue<>(Arrays.asList(3, 1, 2))); - // ...while classes in subpackages keep round-tripping through the bridge: java.util.regex.Pattern and - // java.util.logging.Level are unregistered, non-trivial, Serializable, and their java-serialized graphs stay + // ...while classes in subpackages keep round-tripping through the bridge: + // java.util.regex.Pattern and + // java.util.logging.Level are unregistered, non-trivial, Serializable, and their + // java-serialized graphs stay // inside java.lang for fields, so the pass/fail outcome is decided by their own package. Pattern compiled = (Pattern) roundTrip(shallow, Pattern.compile("bridge-wildcard-probe")); assertEquals("bridge-wildcard-probe", compiled.pattern()); @@ -181,38 +211,48 @@ public void testWildcardDepthCoversDirectMembersAndSubpackages() { @Test public void testInvalidPatternFailsFastAtKryoConstruction() { - // The parser only rejects a few inputs: '!' (no pattern) and a non-numeric maxbytes; malformed class patterns + // The parser only rejects a few inputs: '!' (no pattern) and a non-numeric maxbytes; + // malformed class patterns // are ignored, not rejected. for (String invalid : Arrays.asList("!", "maxbytes=not-a-number")) { Map conf = bridgeConf(invalid); - IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, () -> new KryoValuesSerializer(conf)); - assertTrue(ex.getMessage().contains(Config.TOPOLOGY_FALL_BACK_ON_JAVA_SERIALIZATION_FILTER), + IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, + () -> new KryoValuesSerializer(conf)); + assertTrue(ex.getMessage() + .contains(Config.TOPOLOGY_FALL_BACK_ON_JAVA_SERIALIZATION_FILTER), "error must name the offending config key: " + ex.getMessage()); } } @Test public void testSamplePatternRejectsDenyListedJdkClasses() { - // Parsing the full sample also syntax-checks the third-party gadget entries, whose classes are not on the + // Parsing the full sample also syntax-checks the third-party gadget entries, whose classes + // are not on the // classpath and thus cannot be asserted as Class objects. ObjectInputFilter sample = ObjectInputFilter.Config.createFilter(SAMPLE_PATTERN); - assertEquals(ObjectInputFilter.Status.REJECTED, sample.checkInput(info(BadAttributeValueExpException.class))); - assertEquals(ObjectInputFilter.Status.REJECTED, sample.checkInput(info(java.rmi.MarshalledObject.class))); + assertEquals(ObjectInputFilter.Status.REJECTED, sample + .checkInput(info(BadAttributeValueExpException.class))); + assertEquals(ObjectInputFilter.Status.REJECTED, sample + .checkInput(info(java.rmi.MarshalledObject.class))); } @Test public void testSecurityDocCarriesTheSamplePattern() throws IOException { - // Surefire runs with the module directory as working directory, so the repo-root docs are one level up. + // Surefire runs with the module directory as working directory, so the repo-root docs are + // one level up. Path securityDoc = Paths.get("..", "docs", "SECURITY.md").toAbsolutePath().normalize(); assertTrue(Files.exists(securityDoc), "docs/SECURITY.md not found at " + securityDoc); String doc = Files.readString(securityDoc, StandardCharsets.UTF_8); - assertTrue(doc.contains(SAMPLE_PATTERN), "docs/SECURITY.md must carry this test's sample pattern verbatim"); + assertTrue(doc.contains(SAMPLE_PATTERN), + "docs/SECURITY.md must carry this test's sample pattern verbatim"); } @Test public void testSamplePatternEnforcesMaxBytesLimit() { - // ~11MB of heap churn per run: the payload must exceed the pattern's maxbytes for the cumulative limit to bite - // mid-deserialization (every array read re-invokes the filter, so many small arrays make streamBytes add up). + // ~11MB of heap churn per run: the payload must exceed the pattern's maxbytes for the + // cumulative limit to bite + // mid-deserialization (every array read re-invokes the filter, so many small arrays make + // streamBytes add up). Map conf = bridgeConf(SAMPLE_PATTERN); ArrayDeque big = new ArrayDeque<>(); for (int i = 0; i < 11000; i++) { @@ -221,16 +261,19 @@ public void testSamplePatternEnforcesMaxBytesLimit() { KryoValuesSerializer serializer = new KryoValuesSerializer(conf); KryoValuesDeserializer deserializer = new KryoValuesDeserializer(conf); byte[] bytes = serializer.serialize(Collections.singletonList(big)); - assertTrue(bytes.length > SAMPLE_MAX_BYTES, "payload must exceed the maxbytes limit, was " + bytes.length); + assertTrue(bytes.length > SAMPLE_MAX_BYTES, "payload must exceed the maxbytes limit, was " + + bytes.length); - RuntimeException ex = assertThrows(RuntimeException.class, () -> deserializer.deserialize(bytes)); + RuntimeException ex = assertThrows(RuntimeException.class, () -> deserializer + .deserialize(bytes)); assertTrue(Utils.exceptionCauseIsInstanceOf(InvalidClassException.class, ex), "expected the maxbytes rejection in the cause chain, got: " + ex); } @Test public void testMaxArrayLimitRejectsOversizedArray() { - // One big array can ride past the byte cap, because an array passes the filter before its contents are read; + // One big array can ride past the byte cap, because an array passes the filter before its + // contents are read; // maxarray is what bounds the allocation itself. Map conf = bridgeConf("maxarray=1024"); ArrayDeque payload = new ArrayDeque<>(); @@ -240,8 +283,10 @@ public void testMaxArrayLimitRejectsOversizedArray() { @Test public void testOversizedDeclaredLengthRejectedOnBufferedInput() { - // A fixed-width prefix (Output.writeInt and Input.readInt are symmetric 4-byte reads) declares a thousand - // bytes where only one follows; the mismatch fails up front, before the new byte[len] allocation can balloon. + // A fixed-width prefix (Output.writeInt and Input.readInt are symmetric 4-byte reads) + // declares a thousand + // bytes where only one follows; the mismatch fails up front, before the new byte[len] + // allocation can balloon. Output out = new Output(16); out.writeInt(1000); out.writeByte(0); @@ -254,8 +299,10 @@ public void testOversizedDeclaredLengthRejectedOnBufferedInput() { @Test public void testStreamBackedInputIsExemptFromUpperBoundGuard() throws IOException { - // Stream-backed input is not length-checked: at prefix-read time the stream may have delivered only part of - // the value, with the rest still arriving, so an upper bound there would reject well-formed input and only + // Stream-backed input is not length-checked: at prefix-read time the stream may have + // delivered only part of + // the value, with the rest still arriving, so an upper bound there would reject well-formed + // input and only // the negative-length check applies. ByteArrayOutputStream bos = new ByteArrayOutputStream(); try (ObjectOutputStream oos = new ObjectOutputStream(bos)) { @@ -266,36 +313,45 @@ public void testStreamBackedInputIsExemptFromUpperBoundGuard() throws IOExceptio Output out = new Output(4096, Integer.MAX_VALUE); out.writeInt(payload.length); out.writeBytes(payload); - // Small buffer on purpose: after the 4-byte prefix the buffer holds fewer bytes than declared. + // Small buffer on purpose: after the 4-byte prefix the buffer holds fewer bytes than + // declared. Input streamBacked = new Input(new ByteArrayInputStream(out.toBytes()), 1024); - byte[] result = (byte[]) new SerializableSerializer().read(null, streamBacked, Object.class); - // The declared length counted the java-serialization framing; the object that comes back is the original array. + byte[] result = (byte[]) new SerializableSerializer().read(null, streamBacked, + Object.class); + // The declared length counted the java-serialization framing; the object that comes back is + // the original array. assertEquals(100000, result.length); } @Test public void testNegativeDeclaredLengthRejected() { - // An all-ones 4-byte length prefix decodes as -1: no legitimate writer produces a negative length, buffered or streamed. + // An all-ones 4-byte length prefix decodes as -1: no legitimate writer produces a negative + // length, buffered or streamed. SerializableSerializer serializer = new SerializableSerializer(); KryoException ex = assertThrows(KryoException.class, - () -> serializer.read(null, new Input(new byte[]{(byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF}), + () -> serializer.read(null, new Input(new byte[]{(byte) 0xFF, (byte) 0xFF, + (byte) 0xFF, (byte) 0xFF}), Object.class)); - assertTrue(ex.getMessage().contains("-1"), "error should name the negative length, got: " + ex.getMessage()); + assertTrue(ex.getMessage().contains("-1"), "error should name the negative length, got: " + + ex.getMessage()); } @Test public void testMergeWithExistingReturnsConfiguredWhenNoExistingFilter() { - ObjectInputFilter configured = ObjectInputFilter.Config.createFilter("!java.util.PriorityQueue"); + ObjectInputFilter configured = ObjectInputFilter.Config + .createFilter("!java.util.PriorityQueue"); assertSame(configured, SerializableSerializer.mergeWithExisting(configured, null)); } @Test public void testMergeWithExistingRejectsClassDeniedByEitherFilter() { - ObjectInputFilter configured = ObjectInputFilter.Config.createFilter("!java.util.PriorityQueue"); + ObjectInputFilter configured = ObjectInputFilter.Config + .createFilter("!java.util.PriorityQueue"); ObjectInputFilter existing = ObjectInputFilter.Config.createFilter("!java.util.ArrayDeque"); ObjectInputFilter merged = SerializableSerializer.mergeWithExisting(configured, existing); // The configured filter's denial survives the merge... - assertEquals(ObjectInputFilter.Status.REJECTED, merged.checkInput(info(PriorityQueue.class))); + assertEquals(ObjectInputFilter.Status.REJECTED, merged + .checkInput(info(PriorityQueue.class))); // ...and the existing (e.g. JVM-wide) filter's denial is not replaced by it. assertEquals(ObjectInputFilter.Status.REJECTED, merged.checkInput(info(ArrayDeque.class))); } @@ -305,7 +361,8 @@ public void testMergeWithExistingEnforcesTighterLimit() { ObjectInputFilter configured = ObjectInputFilter.Config.createFilter("maxarray=1000"); ObjectInputFilter existing = ObjectInputFilter.Config.createFilter("maxarray=10"); ObjectInputFilter merged = SerializableSerializer.mergeWithExisting(configured, existing); - // 500 fits the configured limit but exceeds the existing one; the merge keeps the tighter bound. + // 500 fits the configured limit but exceeds the existing one; the merge keeps the tighter + // bound. assertEquals(ObjectInputFilter.Status.REJECTED, merged.checkInput(info(byte[].class, 500))); } } diff --git a/storm-client/test/jvm/org/apache/storm/serialization/SerializationFactoryTest.java b/storm-client/test/jvm/org/apache/storm/serialization/SerializationFactoryTest.java index d7e7bfcd43b..05dc9508d5f 100644 --- a/storm-client/test/jvm/org/apache/storm/serialization/SerializationFactoryTest.java +++ b/storm-client/test/jvm/org/apache/storm/serialization/SerializationFactoryTest.java @@ -1,17 +1,25 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.serialization; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + import com.esotericsoftware.kryo.Kryo; import java.util.Map; import org.apache.storm.Config; @@ -20,9 +28,6 @@ import org.apache.storm.utils.Utils; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; - public class SerializationFactoryTest { @Test @@ -58,7 +63,8 @@ public void test_registers_when_valid_class_name() { @Test public void test_registers_decorator_with_conf() { Map conf = Utils.readDefaultConfig(); - conf.put(Config.TOPOLOGY_KRYO_DECORATORS, java.util.Collections.singletonList(MockKryoDecorator.class.getName())); + conf.put(Config.TOPOLOGY_KRYO_DECORATORS, java.util.Collections + .singletonList(MockKryoDecorator.class.getName())); conf.put("test_key", "test_value"); Kryo kryo = SerializationFactory.getKryo(conf); assertEquals(com.esotericsoftware.kryo.serializers.DefaultSerializers.StringSerializer.class, @@ -69,7 +75,9 @@ public static class MockKryoDecorator implements IKryoDecorator { @Override public void decorate(Kryo k, Map conf) { if ("test_value".equals(conf.get("test_key"))) { - k.register(MockSerObject.class, new com.esotericsoftware.kryo.serializers.DefaultSerializers.StringSerializer()); + k.register(MockSerObject.class, + new com.esotericsoftware.kryo.serializers.DefaultSerializers + .StringSerializer()); } } } diff --git a/storm-client/test/jvm/org/apache/storm/serialization/ThriftBridgeSerializationDelegateTest.java b/storm-client/test/jvm/org/apache/storm/serialization/ThriftBridgeSerializationDelegateTest.java index f986ccc42c3..1de2771c08a 100644 --- a/storm-client/test/jvm/org/apache/storm/serialization/ThriftBridgeSerializationDelegateTest.java +++ b/storm-client/test/jvm/org/apache/storm/serialization/ThriftBridgeSerializationDelegateTest.java @@ -1,24 +1,29 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.serialization; +import static org.junit.jupiter.api.Assertions.assertEquals; + import org.apache.storm.generated.ErrorInfo; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; - - public class ThriftBridgeSerializationDelegateTest { SerializationDelegate testDelegate; diff --git a/storm-client/test/jvm/org/apache/storm/serialization/ZstdBridgeThriftSerializationDelegateRoundTripTest.java b/storm-client/test/jvm/org/apache/storm/serialization/ZstdBridgeThriftSerializationDelegateRoundTripTest.java index eac87d844ee..976254ae143 100644 --- a/storm-client/test/jvm/org/apache/storm/serialization/ZstdBridgeThriftSerializationDelegateRoundTripTest.java +++ b/storm-client/test/jvm/org/apache/storm/serialization/ZstdBridgeThriftSerializationDelegateRoundTripTest.java @@ -1,12 +1,17 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/test/jvm/org/apache/storm/serialization/ZstdBridgeThriftSerializationDelegateTest.java b/storm-client/test/jvm/org/apache/storm/serialization/ZstdBridgeThriftSerializationDelegateTest.java index 13bf7515cc8..d8ca55f8a68 100644 --- a/storm-client/test/jvm/org/apache/storm/serialization/ZstdBridgeThriftSerializationDelegateTest.java +++ b/storm-client/test/jvm/org/apache/storm/serialization/ZstdBridgeThriftSerializationDelegateTest.java @@ -1,12 +1,17 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -36,7 +41,6 @@ import org.mockito.MockedStatic; import org.mockito.junit.jupiter.MockitoExtension; - @ExtendWith(MockitoExtension.class) class ZstdBridgeThriftSerializationDelegateTest { @@ -50,20 +54,22 @@ class ZstdBridgeThriftSerializationDelegateTest { private static final Map TOPO_CONF = Collections.emptyMap(); - private static final byte[] ZSTD_BYTES = {(byte) 0x28, (byte) 0xB5, (byte) 0x2F, (byte) 0xFD, 0x00}; + private static final byte[] ZSTD_BYTES = + {(byte) 0x28, (byte) 0xB5, (byte) 0x2F, (byte) 0xFD, 0x00}; private static final byte[] PLAIN_BYTES = {0x00, 0x01, 0x02, 0x03, 0x04}; private static final byte[] RESULT_BYTES = {(byte) 0xAA, (byte) 0xBB}; - @BeforeEach void setUp() throws Exception { delegate = new ZstdBridgeThriftSerializationDelegate(); - Field defaultField = ZstdBridgeThriftSerializationDelegate.class.getDeclaredField("defaultDelegate"); + Field defaultField = ZstdBridgeThriftSerializationDelegate.class + .getDeclaredField("defaultDelegate"); defaultField.setAccessible(true); defaultField.set(delegate, defaultDelegate); - Field zstdField = ZstdBridgeThriftSerializationDelegate.class.getDeclaredField("zstdDelegate"); + Field zstdField = ZstdBridgeThriftSerializationDelegate.class + .getDeclaredField("zstdDelegate"); zstdField.setAccessible(true); zstdField.set(delegate, zstdDelegate); } diff --git a/storm-client/test/jvm/org/apache/storm/serialization/ZstdThriftSerializationDelegateTest.java b/storm-client/test/jvm/org/apache/storm/serialization/ZstdThriftSerializationDelegateTest.java index 24faf86e153..defcdc29cb4 100644 --- a/storm-client/test/jvm/org/apache/storm/serialization/ZstdThriftSerializationDelegateTest.java +++ b/storm-client/test/jvm/org/apache/storm/serialization/ZstdThriftSerializationDelegateTest.java @@ -1,12 +1,17 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -133,8 +138,8 @@ void deserialize_corruptedBytes_throwsRuntimeException() { byte[] garbage = {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07}; RuntimeException ex = assertThrows(RuntimeException.class, () -> delegate.deserialize(garbage, StormTopology.class)); - assertEquals("Cannot deserialize [" +StormTopology.class.getSimpleName() + "]. " + - "Expected zstd compressed bytes, but received unknown format.", ex.getMessage()); + assertEquals("Cannot deserialize [" + StormTopology.class.getSimpleName() + "]. " + + "Expected zstd compressed bytes, but received unknown format.", ex.getMessage()); } @Test diff --git a/storm-client/test/jvm/org/apache/storm/spout/CheckpointSpoutTest.java b/storm-client/test/jvm/org/apache/storm/spout/CheckpointSpoutTest.java index b3e77c756e0..a1b1ef4c484 100644 --- a/storm-client/test/jvm/org/apache/storm/spout/CheckpointSpoutTest.java +++ b/storm-client/test/jvm/org/apache/storm/spout/CheckpointSpoutTest.java @@ -1,17 +1,27 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.spout; +import static org.apache.storm.spout.CheckPointState.Action; +import static org.apache.storm.spout.CheckPointState.State.COMMITTED; +import static org.junit.jupiter.api.Assertions.assertEquals; + import java.util.HashMap; import java.util.Map; import org.apache.storm.Config; @@ -25,12 +35,8 @@ import org.mockito.ArgumentCaptor; import org.mockito.Mockito; -import static org.apache.storm.spout.CheckPointState.Action; -import static org.apache.storm.spout.CheckPointState.State.COMMITTED; -import static org.junit.jupiter.api.Assertions.assertEquals; - /** - * Unit test for {@link CheckpointSpout} + * Unit test for {@link CheckpointSpout}. */ public class CheckpointSpoutTest { CheckpointSpout spout = new CheckpointSpout(); @@ -100,7 +106,8 @@ public void testPrepare() { public void testPrepareWithFail() { Map topoConf = new HashMap<>(); KeyValueState state = - (KeyValueState) StateFactory.getState("__state", topoConf, mockTopologyContext); + (KeyValueState) StateFactory.getState("__state", topoConf, + mockTopologyContext); CheckPointState txState = new CheckPointState(-1, COMMITTED); state.put("__state", txState); @@ -176,7 +183,8 @@ public void testRecoveryRollback() { Map topoConf = new HashMap<>(); KeyValueState state = - (KeyValueState) StateFactory.getState("test-1", topoConf, mockTopologyContext); + (KeyValueState) StateFactory.getState("test-1", topoConf, + mockTopologyContext); CheckPointState checkPointState = new CheckPointState(100, CheckPointState.State.PREPARING); state.put("__state", checkPointState); @@ -202,7 +210,8 @@ public void testRecoveryRollbackAck() { Map topoConf = new HashMap<>(); KeyValueState state = - (KeyValueState) StateFactory.getState("test-1", topoConf, mockTopologyContext); + (KeyValueState) StateFactory.getState("test-1", topoConf, + mockTopologyContext); CheckPointState checkPointState = new CheckPointState(100, CheckPointState.State.PREPARING); state.put("__state", checkPointState); @@ -232,9 +241,11 @@ public void testRecoveryCommit() { Map topoConf = new HashMap<>(); KeyValueState state = - (KeyValueState) StateFactory.getState("test-1", topoConf, mockTopologyContext); + (KeyValueState) StateFactory.getState("test-1", topoConf, + mockTopologyContext); - CheckPointState checkPointState = new CheckPointState(100, CheckPointState.State.COMMITTING); + CheckPointState checkPointState = new CheckPointState(100, + CheckPointState.State.COMMITTING); state.put("__state", checkPointState); spout.open(mockTopologyContext, mockOutputCollector, 0, state); ArgumentCaptor stream = ArgumentCaptor.forClass(String.class); diff --git a/storm-client/test/jvm/org/apache/storm/state/BaseBinaryStateIteratorTest.java b/storm-client/test/jvm/org/apache/storm/state/BaseBinaryStateIteratorTest.java index d9b82958cb9..ac1f269b2e3 100644 --- a/storm-client/test/jvm/org/apache/storm/state/BaseBinaryStateIteratorTest.java +++ b/storm-client/test/jvm/org/apache/storm/state/BaseBinaryStateIteratorTest.java @@ -1,17 +1,26 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.state; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + import java.util.Arrays; import java.util.Iterator; import java.util.Map; @@ -21,10 +30,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertArrayEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - /** * Unit tests for {@link BaseBinaryStateIterator}. */ @@ -116,7 +121,8 @@ public void testGetEntryNotAvailable() { assertFalse(kvIterator.hasNext()); } - private void assertNextEntry(BaseBinaryStateIterator kvIterator, byte[] expectedKey, + private void assertNextEntry(BaseBinaryStateIterator kvIterator, + byte[] expectedKey, byte[] expectedValue) { assertTrue(kvIterator.hasNext()); Map.Entry entry = kvIterator.next(); @@ -124,7 +130,8 @@ private void assertNextEntry(BaseBinaryStateIterator kvIterator, assertArrayEquals(expectedValue, entry.getValue()); } - private void putEncodedKeyValueToMap(NavigableMap map, byte[] key, byte[] value) { + private void putEncodedKeyValueToMap(NavigableMap map, byte[] key, + byte[] value) { map.put(encoder.encodeKey(key), encoder.encodeValue(value)); } diff --git a/storm-client/test/jvm/org/apache/storm/state/DefaultStateSerializerTest.java b/storm-client/test/jvm/org/apache/storm/state/DefaultStateSerializerTest.java index 15718b2d4d6..f2f49971d3d 100644 --- a/storm-client/test/jvm/org/apache/storm/state/DefaultStateSerializerTest.java +++ b/storm-client/test/jvm/org/apache/storm/state/DefaultStateSerializerTest.java @@ -18,6 +18,11 @@ package org.apache.storm.state; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + import com.esotericsoftware.kryo.Kryo; import com.esotericsoftware.kryo.io.Output; import com.esotericsoftware.kryo.util.DefaultInstantiatorStrategy; @@ -31,13 +36,8 @@ import org.junit.jupiter.api.Test; import org.objenesis.strategy.StdInstantiatorStrategy; -import static org.junit.jupiter.api.Assertions.assertArrayEquals; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertThrows; - /** - * Unit tests for {@link DefaultStateSerializer} + * Unit tests for {@link DefaultStateSerializer}. */ public class DefaultStateSerializerTest { @@ -58,20 +58,23 @@ public void testSerializeDeserialize() { List> classesToRegister = new ArrayList<>(); classesToRegister.add(CheckPointState.class); classesToRegister.add(CheckPointState.State.class); - Serializer s3 = new DefaultStateSerializer<>(Collections.emptyMap(), null, classesToRegister); + Serializer s3 = new DefaultStateSerializer<>(Collections.emptyMap(), null, + classesToRegister); bytes = s3.serialize(cs); assertEquals(cs, s3.deserialize(bytes)); } /** - * The encoder wraps every value as Optional<byte[]>, so byte[] must round-trip even though + * The encoder wraps every value as Optional<byte[]>, so byte[] must round-trip even + * though * no caller registers it explicitly. */ @Test public void testDefaultStateEncoderRoundTrip() { DefaultStateEncoder encoder = - new DefaultStateEncoder<>(new DefaultStateSerializer<>(), new DefaultStateSerializer<>()); + new DefaultStateEncoder<>(new DefaultStateSerializer<>(), + new DefaultStateSerializer<>()); byte[] value = new byte[]{ 1, 2, 3 }; assertEquals("k", encoder.decodeKey(encoder.encodeKey("k"))); @@ -86,19 +89,22 @@ public void testDeserializeRejectsUnregisteredClasses() { // or an unrelated writer could have left in the store Kryo permissive = new Kryo(); permissive.setRegistrationRequired(false); - permissive.setInstantiatorStrategy(new DefaultInstantiatorStrategy(new StdInstantiatorStrategy())); + permissive + .setInstantiatorStrategy(new DefaultInstantiatorStrategy(new StdInstantiatorStrategy())); Output out = new Output(4096); permissive.writeClassAndObject(out, new UnregisteredPojo()); byte[] unregisteredClassBytes = out.toBytes(); Serializer serializer = new DefaultStateSerializer<>(); - assertThrows(IllegalArgumentException.class, () -> serializer.deserialize(unregisteredClassBytes)); + assertThrows(IllegalArgumentException.class, () -> serializer + .deserialize(unregisteredClassBytes)); } @Test public void testSerializeRejectsUnregisteredClasses() { Serializer serializer = new DefaultStateSerializer<>(); - assertThrows(IllegalArgumentException.class, () -> serializer.serialize(new UnregisteredPojo())); + assertThrows(IllegalArgumentException.class, () -> serializer + .serialize(new UnregisteredPojo())); } @Test @@ -115,7 +121,8 @@ public static class UnregisteredPojo { } /** - * Replica of the serializer as it behaved before registration was required: unregistered classes + * Replica of the serializer as it behaved before registration was required: unregistered + * classes * are written by name. State persisted by earlier releases is in this format. */ private static Kryo legacyKryo() { @@ -147,7 +154,8 @@ private static byte[] legacyEncodeValue(Object value) { @Test public void testSequentialReadsOfLegacyEncodedState() { DefaultStateEncoder encoder = - new DefaultStateEncoder<>(new DefaultStateSerializer<>(), new DefaultStateSerializer<>()); + new DefaultStateEncoder<>(new DefaultStateSerializer<>(), + new DefaultStateSerializer<>()); assertEquals("first", encoder.decodeValue(legacyEncodeValue("first"))); assertEquals("second", encoder.decodeValue(legacyEncodeValue("second"))); @@ -161,13 +169,15 @@ public void testSequentialReadsOfLegacyEncodedState() { @Test public void testRejectedPayloadDoesNotBreakLaterReads() { DefaultStateEncoder encoder = - new DefaultStateEncoder<>(new DefaultStateSerializer<>(), new DefaultStateSerializer<>()); + new DefaultStateEncoder<>(new DefaultStateSerializer<>(), + new DefaultStateSerializer<>()); assertThrows(IllegalArgumentException.class, () -> encoder.decodeValue(legacyWrite(new UnregisteredPojo()))); // a legitimate value, in both the current and the legacy encoding assertEquals("still-works", encoder.decodeValue(encoder.encodeValue("still-works"))); - assertEquals("legacy-still-works", encoder.decodeValue(legacyEncodeValue("legacy-still-works"))); + assertEquals("legacy-still-works", encoder + .decodeValue(legacyEncodeValue("legacy-still-works"))); } } diff --git a/storm-client/test/jvm/org/apache/storm/state/InMemoryKeyValueStateTest.java b/storm-client/test/jvm/org/apache/storm/state/InMemoryKeyValueStateTest.java index ed4b7045354..57e9d06c7fb 100644 --- a/storm-client/test/jvm/org/apache/storm/state/InMemoryKeyValueStateTest.java +++ b/storm-client/test/jvm/org/apache/storm/state/InMemoryKeyValueStateTest.java @@ -1,26 +1,32 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.state; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + /** - * Unit tests for {@link InMemoryKeyValueState} + * Unit tests for {@link InMemoryKeyValueState}. */ public class InMemoryKeyValueStateTest { diff --git a/storm-client/test/jvm/org/apache/storm/streams/ProcessorBoltTest.java b/storm-client/test/jvm/org/apache/storm/streams/ProcessorBoltTest.java index 8cd97e71f4a..cf0ba2123a5 100644 --- a/storm-client/test/jvm/org/apache/storm/streams/ProcessorBoltTest.java +++ b/storm-client/test/jvm/org/apache/storm/streams/ProcessorBoltTest.java @@ -1,17 +1,25 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.streams; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; + import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -36,11 +44,8 @@ import org.mockito.ArgumentCaptor; import org.mockito.Mockito; -import static org.junit.jupiter.api.Assertions.assertArrayEquals; -import static org.junit.jupiter.api.Assertions.assertEquals; - /** - * Unit tests for {@link ProcessorBolt} + * Unit tests for {@link ProcessorBolt}. */ public class ProcessorBoltTest { TopologyContext mockTopologyContext; @@ -84,18 +89,24 @@ public void testEmitAndAck() { @Test public void testAggResultAndAck() { - setUpProcessorBolt(new AggregateProcessor<>(new LongSum()), Collections.singleton("inputstream"), true, null); + setUpProcessorBolt(new AggregateProcessor<>(new LongSum()), Collections + .singleton("inputstream"), true, null); bolt.execute(mockTuple2); bolt.execute(mockTuple3); bolt.execute(punctuation); ArgumentCaptor anchor = ArgumentCaptor.forClass(Collection.class); ArgumentCaptor values = ArgumentCaptor.forClass(Values.class); ArgumentCaptor os = ArgumentCaptor.forClass(String.class); - Mockito.verify(mockOutputCollector, Mockito.times(2)).emit(os.capture(), anchor.capture(), values.capture()); - assertArrayEquals(new Object[]{ mockTuple2, mockTuple3, punctuation }, anchor.getAllValues().get(0).toArray()); - assertArrayEquals(new Object[]{ mockTuple2, mockTuple3, punctuation }, anchor.getAllValues().get(1).toArray()); - assertArrayEquals(new Object[]{ new Values(200L), new Values("__punctuation") }, values.getAllValues().toArray()); - assertArrayEquals(new Object[]{ "outputstream", "outputstream__punctuation" }, os.getAllValues().toArray()); + Mockito.verify(mockOutputCollector, Mockito.times(2)).emit(os.capture(), anchor.capture(), + values.capture()); + assertArrayEquals(new Object[]{ mockTuple2, mockTuple3, punctuation }, anchor.getAllValues() + .get(0).toArray()); + assertArrayEquals(new Object[]{ mockTuple2, mockTuple3, punctuation }, anchor.getAllValues() + .get(1).toArray()); + assertArrayEquals(new Object[]{ new Values(200L), new Values("__punctuation") }, values + .getAllValues().toArray()); + assertArrayEquals(new Object[]{ "outputstream", "outputstream__punctuation" }, os + .getAllValues().toArray()); Mockito.verify(mockOutputCollector).ack(mockTuple2); Mockito.verify(mockOutputCollector).ack(mockTuple3); Mockito.verify(mockOutputCollector).ack(punctuation); @@ -133,15 +144,18 @@ private void setUpProcessorBolt(Processor processor, ProcessorNode node = new ProcessorNode(processor, "outputstream", new Fields("value")); node.setWindowedParentStreams(windowedParentStreams); node.setWindowed(isWindowed); - Mockito.when(mockStreamToProcessors.get(Mockito.anyString())).thenReturn(Collections.singletonList(node)); - Mockito.when(mockStreamToProcessors.keySet()).thenReturn(Collections.singleton("inputstream")); + Mockito.when(mockStreamToProcessors.get(Mockito.anyString())).thenReturn(Collections + .singletonList(node)); + Mockito.when(mockStreamToProcessors.keySet()).thenReturn(Collections + .singleton("inputstream")); Map mockSources = Mockito.mock(Map.class); GlobalStreamId mockGlobalStreamId = Mockito.mock(GlobalStreamId.class); Mockito.when(mockTopologyContext.getThisSources()).thenReturn(mockSources); Mockito.when(mockSources.keySet()).thenReturn(Collections.singleton(mockGlobalStreamId)); Mockito.when(mockGlobalStreamId.get_streamId()).thenReturn("inputstream"); Mockito.when(mockGlobalStreamId.get_componentId()).thenReturn("bolt0"); - Mockito.when(mockTopologyContext.getComponentTasks(Mockito.anyString())).thenReturn(Collections.singletonList(1)); + Mockito.when(mockTopologyContext.getComponentTasks(Mockito.anyString())) + .thenReturn(Collections.singletonList(1)); graph.addVertex(node); bolt = new ProcessorBolt("bolt1", graph, Collections.singletonList(node)); if (tsFieldName != null && !tsFieldName.isEmpty()) { diff --git a/storm-client/test/jvm/org/apache/storm/streams/StatefulProcessorBoltTest.java b/storm-client/test/jvm/org/apache/storm/streams/StatefulProcessorBoltTest.java index 2dd14cc6ca6..77f71634028 100644 --- a/storm-client/test/jvm/org/apache/storm/streams/StatefulProcessorBoltTest.java +++ b/storm-client/test/jvm/org/apache/storm/streams/StatefulProcessorBoltTest.java @@ -1,17 +1,25 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.streams; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; + import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -32,11 +40,8 @@ import org.mockito.ArgumentCaptor; import org.mockito.Mockito; -import static org.junit.jupiter.api.Assertions.assertArrayEquals; -import static org.junit.jupiter.api.Assertions.assertEquals; - /** - * Unit tests for {@link StatefulProcessorBolt} + * Unit tests for {@link StatefulProcessorBolt}. */ public class StatefulProcessorBoltTest { TopologyContext mockTopologyContext; @@ -59,7 +64,8 @@ public void setUp() throws Exception { @Test public void testEmitAndAck() { - setUpStatefulProcessorBolt(new UpdateStateByKeyProcessor<>(new StateUpdater() { + setUpStatefulProcessorBolt(new UpdateStateByKeyProcessor<>(new StateUpdater() { @Override public Long init() { return 0L; @@ -85,7 +91,8 @@ public Long apply(Long state, Object value) { private void setUpStatefulProcessorBolt(Processor processor) { ProcessorNode node = new ProcessorNode(processor, "outputstream", new Fields("value")); node.setEmitsPair(true); - Mockito.when(mockStreamToProcessors.get(Mockito.anyString())).thenReturn(Collections.singletonList(node)); + Mockito.when(mockStreamToProcessors.get(Mockito.anyString())).thenReturn(Collections + .singletonList(node)); graph = new DefaultDirectedGraph<>(null, null, false); graph.addVertex(node); bolt = new StatefulProcessorBolt<>("bolt1", graph, Collections.singletonList(node)); diff --git a/storm-client/test/jvm/org/apache/storm/streams/StreamBuilderTest.java b/storm-client/test/jvm/org/apache/storm/streams/StreamBuilderTest.java index dc109661c81..335c5cbfbec 100644 --- a/storm-client/test/jvm/org/apache/storm/streams/StreamBuilderTest.java +++ b/storm-client/test/jvm/org/apache/storm/streams/StreamBuilderTest.java @@ -1,17 +1,27 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.streams; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -42,12 +52,8 @@ import org.junit.jupiter.api.Test; import org.mockito.Mockito; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; - /** - * Unit tests for {@link StreamBuilder} + * Unit tests for {@link StreamBuilder}. */ public class StreamBuilderTest { StreamBuilder streamBuilder; @@ -61,7 +67,8 @@ public void declareOutputFields(OutputFieldsDeclarer declarer) { } @Override - public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { + public void open(Map conf, TopologyContext context, + SpoutOutputCollector collector) { } @@ -81,7 +88,8 @@ public void declareOutputFields(OutputFieldsDeclarer declarer) { } @Override - public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) { + public void prepare(Map topoConf, TopologyContext context, + OutputCollector collector) { } @@ -117,7 +125,8 @@ public void testSpoutToBolt() { String spoutId = topology.get_spouts().keySet().iterator().next(); Map expected = new HashMap<>(); expected.put(new GlobalStreamId(spoutId, "default"), Grouping.shuffle(new NullStruct())); - assertEquals(expected, topology.get_bolts().values().iterator().next().get_common().get_inputs()); + assertEquals(expected, topology.get_bolts().values().iterator().next().get_common() + .get_inputs()); } @Test @@ -130,7 +139,8 @@ public void testBranch() { Map expected = new HashMap<>(); String spoutId = topology.get_spouts().keySet().iterator().next(); expected.put(new GlobalStreamId(spoutId, "default"), Grouping.shuffle(new NullStruct())); - assertEquals(expected, topology.get_bolts().values().iterator().next().get_common().get_inputs()); + assertEquals(expected, topology.get_bolts().values().iterator().next().get_common() + .get_inputs()); assertEquals(1, streams.length); assertEquals(1, streams[0].node.getOutputStreams().size()); String parentStream = streams[0].node.getOutputStreams().iterator().next() + "-branch"; @@ -143,7 +153,8 @@ public void testBranch() { @Test public void testJoin() { - Stream stream = streamBuilder.newStream(newSpout(Utils.DEFAULT_STREAM_ID), new ValueMapper<>(0)); + Stream stream = streamBuilder.newStream(newSpout(Utils.DEFAULT_STREAM_ID), + new ValueMapper<>(0)); Stream[] streams = stream.branch(x -> x % 2 == 0, x -> x % 3 == 0); PairStream s1 = streams[0].mapToPair(x -> Pair.of(x, 1)); PairStream s2 = streams[1].mapToPair(x -> Pair.of(x, 1)); @@ -154,21 +165,26 @@ public void testJoin() { @Test public void testGroupBy() { - PairStream stream = streamBuilder.newStream(newSpout(Utils.DEFAULT_STREAM_ID), new PairValueMapper<>(0, 1), 2); + PairStream stream = streamBuilder + .newStream(newSpout(Utils.DEFAULT_STREAM_ID), new PairValueMapper<>(0, 1), 2); - stream.window(TumblingWindows.of(BaseWindowedBolt.Count.of(10))).aggregateByKey(new Count<>()); + stream.window(TumblingWindows.of(BaseWindowedBolt.Count.of(10))) + .aggregateByKey(new Count<>()); StormTopology topology = streamBuilder.build(); assertEquals(2, topology.get_bolts_size()); Bolt bolt1 = topology.get_bolts().get("bolt1"); Bolt bolt2 = topology.get_bolts().get("bolt2"); - assertEquals(Grouping.shuffle(new NullStruct()), bolt1.get_common().get_inputs().values().iterator().next()); - assertEquals(Grouping.fields(Collections.singletonList("key")), bolt2.get_common().get_inputs().values().iterator().next()); + assertEquals(Grouping.shuffle(new NullStruct()), bolt1.get_common().get_inputs().values() + .iterator().next()); + assertEquals(Grouping.fields(Collections.singletonList("key")), bolt2.get_common() + .get_inputs().values().iterator().next()); } @Test public void testGlobalAggregate() { - Stream stream = streamBuilder.newStream(newSpout(Utils.DEFAULT_STREAM_ID), new ValueMapper<>(0), 2); + Stream stream = streamBuilder.newStream(newSpout(Utils.DEFAULT_STREAM_ID), + new ValueMapper<>(0), 2); stream.aggregate(new Count<>()); @@ -181,15 +197,18 @@ public void testGlobalAggregate() { expected1.put(new GlobalStreamId(spoutId, "default"), Grouping.shuffle(new NullStruct())); Map expected2 = new HashMap<>(); expected2.put(new GlobalStreamId("bolt1", "s1"), Grouping.fields(Collections.emptyList())); - expected2.put(new GlobalStreamId("bolt1", "s1__punctuation"), Grouping.all(new NullStruct())); + expected2.put(new GlobalStreamId("bolt1", "s1__punctuation"), Grouping + .all(new NullStruct())); assertEquals(expected1, bolt1.get_common().get_inputs()); assertEquals(expected2, bolt2.get_common().get_inputs()); } @Test public void testRepartition() { - Stream stream = streamBuilder.newStream(newSpout(Utils.DEFAULT_STREAM_ID), new ValueMapper<>(0)); - stream.repartition(3).filter(x -> true).repartition(2).filter(x -> true).aggregate(new Count<>()); + Stream stream = streamBuilder.newStream(newSpout(Utils.DEFAULT_STREAM_ID), + new ValueMapper<>(0)); + stream.repartition(3).filter(x -> true).repartition(2).filter(x -> true) + .aggregate(new Count<>()); StormTopology topology = streamBuilder.build(); assertEquals(1, topology.get_spouts_size()); SpoutSpec spout = topology.get_spouts().get("spout1"); @@ -209,7 +228,8 @@ public void testRepartition() { public void testBranchAndJoin() { TopologyContext mockContext = Mockito.mock(TopologyContext.class); OutputCollector mockCollector = Mockito.mock(OutputCollector.class); - Stream stream = streamBuilder.newStream(newSpout(Utils.DEFAULT_STREAM_ID), new ValueMapper<>(0), 2); + Stream stream = streamBuilder.newStream(newSpout(Utils.DEFAULT_STREAM_ID), + new ValueMapper<>(0), 2); Stream[] streams = stream.branch(x -> x % 2 == 0, x -> x % 2 == 1); PairStream> joined = streams[0].mapToPair(x -> Pair.of(x, 1)).join(streams[1].mapToPair(x -> Pair.of(x, 1))); @@ -222,7 +242,8 @@ public void testBranchAndJoin() { public void testMultiPartitionByKey() { TopologyContext mockContext = Mockito.mock(TopologyContext.class); OutputCollector mockCollector = Mockito.mock(OutputCollector.class); - Stream stream = streamBuilder.newStream(newSpout(Utils.DEFAULT_STREAM_ID), new ValueMapper<>(0)); + Stream stream = streamBuilder.newStream(newSpout(Utils.DEFAULT_STREAM_ID), + new ValueMapper<>(0)); stream.mapToPair(x -> Pair.of(x, x)) .window(TumblingWindows.of(BaseWindowedBolt.Count.of(10))) .reduceByKey((x, y) -> x + y) @@ -237,9 +258,12 @@ public void testMultiPartitionByKeyWithRepartition() { TopologyContext mockContext = Mockito.mock(TopologyContext.class); OutputCollector mockCollector = Mockito.mock(OutputCollector.class); Map expected = new HashMap<>(); - expected.put(new GlobalStreamId("bolt2", "s3"), Grouping.fields(Collections.singletonList("key"))); - expected.put(new GlobalStreamId("bolt2", "s3__punctuation"), Grouping.all(new NullStruct())); - Stream stream = streamBuilder.newStream(newSpout(Utils.DEFAULT_STREAM_ID), new ValueMapper<>(0)); + expected.put(new GlobalStreamId("bolt2", "s3"), Grouping.fields(Collections + .singletonList("key"))); + expected.put(new GlobalStreamId("bolt2", "s3__punctuation"), Grouping + .all(new NullStruct())); + Stream stream = streamBuilder.newStream(newSpout(Utils.DEFAULT_STREAM_ID), + new ValueMapper<>(0)); stream.mapToPair(x -> Pair.of(x, x)) .window(TumblingWindows.of(BaseWindowedBolt.Count.of(10))) .reduceByKey((x, y) -> x + y) @@ -256,7 +280,8 @@ public void testMultiPartitionByKeyWithRepartition() { public void testPartitionByKeySinglePartition() { TopologyContext mockContext = Mockito.mock(TopologyContext.class); OutputCollector mockCollector = Mockito.mock(OutputCollector.class); - Stream stream = streamBuilder.newStream(newSpout(Utils.DEFAULT_STREAM_ID), new ValueMapper<>(0)); + Stream stream = streamBuilder.newStream(newSpout(Utils.DEFAULT_STREAM_ID), + new ValueMapper<>(0)); stream.mapToPair(x -> Pair.of(x, x)) .reduceByKey((x, y) -> x + y) .print(); diff --git a/storm-client/test/jvm/org/apache/storm/streams/WindowedProcessorBoltTest.java b/storm-client/test/jvm/org/apache/storm/streams/WindowedProcessorBoltTest.java index efc76e021f7..f0f2ed3a3dd 100644 --- a/storm-client/test/jvm/org/apache/storm/streams/WindowedProcessorBoltTest.java +++ b/storm-client/test/jvm/org/apache/storm/streams/WindowedProcessorBoltTest.java @@ -1,17 +1,25 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.streams; +import static org.junit.jupiter.api.Assertions.assertEquals; + import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -35,10 +43,8 @@ import org.mockito.ArgumentCaptor; import org.mockito.Mockito; -import static org.junit.jupiter.api.Assertions.assertEquals; - /** - * Unit test for {@link WindowedProcessorBolt} + * Unit test for {@link WindowedProcessorBolt}. */ public class WindowedProcessorBoltTest { TopologyContext mockTopologyContext; @@ -78,8 +84,10 @@ public void testEmit() throws Exception { private void setUpWindowedProcessorBolt(Processor processor, Window window) { ProcessorNode node = new ProcessorNode(processor, "outputstream", new Fields("value")); node.setWindowed(true); - Mockito.when(mockStreamToProcessors.get(Mockito.anyString())).thenReturn(Collections.singletonList(node)); - Mockito.when(mockStreamToProcessors.keySet()).thenReturn(Collections.singleton("inputstream")); + Mockito.when(mockStreamToProcessors.get(Mockito.anyString())).thenReturn(Collections + .singletonList(node)); + Mockito.when(mockStreamToProcessors.keySet()).thenReturn(Collections + .singleton("inputstream")); graph = new DefaultDirectedGraph<>(null, null, false); graph.addVertex(node); bolt = new WindowedProcessorBolt("bolt1", graph, Collections.singletonList(node), window); diff --git a/storm-client/test/jvm/org/apache/storm/streams/processors/CoGroupByKeyProcessorTest.java b/storm-client/test/jvm/org/apache/storm/streams/processors/CoGroupByKeyProcessorTest.java index 9e6fd276aee..bc2fda0478b 100644 --- a/storm-client/test/jvm/org/apache/storm/streams/processors/CoGroupByKeyProcessorTest.java +++ b/storm-client/test/jvm/org/apache/storm/streams/processors/CoGroupByKeyProcessorTest.java @@ -1,17 +1,25 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.streams.processors; +import static org.junit.jupiter.api.Assertions.assertEquals; + import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -20,8 +28,6 @@ import org.apache.storm.streams.Pair; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; - public class CoGroupByKeyProcessorTest { private CoGroupByKeyProcessor coGroupByKeyProcessor; private final String firstStream = "first"; @@ -31,25 +37,26 @@ public class CoGroupByKeyProcessorTest { private final ProcessorContext, List>>> context = new ProcessorContext, List>>>() { - @Override + @Override public void forward(Pair, List>> input) { - res.add(input); - } + res.add(input); + } - @Override - public void forward(Pair, List>> input, String stream) { - } + @Override + public void forward(Pair, List>> input, + String stream) { + } - @Override + @Override public boolean isWindowed() { - return true; - } + return true; + } - @Override + @Override public Set getWindowedParentStreams() { - return null; - } - }; + return null; + } + }; private final List> firstKeyValues = Arrays.asList( Pair.of(2, 4), @@ -71,7 +78,8 @@ public Set getWindowedParentStreams() { public void testCoGroupByKey() { coGroupByKeyProcessor = new CoGroupByKeyProcessor<>(firstStream, secondStream); processValues(); - List, Collection>>> expected = new ArrayList<>(); + List, Collection>>> expected = + new ArrayList<>(); Collection list1 = new ArrayList<>(); list1.add(25); Collection list2 = new ArrayList<>(); @@ -88,7 +96,6 @@ public void testCoGroupByKey() { assertEquals(expected.get(0), res.get(2)); } - private void processValues() { res.clear(); coGroupByKeyProcessor.init(context); diff --git a/storm-client/test/jvm/org/apache/storm/streams/processors/JoinProcessorTest.java b/storm-client/test/jvm/org/apache/storm/streams/processors/JoinProcessorTest.java index 436461a83be..96de7dbb07a 100644 --- a/storm-client/test/jvm/org/apache/storm/streams/processors/JoinProcessorTest.java +++ b/storm-client/test/jvm/org/apache/storm/streams/processors/JoinProcessorTest.java @@ -1,17 +1,25 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.streams.processors; +import static org.junit.jupiter.api.Assertions.assertEquals; + import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -20,34 +28,33 @@ import org.apache.storm.streams.operations.PairValueJoiner; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; - public class JoinProcessorTest { JoinProcessor, Integer, Integer> joinProcessor; String leftStream = "left"; String rightStream = "right"; List>> res = new ArrayList<>(); - ProcessorContext>> context = new ProcessorContext>>() { - @Override + ProcessorContext>> context = + new ProcessorContext>>() { + @Override public void forward(Pair> input) { - res.add(input); - } + res.add(input); + } - @Override + @Override public void forward(Pair> input, String stream) { - } + } - @Override + @Override public boolean isWindowed() { - return true; - } + return true; + } - @Override + @Override public Set getWindowedParentStreams() { - return null; - } - }; + return null; + } + }; List> leftKeyValues = Arrays.asList( Pair.of(2, 4), diff --git a/storm-client/test/jvm/org/apache/storm/testing/IntegrationTest.java b/storm-client/test/jvm/org/apache/storm/testing/IntegrationTest.java index 6ecb574245a..a2b1bfa9e66 100644 --- a/storm-client/test/jvm/org/apache/storm/testing/IntegrationTest.java +++ b/storm-client/test/jvm/org/apache/storm/testing/IntegrationTest.java @@ -27,11 +27,14 @@ /** * Annotation to mark integration tests. Integration tests will be run during the Maven - * integration-test phase, whereas unit tests will be run during the Maven test phase. + * integration-test phase, whereas unit tests will be run during the Maven + * test phase. *

      - * Integration tests can be in the same package as unit tests. To mark a test as integration test, add the annotation + * Integration tests can be in the same package as unit tests. To mark a test as integration test, + * add the annotation * - * {@literal @}IntegrationTest to the class definition, or to any methods you want to run during the integration test phase. For example: + * {@literal @}IntegrationTest to the class definition, or to any methods you want to run during the + * integration test phase. For example: *

      * {@literal @}IntegrationTest
      public class MyIntegrationTest {
      ...
      } */ diff --git a/storm-client/test/jvm/org/apache/storm/testing/PerformanceTest.java b/storm-client/test/jvm/org/apache/storm/testing/PerformanceTest.java index beb9b0c3195..7057b489792 100644 --- a/storm-client/test/jvm/org/apache/storm/testing/PerformanceTest.java +++ b/storm-client/test/jvm/org/apache/storm/testing/PerformanceTest.java @@ -26,7 +26,8 @@ import org.junit.jupiter.api.Tag; /** - * Annotation to mark performance tests. Performance tests will be run if the profile performance-tests or all-tests are enabled. + * Annotation to mark performance tests. Performance tests will be run if the profile + * performance-tests or all-tests are enabled. *

      * Performance tests can be in the same package as unit tests. To mark a test as a performance test, * add the annotation @PerformanceTest to the class definition. @@ -37,7 +38,8 @@ * ...
      * } *

      - * In general performance tests should have a time limit on them, but the time limit should be liberal enough to account + * In general performance tests should have a time limit on them, but the time limit should be + * liberal enough to account * for running on CI systems like GitHub actions, or the apache jenkins build. */ @Target({ ElementType.TYPE, ElementType.METHOD }) diff --git a/storm-client/test/jvm/org/apache/storm/topology/PersistentWindowedBoltExecutorTest.java b/storm-client/test/jvm/org/apache/storm/topology/PersistentWindowedBoltExecutorTest.java index 7343ac98408..b68b56f3688 100644 --- a/storm-client/test/jvm/org/apache/storm/topology/PersistentWindowedBoltExecutorTest.java +++ b/storm-client/test/jvm/org/apache/storm/topology/PersistentWindowedBoltExecutorTest.java @@ -1,17 +1,29 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.topology; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.contains; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.AdditionalAnswers.returnsArgAt; + import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -48,14 +60,8 @@ import org.mockito.junit.jupiter.MockitoExtension; import org.mockito.stubbing.Answer; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.contains; -import static org.junit.jupiter.api.Assertions.assertArrayEquals; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.AdditionalAnswers.returnsArgAt; - /** - * Unit tests for {@link PersistentWindowedBoltExecutor} + * Unit tests for {@link PersistentWindowedBoltExecutor}. */ @ExtendWith(MockitoExtension.class) public class PersistentWindowedBoltExecutorTest { @@ -104,7 +110,8 @@ public void setUp() throws Exception { tupleTs = System.currentTimeMillis(); Mockito.when(mockBolt.getTimestampExtractor()).thenReturn(mockTimestampExtractor); mockTopologyContext = Mockito.mock(TopologyContext.class); - Mockito.when(mockTopologyContext.getThisStreams()).thenReturn(Collections.singleton(LATE_STREAM)); + Mockito.when(mockTopologyContext.getThisStreams()).thenReturn(Collections + .singleton(LATE_STREAM)); mockOutputCollector = Mockito.mock(OutputCollector.class); executor = new PersistentWindowedBoltExecutor<>(mockBolt); testStormConf.put(Config.TOPOLOGY_BOLTS_WINDOW_LENGTH_COUNT, WINDOW_EVENT_COUNT); @@ -116,14 +123,16 @@ public void setUp() throws Exception { Mockito.when(mockPartitionState.get(Mockito.any(), Mockito.any())).then(returnsArgAt(1)); Mockito.when(mockWindowState.get(Mockito.any(), Mockito.any())).then(returnsArgAt(1)); Mockito.when(mockSystemState.iterator()).thenReturn( - ImmutableMap.>of("es", Optional.empty(), "ts", Optional.empty()).entrySet().iterator()); + ImmutableMap.>of("es", Optional.empty(), "ts", Optional.empty()) + .entrySet().iterator()); executor.prepare(testStormConf, mockTopologyContext, mockOutputCollector, mockWindowState, mockPartitionState, mockSystemState); } @Test public void testExecuteTuple() { - Mockito.when(mockWaterMarkEventGenerator.track(Mockito.any(), Mockito.anyLong())).thenReturn(true); + Mockito.when(mockWaterMarkEventGenerator.track(Mockito.any(), Mockito.anyLong())) + .thenReturn(true); Mockito.when(mockTimestampExtractor.extractTimestamp(Mockito.any())).thenReturn(tupleTs); Tuple mockTuple = Mockito.mock(Tuple.class); executor.initState(null); @@ -135,7 +144,8 @@ public void testExecuteTuple() { @Test public void testExecuteLatetuple() { - Mockito.when(mockWaterMarkEventGenerator.track(Mockito.any(), Mockito.anyLong())).thenReturn(false); + Mockito.when(mockWaterMarkEventGenerator.track(Mockito.any(), Mockito.anyLong())) + .thenReturn(false); Mockito.when(mockTimestampExtractor.extractTimestamp(Mockito.any())).thenReturn(tupleTs); Tuple mockTuple = Mockito.mock(Tuple.class); Mockito.when(mockTuple.getFields()).thenReturn(new Fields("ts")); @@ -156,7 +166,8 @@ public void testExecuteLatetuple() { @Test public void testActivation() { - Mockito.when(mockWaterMarkEventGenerator.track(Mockito.any(), Mockito.anyLong())).thenReturn(true); + Mockito.when(mockWaterMarkEventGenerator.track(Mockito.any(), Mockito.anyLong())) + .thenReturn(true); Mockito.when(mockTimestampExtractor.extractTimestamp(Mockito.any())).thenReturn(tupleTs); executor.initState(null); executor.waterMarkEventGenerator = mockWaterMarkEventGenerator; @@ -164,7 +175,8 @@ public void testActivation() { List mockTuples = getMockTuples(WINDOW_EVENT_COUNT); mockTuples.forEach(t -> executor.execute(t)); // all tuples acked - Mockito.verify(mockOutputCollector, Mockito.times(WINDOW_EVENT_COUNT)).ack(tupleCaptor.capture()); + Mockito.verify(mockOutputCollector, Mockito.times(WINDOW_EVENT_COUNT)).ack(tupleCaptor + .capture()); assertArrayEquals(mockTuples.toArray(), tupleCaptor.getAllValues().toArray()); Mockito.doAnswer((Answer) invocation -> { @@ -183,22 +195,27 @@ public void testActivation() { // partition ids ArgumentCaptor pkCatptor = ArgumentCaptor.forClass(String.class); - Mockito.verify(mockPartitionState, Mockito.times(1)).put(pkCatptor.capture(), partitionValuesCaptor.capture()); + Mockito.verify(mockPartitionState, Mockito.times(1)).put(pkCatptor.capture(), + partitionValuesCaptor.capture()); assertEquals(PARTITION_KEY, pkCatptor.getValue()); List expectedPartitionIds = Collections.singletonList(0L); - assertThat(partitionValuesCaptor.getValue(), contains(expectedPartitionIds.toArray(new Long[0]))); + assertThat(partitionValuesCaptor.getValue(), contains(expectedPartitionIds + .toArray(new Long[0]))); // window partitions - Mockito.verify(mockWindowState, Mockito.times(1)).put(longCaptor.capture(), windowValuesCaptor.capture()); + Mockito.verify(mockWindowState, Mockito.times(1)).put(longCaptor.capture(), + windowValuesCaptor.capture()); assertEquals((long) expectedPartitionIds.get(0), (long) longCaptor.getValue()); assertEquals(WINDOW_EVENT_COUNT, windowValuesCaptor.getValue().size()); List tuples = windowValuesCaptor.getValue() - .getEvents().stream().map(Event::get).collect(Collectors.toList()); + .getEvents().stream().map(Event::get) + .collect(Collectors.toList()); assertArrayEquals(mockTuples.toArray(), tuples.toArray()); // window system state ArgumentCaptor keyCaptor = ArgumentCaptor.forClass(String.class); - Mockito.verify(mockSystemState, Mockito.times(2)).put(keyCaptor.capture(), systemValuesCaptor.capture()); + Mockito.verify(mockSystemState, Mockito.times(2)).put(keyCaptor.capture(), + systemValuesCaptor.capture()); assertEquals(EVICTION_STATE_KEY, keyCaptor.getAllValues().get(0)); assertEquals(Optional.of(Pair.of((long) WINDOW_EVENT_COUNT, (long) WINDOW_EVENT_COUNT)), systemValuesCaptor.getAllValues().get(0)); @@ -208,7 +225,8 @@ public void testActivation() { @Test public void testCacheEviction() { - Mockito.when(mockWaterMarkEventGenerator.track(Mockito.any(), Mockito.anyLong())).thenReturn(true); + Mockito.when(mockWaterMarkEventGenerator.track(Mockito.any(), Mockito.anyLong())) + .thenReturn(true); Mockito.when(mockTimestampExtractor.extractTimestamp(Mockito.any())).thenReturn(tupleTs); executor.initState(null); executor.waterMarkEventGenerator = mockWaterMarkEventGenerator; @@ -218,7 +236,8 @@ public void testCacheEviction() { int numPartitions = tupleCount / WindowState.MAX_PARTITION_EVENTS; int numEvictedPartitions = numPartitions - WindowState.MIN_PARTITIONS; - Mockito.verify(mockWindowState, Mockito.times(numEvictedPartitions)).put(longCaptor.capture(), windowValuesCaptor.capture()); + Mockito.verify(mockWindowState, Mockito.times(numEvictedPartitions)).put(longCaptor + .capture(), windowValuesCaptor.capture()); // number of evicted events assertEquals(numEvictedPartitions * WindowState.MAX_PARTITION_EVENTS, windowValuesCaptor.getAllValues().stream().mapToInt(x -> x.size()).sum()); @@ -227,10 +246,12 @@ public void testCacheEviction() { windowValuesCaptor.getAllValues().forEach(v -> partitionMap.put(v.getId(), v)); ArgumentCaptor stringCaptor = ArgumentCaptor.forClass(String.class); - Mockito.verify(mockPartitionState, Mockito.times(numPartitions)).put(stringCaptor.capture(), partitionValuesCaptor.capture()); + Mockito.verify(mockPartitionState, Mockito.times(numPartitions)).put(stringCaptor.capture(), + partitionValuesCaptor.capture()); // partition ids 0 .. 19 assertThat(partitionValuesCaptor.getAllValues().get(numPartitions - 1), - contains(LongStream.range(0, numPartitions).boxed().collect(Collectors.toList()).toArray(new Long[0]))); + contains(LongStream.range(0, numPartitions).boxed().collect(Collectors.toList()) + .toArray(new Long[0]))); Mockito.when(mockWindowState.get(Mockito.any(), Mockito.any())).then(invocation -> { Long partition = invocation.getArgument(0); @@ -248,7 +269,8 @@ public void testCacheEviction() { long activationTs = tupleTs + 1000; executor.getWindowManager().add(new WaterMarkEvent<>(activationTs)); - Mockito.verify(mockBolt, Mockito.times(tupleCount / WINDOW_EVENT_COUNT)).execute(Mockito.any()); + Mockito.verify(mockBolt, Mockito.times(tupleCount / WINDOW_EVENT_COUNT)).execute(Mockito + .any()); } @Test @@ -269,7 +291,8 @@ public void testRollbackAfterInit() { Mockito.verify(mockBolt, Mockito.times(1)).preRollback(); Mockito.verify(mockPartitionState, Mockito.times(1)).rollback(); ArgumentCaptor stringArgumentCaptor = ArgumentCaptor.forClass(String.class); - Mockito.verify(mockPartitionState, Mockito.times(2)).put(stringArgumentCaptor.capture(), partitionValuesCaptor.capture()); + Mockito.verify(mockPartitionState, Mockito.times(2)).put(stringArgumentCaptor.capture(), + partitionValuesCaptor.capture()); Mockito.verify(mockWindowState, Mockito.times(1)).rollback(); Mockito.verify(mockSystemState, Mockito.times(1)).rollback(); Mockito.verify(mockSystemState, Mockito.times(2)).iterator(); @@ -282,4 +305,4 @@ private List getMockTuples(long count) { } return tuples; } -} \ No newline at end of file +} diff --git a/storm-client/test/jvm/org/apache/storm/topology/SimpleWindowPartitionCacheTest.java b/storm-client/test/jvm/org/apache/storm/topology/SimpleWindowPartitionCacheTest.java index 52523c2cd0b..30255e05250 100644 --- a/storm-client/test/jvm/org/apache/storm/topology/SimpleWindowPartitionCacheTest.java +++ b/storm-client/test/jvm/org/apache/storm/topology/SimpleWindowPartitionCacheTest.java @@ -1,17 +1,27 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

      * http://www.apache.org/licenses/LICENSE-2.0 *

      - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.topology; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -23,13 +33,8 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; - /** - * Unit tests for {@link SimpleWindowPartitionCache} + * Unit tests for {@link SimpleWindowPartitionCache}. */ public class SimpleWindowPartitionCacheTest { @@ -39,14 +44,16 @@ public void setUp() throws Exception { @Test public void testBuildInvalid1() { - assertThrows(IllegalArgumentException.class, () -> SimpleWindowPartitionCache.newBuilder() + assertThrows(IllegalArgumentException.class, () -> SimpleWindowPartitionCache.newBuilder() .maximumSize(0) .build(null)); } @Test public void testBuildInvalid2() { - assertThrows(IllegalArgumentException.class, () -> SimpleWindowPartitionCache.newBuilder() + assertThrows(IllegalArgumentException.class, () -> SimpleWindowPartitionCache.newBuilder() .maximumSize(-1) .build(null)); } @@ -158,7 +165,6 @@ public void testInvalidate() { assertTrue(cache.asMap().isEmpty()); } - @Timeout(10000) @Test public void testConcurrentGet() throws Exception { @@ -221,4 +227,4 @@ public void testEviction() { cache.get(2); assertEquals(Collections.singletonList(0), removed); } -} \ No newline at end of file +} diff --git a/storm-client/test/jvm/org/apache/storm/topology/StatefulBoltExecutorTest.java b/storm-client/test/jvm/org/apache/storm/topology/StatefulBoltExecutorTest.java index 1824828183d..3ba7e7cdb1d 100644 --- a/storm-client/test/jvm/org/apache/storm/topology/StatefulBoltExecutorTest.java +++ b/storm-client/test/jvm/org/apache/storm/topology/StatefulBoltExecutorTest.java @@ -1,17 +1,30 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.topology; +import static org.apache.storm.spout.CheckPointState.Action.COMMIT; +import static org.apache.storm.spout.CheckPointState.Action.INITSTATE; +import static org.apache.storm.spout.CheckPointState.Action.PREPARE; +import static org.apache.storm.spout.CheckPointState.Action.ROLLBACK; +import static org.apache.storm.spout.CheckpointSpout.CHECKPOINT_FIELD_ACTION; +import static org.apache.storm.spout.CheckpointSpout.CHECKPOINT_FIELD_TXID; +import static org.mockito.Mockito.mock; + import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -26,16 +39,8 @@ import org.junit.jupiter.api.Test; import org.mockito.Mockito; -import static org.apache.storm.spout.CheckPointState.Action.COMMIT; -import static org.apache.storm.spout.CheckPointState.Action.INITSTATE; -import static org.apache.storm.spout.CheckPointState.Action.PREPARE; -import static org.apache.storm.spout.CheckPointState.Action.ROLLBACK; -import static org.apache.storm.spout.CheckpointSpout.CHECKPOINT_FIELD_ACTION; -import static org.apache.storm.spout.CheckpointSpout.CHECKPOINT_FIELD_TXID; -import static org.mockito.Mockito.mock; - /** - * Unit tests for {@link StatefulBoltExecutor} + * Unit tests for {@link StatefulBoltExecutor}. */ public class StatefulBoltExecutorTest { private StatefulBoltExecutor> executor; @@ -56,10 +61,13 @@ public void setUp() throws Exception { mockState = Mockito.mock(KeyValueState.class); Mockito.when(mockTopologyContext.getThisComponentId()).thenReturn("test"); Mockito.when(mockTopologyContext.getThisTaskId()).thenReturn(1); - GlobalStreamId globalStreamId = new GlobalStreamId("test", CheckpointSpout.CHECKPOINT_STREAM_ID); - Map thisSources = Collections.singletonMap(globalStreamId, mock(Grouping.class)); + GlobalStreamId globalStreamId = new GlobalStreamId("test", + CheckpointSpout.CHECKPOINT_STREAM_ID); + Map thisSources = Collections.singletonMap(globalStreamId, + mock(Grouping.class)); Mockito.when(mockTopologyContext.getThisSources()).thenReturn(thisSources); - Mockito.when(mockTopologyContext.getComponentTasks(Mockito.any())).thenReturn(Collections.singletonList(1)); + Mockito.when(mockTopologyContext.getComponentTasks(Mockito.any())).thenReturn(Collections + .singletonList(1)); mockTuple = Mockito.mock(Tuple.class); mockCheckpointTuple = Mockito.mock(Tuple.class); executor.prepare(mockStormConf, mockTopologyContext, mockOutputCollector, mockState); @@ -76,9 +84,12 @@ public void testHandleTupleBeforeInit() { public void testHandleTuple() { Mockito.when(mockTuple.getSourceStreamId()).thenReturn("default"); executor.execute(mockTuple); - Mockito.when(mockCheckpointTuple.getSourceStreamId()).thenReturn(CheckpointSpout.CHECKPOINT_STREAM_ID); - Mockito.when(mockCheckpointTuple.getValueByField(CHECKPOINT_FIELD_ACTION)).thenReturn(INITSTATE); - Mockito.when(mockCheckpointTuple.getLongByField(CHECKPOINT_FIELD_TXID)).thenReturn(Long.valueOf(0)); + Mockito.when(mockCheckpointTuple.getSourceStreamId()) + .thenReturn(CheckpointSpout.CHECKPOINT_STREAM_ID); + Mockito.when(mockCheckpointTuple.getValueByField(CHECKPOINT_FIELD_ACTION)) + .thenReturn(INITSTATE); + Mockito.when(mockCheckpointTuple.getLongByField(CHECKPOINT_FIELD_TXID)).thenReturn(Long + .valueOf(0)); Mockito.doNothing().when(mockOutputCollector).ack(mockCheckpointTuple); executor.execute(mockCheckpointTuple); Mockito.verify(mockBolt, Mockito.times(1)).execute(mockTuple); @@ -89,9 +100,12 @@ public void testHandleTuple() { public void testRollback() { Mockito.when(mockTuple.getSourceStreamId()).thenReturn("default"); executor.execute(mockTuple); - Mockito.when(mockCheckpointTuple.getSourceStreamId()).thenReturn(CheckpointSpout.CHECKPOINT_STREAM_ID); - Mockito.when(mockCheckpointTuple.getValueByField(CHECKPOINT_FIELD_ACTION)).thenReturn(ROLLBACK); - Mockito.when(mockCheckpointTuple.getLongByField(CHECKPOINT_FIELD_TXID)).thenReturn(Long.valueOf(0)); + Mockito.when(mockCheckpointTuple.getSourceStreamId()) + .thenReturn(CheckpointSpout.CHECKPOINT_STREAM_ID); + Mockito.when(mockCheckpointTuple.getValueByField(CHECKPOINT_FIELD_ACTION)) + .thenReturn(ROLLBACK); + Mockito.when(mockCheckpointTuple.getLongByField(CHECKPOINT_FIELD_TXID)).thenReturn(Long + .valueOf(0)); Mockito.doNothing().when(mockOutputCollector).ack(mockCheckpointTuple); executor.execute(mockCheckpointTuple); Mockito.verify(mockState, Mockito.times(1)).rollback(); @@ -101,9 +115,12 @@ public void testRollback() { public void testCommit() { Mockito.when(mockTuple.getSourceStreamId()).thenReturn("default"); executor.execute(mockTuple); - Mockito.when(mockCheckpointTuple.getSourceStreamId()).thenReturn(CheckpointSpout.CHECKPOINT_STREAM_ID); - Mockito.when(mockCheckpointTuple.getValueByField(CHECKPOINT_FIELD_ACTION)).thenReturn(COMMIT); - Mockito.when(mockCheckpointTuple.getLongByField(CHECKPOINT_FIELD_TXID)).thenReturn(Long.valueOf(0)); + Mockito.when(mockCheckpointTuple.getSourceStreamId()) + .thenReturn(CheckpointSpout.CHECKPOINT_STREAM_ID); + Mockito.when(mockCheckpointTuple.getValueByField(CHECKPOINT_FIELD_ACTION)) + .thenReturn(COMMIT); + Mockito.when(mockCheckpointTuple.getLongByField(CHECKPOINT_FIELD_TXID)).thenReturn(Long + .valueOf(0)); Mockito.doNothing().when(mockOutputCollector).ack(mockCheckpointTuple); executor.execute(mockCheckpointTuple); Mockito.verify(mockBolt, Mockito.times(1)).preCommit(0L); @@ -114,14 +131,19 @@ public void testCommit() { public void testPrepareAndRollbackBeforeInitstate() { Mockito.when(mockTuple.getSourceStreamId()).thenReturn("default"); executor.execute(mockTuple); - Mockito.when(mockCheckpointTuple.getSourceStreamId()).thenReturn(CheckpointSpout.CHECKPOINT_STREAM_ID); - Mockito.when(mockCheckpointTuple.getValueByField(CHECKPOINT_FIELD_ACTION)).thenReturn(PREPARE); - Mockito.when(mockCheckpointTuple.getLongByField(CHECKPOINT_FIELD_TXID)).thenReturn(Long.valueOf(100)); + Mockito.when(mockCheckpointTuple.getSourceStreamId()) + .thenReturn(CheckpointSpout.CHECKPOINT_STREAM_ID); + Mockito.when(mockCheckpointTuple.getValueByField(CHECKPOINT_FIELD_ACTION)) + .thenReturn(PREPARE); + Mockito.when(mockCheckpointTuple.getLongByField(CHECKPOINT_FIELD_TXID)).thenReturn(Long + .valueOf(100)); executor.execute(mockCheckpointTuple); Mockito.verify(mockOutputCollector, Mockito.times(1)).fail(mockCheckpointTuple); - Mockito.when(mockCheckpointTuple.getValueByField(CHECKPOINT_FIELD_ACTION)).thenReturn(ROLLBACK); - Mockito.when(mockCheckpointTuple.getLongByField(CHECKPOINT_FIELD_TXID)).thenReturn(Long.valueOf(100)); + Mockito.when(mockCheckpointTuple.getValueByField(CHECKPOINT_FIELD_ACTION)) + .thenReturn(ROLLBACK); + Mockito.when(mockCheckpointTuple.getLongByField(CHECKPOINT_FIELD_TXID)).thenReturn(Long + .valueOf(100)); Mockito.doNothing().when(mockOutputCollector).ack(mockCheckpointTuple); executor.execute(mockCheckpointTuple); Mockito.verify(mockState, Mockito.times(1)).rollback(); @@ -130,14 +152,19 @@ public void testPrepareAndRollbackBeforeInitstate() { @Test public void testCommitBeforeInitstate() { Mockito.when(mockTuple.getSourceStreamId()).thenReturn("default"); - Mockito.when(mockCheckpointTuple.getSourceStreamId()).thenReturn(CheckpointSpout.CHECKPOINT_STREAM_ID); - Mockito.when(mockCheckpointTuple.getValueByField(CHECKPOINT_FIELD_ACTION)).thenReturn(COMMIT); - Mockito.when(mockCheckpointTuple.getLongByField(CHECKPOINT_FIELD_TXID)).thenReturn(Long.valueOf(100)); + Mockito.when(mockCheckpointTuple.getSourceStreamId()) + .thenReturn(CheckpointSpout.CHECKPOINT_STREAM_ID); + Mockito.when(mockCheckpointTuple.getValueByField(CHECKPOINT_FIELD_ACTION)) + .thenReturn(COMMIT); + Mockito.when(mockCheckpointTuple.getLongByField(CHECKPOINT_FIELD_TXID)).thenReturn(Long + .valueOf(100)); executor.execute(mockCheckpointTuple); Mockito.verify(mockOutputCollector, Mockito.times(1)).ack(mockCheckpointTuple); - Mockito.when(mockCheckpointTuple.getValueByField(CHECKPOINT_FIELD_ACTION)).thenReturn(ROLLBACK); - Mockito.when(mockCheckpointTuple.getLongByField(CHECKPOINT_FIELD_TXID)).thenReturn(Long.valueOf(100)); + Mockito.when(mockCheckpointTuple.getValueByField(CHECKPOINT_FIELD_ACTION)) + .thenReturn(ROLLBACK); + Mockito.when(mockCheckpointTuple.getLongByField(CHECKPOINT_FIELD_TXID)).thenReturn(Long + .valueOf(100)); executor.execute(mockCheckpointTuple); Mockito.verify(mockState, Mockito.times(1)).rollback(); } @@ -145,19 +172,27 @@ public void testCommitBeforeInitstate() { @Test public void testPrepareAndCommit() { Mockito.when(mockTuple.getSourceStreamId()).thenReturn("default"); - Mockito.when(mockCheckpointTuple.getSourceStreamId()).thenReturn(CheckpointSpout.CHECKPOINT_STREAM_ID); - Mockito.when(mockCheckpointTuple.getValueByField(CHECKPOINT_FIELD_ACTION)).thenReturn(INITSTATE); - Mockito.when(mockCheckpointTuple.getLongByField(CHECKPOINT_FIELD_TXID)).thenReturn(Long.valueOf(0)); + Mockito.when(mockCheckpointTuple.getSourceStreamId()) + .thenReturn(CheckpointSpout.CHECKPOINT_STREAM_ID); + Mockito.when(mockCheckpointTuple.getValueByField(CHECKPOINT_FIELD_ACTION)) + .thenReturn(INITSTATE); + Mockito.when(mockCheckpointTuple.getLongByField(CHECKPOINT_FIELD_TXID)).thenReturn(Long + .valueOf(0)); executor.execute(mockCheckpointTuple); executor.execute(mockTuple); - Mockito.when(mockCheckpointTuple.getSourceStreamId()).thenReturn(CheckpointSpout.CHECKPOINT_STREAM_ID); - Mockito.when(mockCheckpointTuple.getValueByField(CHECKPOINT_FIELD_ACTION)).thenReturn(PREPARE); - Mockito.when(mockCheckpointTuple.getLongByField(CHECKPOINT_FIELD_TXID)).thenReturn(Long.valueOf(100)); + Mockito.when(mockCheckpointTuple.getSourceStreamId()) + .thenReturn(CheckpointSpout.CHECKPOINT_STREAM_ID); + Mockito.when(mockCheckpointTuple.getValueByField(CHECKPOINT_FIELD_ACTION)) + .thenReturn(PREPARE); + Mockito.when(mockCheckpointTuple.getLongByField(CHECKPOINT_FIELD_TXID)).thenReturn(Long + .valueOf(100)); executor.execute(mockCheckpointTuple); executor.execute(mockTuple); - Mockito.when(mockCheckpointTuple.getValueByField(CHECKPOINT_FIELD_ACTION)).thenReturn(COMMIT); - Mockito.when(mockCheckpointTuple.getLongByField(CHECKPOINT_FIELD_TXID)).thenReturn(Long.valueOf(100)); + Mockito.when(mockCheckpointTuple.getValueByField(CHECKPOINT_FIELD_ACTION)) + .thenReturn(COMMIT); + Mockito.when(mockCheckpointTuple.getLongByField(CHECKPOINT_FIELD_TXID)).thenReturn(Long + .valueOf(100)); executor.execute(mockCheckpointTuple); mockOutputCollector.ack(mockTuple); Mockito.verify(mockState, Mockito.times(1)).commit(100L); diff --git a/storm-client/test/jvm/org/apache/storm/topology/StatefulWindowedBoltExecutorTest.java b/storm-client/test/jvm/org/apache/storm/topology/StatefulWindowedBoltExecutorTest.java index 248658337ae..4d1dae298fd 100644 --- a/storm-client/test/jvm/org/apache/storm/topology/StatefulWindowedBoltExecutorTest.java +++ b/storm-client/test/jvm/org/apache/storm/topology/StatefulWindowedBoltExecutorTest.java @@ -1,17 +1,26 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.topology; +import static org.apache.storm.topology.StatefulWindowedBoltExecutor.TaskStream; +import static org.apache.storm.topology.StatefulWindowedBoltExecutor.WindowState; +import static org.junit.jupiter.api.Assertions.assertThrows; + import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -30,12 +39,8 @@ import org.junit.jupiter.api.Test; import org.mockito.Mockito; -import static org.apache.storm.topology.StatefulWindowedBoltExecutor.TaskStream; -import static org.apache.storm.topology.StatefulWindowedBoltExecutor.WindowState; -import static org.junit.jupiter.api.Assertions.assertThrows; - /** - * Unit tests for {@link StatefulWindowedBoltExecutor} + * Unit tests for {@link StatefulWindowedBoltExecutor}. */ public class StatefulWindowedBoltExecutorTest { StatefulWindowedBoltExecutor> executor; @@ -59,7 +64,6 @@ public void testPrepare() { () -> executor.prepare(mockStormConf, mockTopologyContext, mockOutputCollector)); } - @Test public void testPrepareWithMsgid() { mockStormConf.put(Config.TOPOLOGY_BOLTS_MESSAGE_ID_FIELD_NAME, "msgid"); @@ -83,7 +87,8 @@ public void testExecute() throws Exception { } Mockito.verify(mockBolt, Mockito.times(1)).execute(getTupleWindow(tuples)); WindowState expectedState = new WindowState(Long.MIN_VALUE, 4); - Mockito.verify(mockState, Mockito.times(1)).put(Mockito.any(TaskStream.class), Mockito.eq(expectedState)); + Mockito.verify(mockState, Mockito.times(1)).put(Mockito.any(TaskStream.class), Mockito + .eq(expectedState)); } @Test @@ -95,8 +100,10 @@ public void testRecovery() { mockState = Mockito.mock(KeyValueState.class); Map mockMap = Mockito.mock(Map.class); Mockito.when(mockTopologyContext.getThisSources()).thenReturn(mockMap); - Mockito.when(mockTopologyContext.getComponentTasks(Mockito.anyString())).thenReturn(Collections.singletonList(1)); - Mockito.when(mockMap.keySet()).thenReturn(Collections.singleton(new GlobalStreamId("a", "s"))); + Mockito.when(mockTopologyContext.getComponentTasks(Mockito.anyString())) + .thenReturn(Collections.singletonList(1)); + Mockito.when(mockMap.keySet()).thenReturn(Collections.singleton(new GlobalStreamId("a", + "s"))); WindowState mockWindowState = new WindowState(4, 4); Mockito.when(mockState.get(Mockito.any(TaskStream.class))).thenReturn(mockWindowState); executor.prepare(mockStormConf, mockTopologyContext, mockOutputCollector, mockState); @@ -106,7 +113,8 @@ public void testRecovery() { executor.execute(tuple); } WindowState expectedState = new WindowState(4, 9); - Mockito.verify(mockState, Mockito.times(1)).put(Mockito.any(TaskStream.class), Mockito.eq(expectedState)); + Mockito.verify(mockState, Mockito.times(1)).put(Mockito.any(TaskStream.class), Mockito + .eq(expectedState)); } private TupleWindow getTupleWindow(List tuples) { @@ -119,7 +127,8 @@ private List getMockTuples(int count) { Tuple mockTuple = Mockito.mock(Tuple.class); Mockito.when(mockTuple.getLongByField("msgid")).thenReturn(i); Mockito.when(mockTuple.getSourceTask()).thenReturn(1); - Mockito.when(mockTuple.getSourceGlobalStreamId()).thenReturn(new GlobalStreamId("a", "s")); + Mockito.when(mockTuple.getSourceGlobalStreamId()).thenReturn(new GlobalStreamId("a", + "s")); mockTuples.add(mockTuple); } return mockTuples; diff --git a/storm-client/test/jvm/org/apache/storm/topology/TopologyBuilderTest.java b/storm-client/test/jvm/org/apache/storm/topology/TopologyBuilderTest.java index 68992704790..0f8f5cbd476 100644 --- a/storm-client/test/jvm/org/apache/storm/topology/TopologyBuilderTest.java +++ b/storm-client/test/jvm/org/apache/storm/topology/TopologyBuilderTest.java @@ -1,17 +1,28 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.topology; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.mock; + import java.util.Map; import java.util.Set; import org.apache.storm.generated.GlobalStreamId; @@ -25,17 +36,13 @@ import org.apache.storm.tuple.Tuple; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.Mockito.mock; - public class TopologyBuilderTest { private final TopologyBuilder builder = new TopologyBuilder(); @Test public void testSetRichBolt() { - assertThrows(IllegalArgumentException.class, () -> builder.setBolt("bolt", mock(IRichBolt.class), 0)); + assertThrows(IllegalArgumentException.class, () -> builder.setBolt("bolt", + mock(IRichBolt.class), 0)); } @Test @@ -92,7 +99,8 @@ private IRichSpout makeDummySpout() { public void declareOutputFields(OutputFieldsDeclarer declarer) {} @Override - public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {} + public void open(Map conf, TopologyContext context, + SpoutOutputCollector collector) {} @Override public void nextTuple() {} diff --git a/storm-client/test/jvm/org/apache/storm/topology/WindowedBoltExecutorTest.java b/storm-client/test/jvm/org/apache/storm/topology/WindowedBoltExecutorTest.java index 98c8be8de49..9293e7f718f 100644 --- a/storm-client/test/jvm/org/apache/storm/topology/WindowedBoltExecutorTest.java +++ b/storm-client/test/jvm/org/apache/storm/topology/WindowedBoltExecutorTest.java @@ -1,17 +1,32 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.topology; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.core.Is.is; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -40,17 +55,8 @@ import org.mockito.ArgumentCaptor; import org.mockito.Mockito; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.core.Is.is; -import static org.junit.jupiter.api.Assertions.assertArrayEquals; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertInstanceOf; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; - /** - * Unit tests for {@link WindowedBoltExecutor} + * Unit tests for {@link WindowedBoltExecutor}. */ public class WindowedBoltExecutorTest { @@ -70,7 +76,8 @@ public Fields getComponentOutputFields(String componentId, String streamId) { }; } - private Tuple getTuple(String streamId, final Fields fields, Values values, String srcComponent) { + private Tuple getTuple(String streamId, final Fields fields, Values values, + String srcComponent) { return new TupleImpl(getContext(fields), values, srcComponent, 1, streamId) { @Override public GlobalStreamId getSourceGlobalStreamId() { @@ -121,27 +128,31 @@ public void testExecuteWithTs() { for (long ts : timestamps) { executor.execute(getTuple("s1", new Fields("ts"), new Values(ts), "s1Src")); } - //Thread.sleep(120); + // Thread.sleep(120); executor.waterMarkEventGenerator.run(); - //System.out.println(testWindowedBolt.tupleWindows); + // System.out.println(testWindowedBolt.tupleWindows); assertEquals(3, testWindowedBolt.tupleWindows.size()); TupleWindow first = testWindowedBolt.tupleWindows.get(0); assertArrayEquals(new long[]{ 603, 605, 607 }, new long[]{ - (long) first.get().get(0).getValue(0), (long) first.get().get(1).getValue(0), + (long) first.get().get(0).getValue(0), (long) first.get().get(1) + .getValue(0), (long) first.get().get(2).getValue(0) }); TupleWindow second = testWindowedBolt.tupleWindows.get(1); assertArrayEquals(new long[]{ 603, 605, 607, 618 }, new long[]{ - (long) second.get().get(0).getValue(0), (long) second.get().get(1).getValue(0), - (long) second.get().get(2).getValue(0), (long) second.get().get(3).getValue(0) + (long) second.get().get(0).getValue(0), (long) second.get().get(1) + .getValue(0), + (long) second.get().get(2).getValue(0), (long) second.get().get(3) + .getValue(0) }); TupleWindow third = testWindowedBolt.tupleWindows.get(2); assertArrayEquals(new long[]{ 618, 626 }, - new long[]{ (long) third.get().get(0).getValue(0), (long) third.get().get(1).getValue(0) }); + new long[]{ (long) third.get().get(0).getValue(0), (long) third.get() + .get(1).getValue(0) }); } @Test @@ -158,12 +169,14 @@ public void testPrepareLateTupleStreamWithoutTs() { executor = new WindowedBoltExecutor(testWindowedBolt); TopologyContext context = getTopologyContext(); // emulate the call of withLateTupleStream method - Mockito.when(context.getThisStreams()).thenReturn(new HashSet<>(Arrays.asList("default", "$late"))); + Mockito.when(context.getThisStreams()).thenReturn(new HashSet<>(Arrays.asList("default", + "$late"))); try { executor.prepare(conf, context, getOutputCollector()); fail(); } catch (IllegalArgumentException e) { - assertThat(e.getMessage(), is("Late tuple stream can be defined only when specifying a timestamp field")); + assertThat(e.getMessage(), + is("Late tuple stream can be defined only when specifying a timestamp field")); } } @@ -185,7 +198,9 @@ public void testPrepareLateTupleStreamWithoutBuilder() { executor.prepare(conf, context, getOutputCollector()); fail(); } catch (IllegalArgumentException e) { - assertThat(e.getMessage(), is("Stream for late tuples must be defined with the builder method withLateTupleStream")); + assertThat(e.getMessage(), + is("Stream for late tuples must be defined with the builder method " + + "withLateTupleStream")); } } @@ -195,7 +210,8 @@ public void testExecuteWithLateTupleStream() { testWindowedBolt.withTimestampField("ts"); executor = new WindowedBoltExecutor(testWindowedBolt); TopologyContext context = getTopologyContext(); - Mockito.when(context.getThisStreams()).thenReturn(new HashSet<>(Arrays.asList("default", "$late"))); + Mockito.when(context.getThisStreams()).thenReturn(new HashSet<>(Arrays.asList("default", + "$late"))); OutputCollector outputCollector = Mockito.mock(OutputCollector.class); Map conf = new HashMap<>(); @@ -204,7 +220,7 @@ public void testExecuteWithLateTupleStream() { conf.put(Config.TOPOLOGY_BOLTS_SLIDING_INTERVAL_DURATION_MS, 10); conf.put(Config.TOPOLOGY_BOLTS_LATE_TUPLE_STREAM, "$late"); conf.put(Config.TOPOLOGY_BOLTS_TUPLE_TIMESTAMP_MAX_LAG_MS, 5); - //Trigger manually to avoid timing issues + // Trigger manually to avoid timing issues conf.put(Config.TOPOLOGY_BOLTS_WATERMARK_EVENT_INTERVAL_MS, 1_000_000); executor.prepare(conf, context, outputCollector); @@ -216,12 +232,13 @@ public void testExecuteWithLateTupleStream() { tuples.add(tuple); executor.execute(tuple); - //Update the watermark to this timestamp + // Update the watermark to this timestamp executor.waterMarkEventGenerator.run(); } System.out.println(testWindowedBolt.tupleWindows); Tuple tuple = tuples.get(tuples.size() - 1); - Mockito.verify(outputCollector).emit("$late", Collections.singletonList(tuple), new Values(new DetachedTuple(tuple))); + Mockito.verify(outputCollector).emit("$late", Collections.singletonList(tuple), + new Values(new DetachedTuple(tuple))); } @Test @@ -230,7 +247,8 @@ public void testLateTupleStreamEmitsSerializableTuple() throws Exception { testWindowedBolt.withTimestampField("ts"); executor = new WindowedBoltExecutor(testWindowedBolt); TopologyContext context = getTopologyContext(); - Mockito.when(context.getThisStreams()).thenReturn(new HashSet<>(Arrays.asList("default", "$late"))); + Mockito.when(context.getThisStreams()).thenReturn(new HashSet<>(Arrays.asList("default", + "$late"))); OutputCollector outputCollector = Mockito.mock(OutputCollector.class); Map conf = new HashMap<>(); @@ -239,7 +257,7 @@ public void testLateTupleStreamEmitsSerializableTuple() throws Exception { conf.put(Config.TOPOLOGY_BOLTS_SLIDING_INTERVAL_DURATION_MS, 10); conf.put(Config.TOPOLOGY_BOLTS_LATE_TUPLE_STREAM, "$late"); conf.put(Config.TOPOLOGY_BOLTS_TUPLE_TIMESTAMP_MAX_LAG_MS, 5); - //Trigger manually to avoid timing issues + // Trigger manually to avoid timing issues conf.put(Config.TOPOLOGY_BOLTS_WATERMARK_EVENT_INTERVAL_MS, 1_000_000); executor.prepare(conf, context, outputCollector); @@ -252,7 +270,8 @@ public void testLateTupleStreamEmitsSerializableTuple() throws Exception { } ArgumentCaptor> valuesCaptor = ArgumentCaptor.forClass(List.class); - Mockito.verify(outputCollector).emit(Mockito.eq("$late"), Mockito.anyCollection(), valuesCaptor.capture()); + Mockito.verify(outputCollector).emit(Mockito.eq("$late"), Mockito.anyCollection(), + valuesCaptor.capture()); Object lateValue = valuesCaptor.getValue().get(0); assertInstanceOf(DetachedTuple.class, lateValue); DetachedTuple detached = (DetachedTuple) lateValue; @@ -260,7 +279,8 @@ public void testLateTupleStreamEmitsSerializableTuple() throws Exception { assertEquals(lateTuple.getSourceComponent(), detached.getSourceComponent()); assertEquals(lateTuple.getSourceStreamId(), detached.getSourceStreamId()); - // STORM-4000 regression: the late tuple must survive Kryo serialization without java fallback + // STORM-4000 regression: the late tuple must survive Kryo serialization without java + // fallback Map serConf = Utils.readDefaultConfig(); serConf.put(Config.TOPOLOGY_FALL_BACK_ON_JAVA_SERIALIZATION, false); byte[] serialized = new KryoValuesSerializer(serConf).serialize(new Values(lateValue)); @@ -281,7 +301,7 @@ private static class TestWindowedBolt extends BaseWindowedBolt { @Override public void execute(TupleWindow input) { - //System.out.println(input); + // System.out.println(input); tupleWindows.add(input); } } diff --git a/storm-client/test/jvm/org/apache/storm/trident/TestTridentTopology.java b/storm-client/test/jvm/org/apache/storm/trident/TestTridentTopology.java index 866a76de893..fa9be183d53 100644 --- a/storm-client/test/jvm/org/apache/storm/trident/TestTridentTopology.java +++ b/storm-client/test/jvm/org/apache/storm/trident/TestTridentTopology.java @@ -1,17 +1,25 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.trident; +import static org.junit.jupiter.api.Assertions.assertEquals; + import java.util.Map; import java.util.Set; import org.apache.storm.generated.Bolt; @@ -25,21 +33,20 @@ import org.apache.storm.tuple.Values; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; - public class TestTridentTopology { private StormTopology buildTopology() { FixedBatchSpout spout = new FixedBatchSpout(new Fields("sentence"), 3, new Values("the cow jumped over the moon"), - new Values("the man went to the store and bought some candy"), + new Values("the man went to the store and " + + "bought some candy"), new Values("four score and seven years ago"), new Values("how many apples can you eat")); spout.setCycle(true); TridentTopology topology = new TridentTopology(); topology.newStream("spout", spout) - //no name + // no name .each(new Fields("sentence"), new Split(), new Fields("word")) .partitionBy(new Fields("word")) .name("abc") diff --git a/storm-client/test/jvm/org/apache/storm/trident/TridentWindowingTest.java b/storm-client/test/jvm/org/apache/storm/trident/TridentWindowingTest.java index 84437dfc89d..d9d77a77347 100644 --- a/storm-client/test/jvm/org/apache/storm/trident/TridentWindowingTest.java +++ b/storm-client/test/jvm/org/apache/storm/trident/TridentWindowingTest.java @@ -1,17 +1,25 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

      * http://www.apache.org/licenses/LICENSE-2.0 *

      - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.trident; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + import java.util.concurrent.TimeUnit; import org.apache.storm.topology.base.BaseWindowedBolt; import org.apache.storm.trident.windowing.InMemoryWindowsStore; @@ -26,9 +34,6 @@ import org.apache.storm.trident.windowing.strategy.WindowStrategy; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; - /** * */ @@ -37,10 +42,12 @@ public class TridentWindowingTest { @Test public void testWindowStrategyInstances() { - WindowStrategy tumblingCountStrategy = TumblingCountWindow.of(10).getWindowStrategy(); + WindowStrategy tumblingCountStrategy = TumblingCountWindow.of(10) + .getWindowStrategy(); assertTrue(tumblingCountStrategy instanceof TumblingCountWindowStrategy); - WindowStrategy slidingCountStrategy = SlidingCountWindow.of(100, 10).getWindowStrategy(); + WindowStrategy slidingCountStrategy = SlidingCountWindow.of(100, 10) + .getWindowStrategy(); assertTrue(slidingCountStrategy instanceof SlidingCountWindowStrategy); WindowStrategy tumblingDurationStrategy = TumblingDurationWindow.of( @@ -70,7 +77,8 @@ public void testWindowConfig() { windowLength = 20; TumblingDurationWindow tumblingDurationWindow = - TumblingDurationWindow.of(new BaseWindowedBolt.Duration(windowLength, TimeUnit.SECONDS)); + TumblingDurationWindow.of(new BaseWindowedBolt.Duration(windowLength, + TimeUnit.SECONDS)); assertTrue(tumblingDurationWindow.getWindowLength() == windowLength * 1000); assertTrue(tumblingDurationWindow.getSlidingLength() == windowLength * 1000); @@ -78,7 +86,8 @@ public void testWindowConfig() { slidingLength = 10; SlidingDurationWindow slidingDurationWindow = SlidingDurationWindow.of(new BaseWindowedBolt.Duration(windowLength, TimeUnit.SECONDS), - new BaseWindowedBolt.Duration(slidingLength, TimeUnit.SECONDS)); + new BaseWindowedBolt.Duration(slidingLength, + TimeUnit.SECONDS)); assertTrue(slidingDurationWindow.getWindowLength() == windowLength * 1000); assertTrue(slidingDurationWindow.getSlidingLength() == slidingLength * 1000); } diff --git a/storm-client/test/jvm/org/apache/storm/tuple/DetachedTupleTest.java b/storm-client/test/jvm/org/apache/storm/tuple/DetachedTupleTest.java index 6d56f143142..beea6555498 100644 --- a/storm-client/test/jvm/org/apache/storm/tuple/DetachedTupleTest.java +++ b/storm-client/test/jvm/org/apache/storm/tuple/DetachedTupleTest.java @@ -1,17 +1,31 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.tuple; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + import java.util.Arrays; import java.util.HashMap; import java.util.List; @@ -26,14 +40,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertInstanceOf; -import static org.junit.jupiter.api.Assertions.assertNotEquals; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; - /** * Unit tests for {@link DetachedTuple}. */ @@ -58,7 +64,8 @@ public Fields getComponentOutputFields(String componentId, String streamId) { @BeforeEach public void setUp() { Fields fields = new Fields("id", "ts"); - sourceTuple = new TupleImpl(getContext(fields), new Values(42, 1000L), "srcComponent", 7, "srcStream"); + sourceTuple = new TupleImpl(getContext(fields), new Values(42, 1000L), "srcComponent", 7, + "srcStream"); detached = new DetachedTuple(sourceTuple); } @@ -67,7 +74,8 @@ public void testSnapshotsSourceMetadata() { assertEquals("srcComponent", detached.getSourceComponent()); assertEquals(7, detached.getSourceTask()); assertEquals("srcStream", detached.getSourceStreamId()); - assertEquals(new GlobalStreamId("srcComponent", "srcStream"), detached.getSourceGlobalStreamId()); + assertEquals(new GlobalStreamId("srcComponent", "srcStream"), detached + .getSourceGlobalStreamId()); } @Test @@ -95,7 +103,8 @@ public void testValueBasedEquality() { assertEquals(detached, other); assertEquals(detached.hashCode(), other.hashCode()); - Tuple differentTuple = new TupleImpl(getContext(new Fields("id", "ts")), new Values(43, 1000L), "srcComponent", 7, "srcStream"); + Tuple differentTuple = new TupleImpl(getContext(new Fields("id", "ts")), new Values(43, + 1000L), "srcComponent", 7, "srcStream"); assertNotEquals(detached, new DetachedTuple(differentTuple)); } @@ -109,7 +118,8 @@ public void testNotEqualToNullOrOtherType() { @Test public void testTypedGetters() { Fields fields = new Fields("bool", "byte", "short"); - Tuple typed = new TupleImpl(getContext(fields), new Values(true, (byte) 1, (short) 2), "srcComponent", 0, "srcStream"); + Tuple typed = new TupleImpl(getContext(fields), new Values(true, (byte) 1, (short) 2), + "srcComponent", 0, "srcStream"); DetachedTuple typedDetached = new DetachedTuple(typed); assertEquals(Boolean.TRUE, typedDetached.getBoolean(0)); @@ -123,7 +133,8 @@ public void testTypedGetters() { @Test public void testKryoRoundTripWithNullValue() { Fields fields = new Fields("id", "ts"); - Tuple withNull = new TupleImpl(getContext(fields), new Values(42, null), "srcComponent", 7, "srcStream"); + Tuple withNull = new TupleImpl(getContext(fields), new Values(42, null), "srcComponent", 7, + "srcStream"); DetachedTuple detachedWithNull = new DetachedTuple(withNull); Map conf = Utils.readDefaultConfig(); @@ -140,7 +151,8 @@ public void testKryoRoundTripWithNullValue() { /** * The {@code fields} member is transient and rebuilt lazily. Populate it via {@link DetachedTuple#getFields()} - * before serializing so the transient-skip path is actually exercised: the deserialized copy must rebuild its + * before serializing so the transient-skip path is actually exercised: the deserialized copy + * must rebuild its * fields from scratch rather than carry a serialized {@link Fields} instance. */ @Test @@ -161,8 +173,10 @@ public void testKryoRoundTripWithPopulatedFields() { } /** - * STORM-4000 regression: a tuple emitted on the late tuple stream must survive Kryo serialization. With - * {@link Config#TOPOLOGY_FALL_BACK_ON_JAVA_SERIALIZATION} disabled, serializing a {@link TupleImpl} (with its + * STORM-4000 regression: a tuple emitted on the late tuple stream must survive Kryo + * serialization. With + * {@link Config#TOPOLOGY_FALL_BACK_ON_JAVA_SERIALIZATION} disabled, serializing a {@link + * TupleImpl} (with its * attached topology context) fails, while a {@link DetachedTuple} round-trips cleanly. */ @Test diff --git a/storm-client/test/jvm/org/apache/storm/tuple/FieldsTest.java b/storm-client/test/jvm/org/apache/storm/tuple/FieldsTest.java index 37a46e3835d..494f23768da 100644 --- a/storm-client/test/jvm/org/apache/storm/tuple/FieldsTest.java +++ b/storm-client/test/jvm/org/apache/storm/tuple/FieldsTest.java @@ -1,27 +1,33 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.tuple; -import java.util.Arrays; -import java.util.Iterator; -import java.util.List; -import org.junit.jupiter.api.Test; - import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import org.junit.jupiter.api.Test; + public class FieldsTest { @Test diff --git a/storm-client/test/jvm/org/apache/storm/tuple/MessageIdTest.java b/storm-client/test/jvm/org/apache/storm/tuple/MessageIdTest.java index a8ef0559bd4..e43fba05a22 100644 --- a/storm-client/test/jvm/org/apache/storm/tuple/MessageIdTest.java +++ b/storm-client/test/jvm/org/apache/storm/tuple/MessageIdTest.java @@ -1,27 +1,33 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.tuple; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + import java.util.HashSet; import java.util.Random; import java.util.Set; import org.apache.storm.utils.KeyStreamRandom; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; - public class MessageIdTest { private static final long LCG_MULTIPLIER = 0x5DEECE66DL; @@ -49,8 +55,10 @@ public void generatedIdsDoNotRevealTheFollowingIds() { } /** - * Guards the test above: the same prediction does work against a java.util.Random, so a failure of - * generatedIdsDoNotRevealTheFollowingIds means the ids really are predictable rather than the prediction being broken. + * Guards the test above: the same prediction does work against a java.util.Random, so a failure + * of + * generatedIdsDoNotRevealTheFollowingIds means the ids really are predictable rather than the + * prediction being broken. */ @Test public void lcgPredictionWorksAgainstJavaUtilRandom() { @@ -73,7 +81,8 @@ public void makeRootIdKeepsTheGeneratedIds() { } /** - * Recovers the 48 bit state of a java.util.Random from a single nextLong output and returns the nextLong it would produce + * Recovers the 48 bit state of a java.util.Random from a single nextLong output and returns the + * nextLong it would produce * afterwards, or 0 if the state could not be recovered. */ private static long predictNextFromLcgState(long observed) { diff --git a/storm-client/test/jvm/org/apache/storm/tuple/ValuesTest.java b/storm-client/test/jvm/org/apache/storm/tuple/ValuesTest.java index 7e7a28219d6..c3ee0b5b7de 100644 --- a/storm-client/test/jvm/org/apache/storm/tuple/ValuesTest.java +++ b/storm-client/test/jvm/org/apache/storm/tuple/ValuesTest.java @@ -6,10 +6,10 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - *

      - * http://www.apache.org/licenses/LICENSE-2.0 - *

      - * Unless required by applicable law or agreed to in writing, software + * + *

      http://www.apache.org/licenses/LICENSE-2.0 + * + *

      Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and diff --git a/storm-client/test/jvm/org/apache/storm/utils/ConfigUtilsTest.java b/storm-client/test/jvm/org/apache/storm/utils/ConfigUtilsTest.java index 9897eb75b8b..7c0ab28fd6a 100644 --- a/storm-client/test/jvm/org/apache/storm/utils/ConfigUtilsTest.java +++ b/storm-client/test/jvm/org/apache/storm/utils/ConfigUtilsTest.java @@ -1,17 +1,29 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.utils; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + import java.net.UnknownHostException; import java.util.Arrays; import java.util.Collections; @@ -22,12 +34,6 @@ import org.apache.storm.Config; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; - public class ConfigUtilsTest { private Map mockMap(String key, Object value) { @@ -55,7 +61,8 @@ public void getValueAsList_nullKeyNotSupported() { @Test public void getValueAsList_nullConfig() { - assertThrows(IllegalArgumentException.class, () -> ConfigUtils.getValueAsList(Config.WORKER_CHILDOPTS, null)); + assertThrows(IllegalArgumentException.class, () -> ConfigUtils + .getValueAsList(Config.WORKER_CHILDOPTS, null)); } @Test @@ -102,8 +109,10 @@ public void getValueAsList_nonStringList() { @Deprecated @Test public void getBlobstoreHDFSPrincipal() throws UnknownHostException { - Map conf = mockMap(Config.BLOBSTORE_HDFS_PRINCIPAL, "primary/_HOST@EXAMPLE.COM"); - assertEquals(Config.getBlobstoreHDFSPrincipal(conf), "primary/" + Utils.localHostname() + "@EXAMPLE.COM"); + Map conf = mockMap(Config.BLOBSTORE_HDFS_PRINCIPAL, + "primary/_HOST@EXAMPLE.COM"); + assertEquals(Config.getBlobstoreHDFSPrincipal(conf), "primary/" + Utils.localHostname() + + "@EXAMPLE.COM"); String principal = "primary/_HOST_HOST@EXAMPLE.COM"; conf.put(Config.BLOBSTORE_HDFS_PRINCIPAL, principal); @@ -132,8 +141,10 @@ public void getBlobstoreHDFSPrincipal() throws UnknownHostException { @Test public void getHfdsPrincipal() throws UnknownHostException { - Map conf = mockMap(Config.STORM_HDFS_LOGIN_PRINCIPAL, "primary/_HOST@EXAMPLE.COM"); - assertEquals(Config.getHdfsPrincipal(conf), "primary/" + Utils.localHostname() + "@EXAMPLE.COM"); + Map conf = mockMap(Config.STORM_HDFS_LOGIN_PRINCIPAL, + "primary/_HOST@EXAMPLE.COM"); + assertEquals(Config.getHdfsPrincipal(conf), "primary/" + Utils.localHostname() + + "@EXAMPLE.COM"); String principal = "primary/_HOST_HOST@EXAMPLE.COM"; conf.put(Config.STORM_HDFS_LOGIN_PRINCIPAL, principal); @@ -247,13 +258,15 @@ public void maskPasswords_masksZookeeperAndNettyTlsStorePasswords() { @Test public void maskCredentials_masksKeysThatOnlyPluginsDeclare() { Map conf = new HashMap<>(); - conf.put("storm.daemon.metrics.reporter.plugin.prometheus.basic_auth_password", "plugin-secret"); + conf.put("storm.daemon.metrics.reporter.plugin.prometheus.basic_auth_password", + "plugin-secret"); conf.put("storm.zookeeper.auth.password", "zk-pass"); conf.put("some.plugin.shared_secret", "shared"); Map masked = ConfigUtils.maskCredentials(conf); - assertEquals("*****", masked.get("storm.daemon.metrics.reporter.plugin.prometheus.basic_auth_password")); + assertEquals("*****", masked + .get("storm.daemon.metrics.reporter.plugin.prometheus.basic_auth_password")); assertEquals("*****", masked.get("storm.zookeeper.auth.password")); assertEquals("*****", masked.get("some.plugin.shared_secret")); } @@ -263,7 +276,8 @@ public void maskCredentials_masksTheAnnotatedKeysAsWell() { Map conf = new HashMap<>(); conf.put(Config.STORM_ZOOKEEPER_AUTH_PAYLOAD, "zk-user:zk-secret"); - assertEquals("*****", ConfigUtils.maskCredentials(conf).get(Config.STORM_ZOOKEEPER_AUTH_PAYLOAD)); + assertEquals("*****", ConfigUtils.maskCredentials(conf) + .get(Config.STORM_ZOOKEEPER_AUTH_PAYLOAD)); } @Test @@ -278,7 +292,8 @@ public void maskCredentials_leavesNonStringValuesAlone() { assertEquals(30, masked.get("task.credentials.poll.secs")); assertEquals(600, masked.get("nimbus.credential.renewers.freq.secs")); - assertEquals(Collections.singletonList("org.example.AutoCreds"), masked.get("topology.auto-credentials")); + assertEquals(Collections.singletonList("org.example.AutoCreds"), masked + .get("topology.auto-credentials")); assertEquals(Collections.singletonList("nimbus1"), masked.get("nimbus.seeds")); } @@ -286,7 +301,9 @@ public void maskCredentials_leavesNonStringValuesAlone() { public void isCredentialKey_recognisesAnnotatedAndPluginDeclaredKeys() { assertTrue(ConfigUtils.isCredentialKey(Config.STORM_ZOOKEEPER_AUTH_PAYLOAD)); assertTrue(ConfigUtils.isCredentialKey(Config.NIMBUS_THRIFT_TLS_CLIENT_KEYSTORE_PASSWORD)); - assertTrue(ConfigUtils.isCredentialKey("storm.daemon.metrics.reporter.plugin.prometheus.basic_auth_password")); + assertTrue(ConfigUtils + .isCredentialKey("storm.daemon.metrics.reporter.plugin.prometheus.basic_auth_passw" + + "ord")); assertTrue(ConfigUtils.isCredentialKey("some.plugin.shared_secret")); } @@ -307,6 +324,7 @@ public void maskPasswords_keepsOrdinaryValues() { Map masked = ConfigUtils.maskPasswords(conf); assertEquals(Collections.singletonList("zk1"), masked.get(Config.STORM_ZOOKEEPER_SERVERS)); - assertEquals("/etc/storm/nimbus.jks", masked.get(Config.NIMBUS_THRIFT_TLS_SERVER_KEYSTORE_PATH)); + assertEquals("/etc/storm/nimbus.jks", masked + .get(Config.NIMBUS_THRIFT_TLS_SERVER_KEYSTORE_PATH)); } } diff --git a/storm-client/test/jvm/org/apache/storm/utils/CuratorUtilsTest.java b/storm-client/test/jvm/org/apache/storm/utils/CuratorUtilsTest.java index 68f730fc6fe..78408ea5de5 100644 --- a/storm-client/test/jvm/org/apache/storm/utils/CuratorUtilsTest.java +++ b/storm-client/test/jvm/org/apache/storm/utils/CuratorUtilsTest.java @@ -18,13 +18,16 @@ package org.apache.storm.utils; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; + import java.io.File; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; - import org.apache.curator.test.InstanceSpec; +import org.apache.curator.test.TestingServer; import org.apache.storm.Config; import org.apache.storm.cluster.DaemonType; import org.apache.storm.shade.org.apache.curator.ensemble.fixed.FixedEnsembleProvider; @@ -36,14 +39,9 @@ import org.apache.storm.shade.org.apache.zookeeper.client.ZKClientConfig; import org.apache.storm.shade.org.apache.zookeeper.common.ClientX509Util; import org.junit.jupiter.api.Test; -import org.apache.curator.test.TestingServer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - -import static org.junit.jupiter.api.Assertions.assertArrayEquals; -import static org.junit.jupiter.api.Assertions.assertEquals; - public class CuratorUtilsTest { private static final Logger LOG = LoggerFactory.getLogger(CuratorUtilsTest.class); private static final int JUTE_MAXBUFFER = 400000000; @@ -70,7 +68,8 @@ public void newCuratorUsesExponentialBackoffTest() { config.put(Config.STORM_ZOOKEEPER_RETRY_INTERVAL_CEILING, expectedCeiling); CuratorFramework curator = CuratorUtils.newCurator(config, - Collections.singletonList("bogus_server"), 42, "", DaemonType.WORKER.getDefaultZkAcls(config)); + Collections.singletonList("bogus_server"), 42, "", DaemonType.WORKER + .getDefaultZkAcls(config)); StormBoundedExponentialBackoffRetry policy = (StormBoundedExponentialBackoffRetry) curator.getZookeeperClient().getRetryPolicy(); assertEquals(policy.getBaseSleepTimeMs(), expectedInterval); @@ -140,10 +139,12 @@ private CuratorFrameworkFactory.Builder setupBuilder(boolean withAuth) { public Map setUpSecureConfig(String testDataPath) throws Exception { System.setProperty("zookeeper.ssl.keyStore.location", testDataPath + "testKeyStore.jks"); System.setProperty("zookeeper.ssl.keyStore.password", "testpass"); - System.setProperty("zookeeper.ssl.trustStore.location", testDataPath + "testTrustStore.jks"); + System.setProperty("zookeeper.ssl.trustStore.location", testDataPath + + "testTrustStore.jks"); System.setProperty("zookeeper.ssl.trustStore.password", "testpass"); System.setProperty("zookeeper.request.timeout", "12345"); - System.setProperty("zookeeper.serverCnxnFactory", "org.apache.zookeeper.server.NettyServerCnxnFactory"); + System.setProperty("zookeeper.serverCnxnFactory", + "org.apache.zookeeper.server.NettyServerCnxnFactory"); System.setProperty("jute.maxbuffer", String.valueOf(JUTE_MAXBUFFER)); System.setProperty("javax.net.debug", "ssl"); @@ -187,11 +188,13 @@ public void testSecureZKConfiguration() throws Exception { CuratorFramework curatorFramework = builder.build(); curatorFramework.start(); ZooKeeper zk = curatorFramework.getZookeeperClient().getZooKeeper(); - validateSSLConfiguration(ObjectReader.getString(conf.get(Config.STORM_ZOOKEEPER_SSL_KEYSTORE_PATH)), + validateSSLConfiguration(ObjectReader.getString(conf + .get(Config.STORM_ZOOKEEPER_SSL_KEYSTORE_PATH)), ObjectReader.getString(conf.get(Config.STORM_ZOOKEEPER_SSL_KEYSTORE_PASSWORD)), ObjectReader.getString(conf.get(Config.STORM_ZOOKEEPER_SSL_TRUSTSTORE_PATH)), ObjectReader.getString(conf.get(Config.STORM_ZOOKEEPER_SSL_TRUSTSTORE_PASSWORD)), - ObjectReader.getBoolean(conf.get(Config.STORM_ZOOKEEPER_SSL_HOSTNAME_VERIFICATION), true), + ObjectReader.getBoolean(conf.get(Config.STORM_ZOOKEEPER_SSL_HOSTNAME_VERIFICATION), + true), zk); this.server.close(); } @@ -200,56 +203,62 @@ private void validateSSLConfiguration(String keystoreLocation, String keystorePa String truststoreLocation, String truststorePassword, Boolean hostNameVerification, ZooKeeper zk) { try (ClientX509Util x509Util = new ClientX509Util()) { - //testing if custom values are set properly + // testing if custom values are set properly assertEquals(keystoreLocation, - zk.getClientConfig().getProperty(x509Util.getSslKeystoreLocationProperty()) - , "Validate that expected clientConfig is set in ZK config"); + zk.getClientConfig().getProperty(x509Util.getSslKeystoreLocationProperty()), + "Validate that expected clientConfig is set in ZK config"); assertEquals(keystorePassword, - zk.getClientConfig().getProperty(x509Util.getSslKeystorePasswdProperty()) - , "Validate that expected clientConfig is set in ZK config"); + zk.getClientConfig().getProperty(x509Util.getSslKeystorePasswdProperty()), + "Validate that expected clientConfig is set in ZK config"); assertEquals(truststoreLocation, - zk.getClientConfig().getProperty(x509Util.getSslTruststoreLocationProperty()) - , "Validate that expected clientConfig is set in ZK config"); + zk.getClientConfig().getProperty(x509Util.getSslTruststoreLocationProperty()), + "Validate that expected clientConfig is set in ZK config"); assertEquals(truststorePassword, - zk.getClientConfig().getProperty(x509Util.getSslTruststorePasswdProperty()) - , "Validate that expected clientConfig is set in ZK config"); + zk.getClientConfig().getProperty(x509Util.getSslTruststorePasswdProperty()), + "Validate that expected clientConfig is set in ZK config"); assertEquals(hostNameVerification.toString(), - zk.getClientConfig().getProperty(x509Util.getSslHostnameVerificationEnabledProperty()) - , "Validate that expected clientConfig is set in ZK config"); + zk.getClientConfig().getProperty(x509Util + .getSslHostnameVerificationEnabledProperty()), + "Validate that expected clientConfig is set in ZK config"); } - //testing if constant values hardcoded into the code are set properly + // testing if constant values hardcoded into the code are set properly assertEquals(Boolean.TRUE.toString(), - zk.getClientConfig().getProperty(ZKClientConfig.SECURE_CLIENT) - , "Validate that expected clientConfig is set in ZK config"); + zk.getClientConfig().getProperty(ZKClientConfig.SECURE_CLIENT), + "Validate that expected clientConfig is set in ZK config"); assertEquals(ClientCnxnSocketNetty.class.getCanonicalName(), - zk.getClientConfig().getProperty(ZKClientConfig.ZOOKEEPER_CLIENT_CNXN_SOCKET) - , "Validate that expected clientConfig is set in ZK config"); + zk.getClientConfig().getProperty(ZKClientConfig.ZOOKEEPER_CLIENT_CNXN_SOCKET), + "Validate that expected clientConfig is set in ZK config"); } @Test public void testTruststoreKeystoreConfiguration() { LOG.info("Entered to the testTruststoreKeystoreConfiguration test case."); - /* - By default the truststore/keystore configurations are not set, hence the values are null. - Validate that the null values are converted into empty strings by the class. - */ + /* + By default the truststore/keystore configurations are not set, hence the values are null. + Validate that the null values are converted into empty strings by the class. + */ Map conf = new HashMap<>(); CuratorUtils.SslConf zkSslConf = CuratorUtils.getSslConf(conf); assertEquals("", - zkSslConf.getKeystoreLocation(), "Validate that null value is converted to empty string."); + zkSslConf + .getKeystoreLocation(), "Validate that null value is converted to empty string."); assertEquals("", - zkSslConf.getKeystorePassword(), "Validate that null value is converted to empty string."); + zkSslConf + .getKeystorePassword(), "Validate that null value is converted to empty string."); assertEquals("", - zkSslConf.getTruststoreLocation(), "Validate that null value is converted to empty string."); + zkSslConf + .getTruststoreLocation(), "Validate that null value is converted to empty string."); assertEquals("", - zkSslConf.getTruststorePassword(), "Validate that null value is converted to empty string."); + zkSslConf + .getTruststorePassword(), "Validate that null value is converted to empty string."); assertEquals(true, - zkSslConf.getHostnameVerification(), "Validate that null value is converted to false."); + zkSslConf + .getHostnameVerification(), "Validate that null value is converted to false."); - //Validate that non-null values will remain intact + // Validate that non-null values will remain intact conf.put(Config.STORM_ZOOKEEPER_SSL_KEYSTORE_PATH, "/keystore.jks"); conf.put(Config.STORM_ZOOKEEPER_SSL_KEYSTORE_PASSWORD, "keystorePassword"); conf.put(Config.STORM_ZOOKEEPER_SSL_TRUSTSTORE_PATH, "/truststore.jks"); diff --git a/storm-client/test/jvm/org/apache/storm/utils/DefaultShellLogHandlerTest.java b/storm-client/test/jvm/org/apache/storm/utils/DefaultShellLogHandlerTest.java index bae38507839..055eac5c7f1 100644 --- a/storm-client/test/jvm/org/apache/storm/utils/DefaultShellLogHandlerTest.java +++ b/storm-client/test/jvm/org/apache/storm/utils/DefaultShellLogHandlerTest.java @@ -1,27 +1,33 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.utils; -import org.apache.storm.multilang.ShellMsg; -import org.apache.storm.multilang.ShellMsg.ShellLogLevel; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import org.apache.storm.multilang.ShellMsg.ShellLogLevel; +import org.apache.storm.multilang.ShellMsg; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + public class DefaultShellLogHandlerTest { private DefaultShellLogHandler logHandler; @@ -98,4 +104,4 @@ public void handleLog_valid() { verify(msg).getMsg(); verify(process).getProcessInfoString(); } -} \ No newline at end of file +} diff --git a/storm-client/test/jvm/org/apache/storm/utils/JCQueueBackpressureTest.java b/storm-client/test/jvm/org/apache/storm/utils/JCQueueBackpressureTest.java index 6762382157c..a5b77e34f34 100644 --- a/storm-client/test/jvm/org/apache/storm/utils/JCQueueBackpressureTest.java +++ b/storm-client/test/jvm/org/apache/storm/utils/JCQueueBackpressureTest.java @@ -1,31 +1,38 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.utils; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + import java.util.Collections; import org.apache.storm.metrics2.StormMetricRegistry; import org.apache.storm.policy.WaitStrategyPark; import org.apache.storm.utils.JCQueue.Consumer; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - public class JCQueueBackpressureTest { private static JCQueue createQueue(String name, int queueSize) { - return new JCQueue(name, name, queueSize, 0, 1, new WaitStrategyPark(0), "test", "test", Collections.singletonList(1000), 1000, new StormMetricRegistry()); + return new JCQueue(name, name, queueSize, 0, 1, new WaitStrategyPark(0), "test", "test", + Collections.singletonList(1000), 1000, new StormMetricRegistry()); } @Test @@ -88,7 +95,7 @@ public void accept(Object o) { } @Override - public void flush() throws InterruptedException { } + public void flush() throws InterruptedException {} } } diff --git a/storm-client/test/jvm/org/apache/storm/utils/JCQueueTest.java b/storm-client/test/jvm/org/apache/storm/utils/JCQueueTest.java index f2b5634734f..6028638eec8 100644 --- a/storm-client/test/jvm/org/apache/storm/utils/JCQueueTest.java +++ b/storm-client/test/jvm/org/apache/storm/utils/JCQueueTest.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.utils; @@ -36,8 +42,8 @@ public class JCQueueTest { - private final static int TIMEOUT = 5000; // MS - private final static int PRODUCER_NUM = 4; + private static final int TIMEOUT = 5000; // MS + private static final int PRODUCER_NUM = 4; IWaitStrategy waitStrategy = new WaitStrategyPark(100); @Test @@ -204,7 +210,8 @@ public void testDynamicBatchShrinksOnPartialFlush() throws Exception { public void testDynamicBatchEmptyFlushIsNoOp() throws Exception { JCQueue queue = createQueue("dynEmpty", 1024); JCQueue.DynamicBatchInserter inserter = new JCQueue.DynamicBatchInserter(queue, 8); - growEffectiveTo(inserter, 4); // leaves the batch empty after the growing flush + growEffectiveTo(inserter, + 4); // leaves the batch empty after the growing flush inserter.flush(); // empty batch -> no adaptation assertEquals(4, inserter.batchSize()); @@ -235,7 +242,8 @@ public void testDynamicBatchGrowsNotShrinksUnderBackpressure() throws Exception assertTrue(inserter.tryPublish(2L)); // batch size 1 assertTrue(inserter.tryPublish(3L)); // batch size 2 (== effective) assertFalse(inserter.tryPublish(4L)); // full batch, queue full -> tryFlush fails - // A full batch that could not be published must be read as heavy load (grow), not light load (shrink). + // A full batch that could not be published must be read as heavy load (grow), not light + // load (shrink). assertEquals(3, inserter.batchSize()); } @@ -254,7 +262,8 @@ public void testControlTuplesDrainedBeforeData() { queue.consume(collectingConsumer(drained)); assertEquals(9, drained.size()); - assertEquals("CTRL", drained.get(0), "control tuple must be drained ahead of the data backlog"); + assertEquals("CTRL", drained.get(0), + "control tuple must be drained ahead of the data backlog"); } @Test @@ -267,7 +276,8 @@ public void testControlLaneDropsOnFullWithoutBlocking() { } assertTrue(accepted > 0 && accepted < 64, "control lane must be bounded"); - // a full control lane drops (returns false) instead of blocking, and leaves the data path unaffected + // a full control lane drops (returns false) instead of blocking, and leaves the data path + // unaffected assertFalse(queue.tryPublishControl("EXTRA")); assertTrue(queue.tryPublishDirect("DATA")); assertEquals(accepted + 1, queue.size()); @@ -333,7 +343,8 @@ public void flush() { assertEquals(1, consumed); assertEquals(1, flushCount.get(), - "a drain that consumed only control tuples must still flush the consumer (e.g. deliver a flush tuple's effect)"); + "a drain that consumed only control tuples must still flush the consumer (e.g. " + + "deliver a flush tuple's effect)"); } @Test @@ -344,7 +355,8 @@ public void testExitConditionStopsControlDrain() { List drained = new ArrayList<>(); int consumed = queue.consume(collectingConsumer(drained), () -> false); - assertEquals(0, consumed, "exit condition must be honored before draining the control lane"); + assertEquals(0, consumed, + "exit condition must be honored before draining the control lane"); assertTrue(drained.isEmpty()); assertEquals(1, queue.size(), "unconsumed control tuple must remain queued"); } @@ -352,7 +364,8 @@ public void testExitConditionStopsControlDrain() { @Test public void testControlDropIsCountedInMetrics() { StormMetricRegistry registry = new StormMetricRegistry(); - JCQueue queue = new JCQueue("controlDropMetric", "controlDropMetric", 16, 0, 1, waitStrategy, + JCQueue queue = new JCQueue("controlDropMetric", "controlDropMetric", 16, 0, 1, + waitStrategy, "test", "test", Collections.singletonList(1000), 1000, registry, false, 2); int accepted = 0; @@ -360,7 +373,8 @@ public void testControlDropIsCountedInMetrics() { accepted++; } assertTrue(accepted > 0 && accepted < 64); - assertFalse(queue.tryPublishControl("EXTRA")); // second counted drop (first happened when the fill loop stopped) + assertFalse(queue + .tryPublishControl("EXTRA")); // second counted drop (first happened when the fill loop stopped) assertEquals((long) accepted, gaugeValue(registry, "control_population"), "control_population gauge must report the lane occupancy"); @@ -383,7 +397,8 @@ public void testQueueLoadExcludesControlLane() { assertTrue(queue.tryPublishControl("CTRL")); - assertEquals(0.0, queue.getQueueLoad(), 0.0, "control lane must not contribute to the data-plane load"); + assertEquals(0.0, queue.getQueueLoad(), 0.0, + "control lane must not contribute to the data-plane load"); assertEquals(1, queue.size(), "control lane must contribute to size()"); } @@ -394,7 +409,8 @@ public void testQueueIsCollectedAfterLongLivedProducerPublishes() { try { WeakReference ref = publishFromLongLivedThread(producerPool); assertTrue(awaitGarbageCollection(ref), - "JCQueue was not garbage collected — BatchInserter may still hold a strong reference to it"); + "JCQueue was not garbage collected — BatchInserter may still hold a strong " + + "reference to it"); } finally { producerPool.shutdownNow(); } @@ -404,7 +420,8 @@ public void testQueueIsCollectedAfterLongLivedProducerPublishes() { // Publishes a few tuples from a pooled thread so that the BatchInserter's ThreadLocal entry is // created on that thread. Once .get() returns the lambda is done and its closure ref is gone; // when the method returns the local `queue` variable goes out of scope. The only remaining - // reference is the WeakReference — if it is not cleared after GC, the ThreadLocal cycle is still live. + // reference is the WeakReference — if it is not cleared after GC, the ThreadLocal cycle is + // still live. private WeakReference publishFromLongLivedThread(ExecutorService pool) throws Exception { JCQueue queue = createQueue("leak", 100, 1024); pool.submit(() -> { @@ -428,7 +445,8 @@ private boolean awaitGarbageCollection(WeakReference ref) throws Interr } /** Drive the inserter with full flushes until the effective batch size reaches the target. */ - private void growEffectiveTo(JCQueue.DynamicBatchInserter inserter, int target) throws InterruptedException { + private void growEffectiveTo(JCQueue.DynamicBatchInserter inserter, + int target) throws InterruptedException { long val = 0; while (inserter.batchSize() < target) { inserter.publish(val++); @@ -440,7 +458,8 @@ private void run(Runnable producer, Runnable consumer, JCQueue queue) run(producer, consumer, queue, 20, PRODUCER_NUM); } - private void run(Runnable producer, Runnable consumer, JCQueue queue, int sleepMs, int producerNum) + private void run(Runnable producer, Runnable consumer, JCQueue queue, int sleepMs, + int producerNum) throws InterruptedException { Thread[] producerThreads = new Thread[producerNum]; @@ -471,16 +490,19 @@ private JCQueue createQueue(String name, int queueSize) { } private JCQueue createQueue(String name, int batchSize, int queueSize) { - return new JCQueue(name, name, queueSize, 0, batchSize, waitStrategy, "test", "test", Collections.singletonList(1000), 1000, new StormMetricRegistry()); + return new JCQueue(name, name, queueSize, 0, batchSize, waitStrategy, "test", "test", + Collections.singletonList(1000), 1000, new StormMetricRegistry()); } private JCQueue createQueue(String name, int batchSize, int queueSize, boolean dynamicBatch) { - return new JCQueue(name, name, queueSize, 0, batchSize, waitStrategy, "test", "test", Collections.singletonList(1000), 1000, + return new JCQueue(name, name, queueSize, 0, batchSize, waitStrategy, "test", "test", + Collections.singletonList(1000), 1000, new StormMetricRegistry(), dynamicBatch); } private JCQueue createQueueWithControlLane(String name, int queueSize, int controlQueueSize) { - return new JCQueue(name, name, queueSize, 0, 1, waitStrategy, "test", "test", Collections.singletonList(1000), 1000, + return new JCQueue(name, name, queueSize, 0, 1, waitStrategy, "test", "test", Collections + .singletonList(1000), 1000, new StormMetricRegistry(), false, controlQueueSize); } @@ -512,11 +534,12 @@ public IncProducer(JCQueue queue, long _max, long min) { @Override public void run() { try { - for (long i = 0; i < _max && (!Thread.currentThread().isInterrupted() || i < min); i++) { + for (long i = 0; i < _max && (!Thread.currentThread().isInterrupted() + || i < min); i++) { queue.publish(i); } } catch (InterruptedException e) { - //Just quit + // Just quit } } } @@ -533,7 +556,7 @@ private static class ConsumerThd implements Runnable { @Override public void run() { - //The producers are shut down first, so keep going until the queue is empty. + // The producers are shut down first, so keep going until the queue is empty. while (!Thread.currentThread().isInterrupted() || queue.size() != 0) { queue.consume(handler); } diff --git a/storm-client/test/jvm/org/apache/storm/utils/MockTupleHelpers.java b/storm-client/test/jvm/org/apache/storm/utils/MockTupleHelpers.java index b3f61821f53..fe6271100e8 100644 --- a/storm-client/test/jvm/org/apache/storm/utils/MockTupleHelpers.java +++ b/storm-client/test/jvm/org/apache/storm/utils/MockTupleHelpers.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-client/test/jvm/org/apache/storm/utils/ReflectionUtilsTest.java b/storm-client/test/jvm/org/apache/storm/utils/ReflectionUtilsTest.java index 71ebc71e40b..dd8325c35ec 100644 --- a/storm-client/test/jvm/org/apache/storm/utils/ReflectionUtilsTest.java +++ b/storm-client/test/jvm/org/apache/storm/utils/ReflectionUtilsTest.java @@ -40,36 +40,43 @@ public void testSchedulerStrategyWithoutWhitelistOnlyAllowsShippedStrategies() { assertEquals(ReflectionUtils.DEFAULT_SCHEDULER_STRATEGIES, e.getAllowedStrategies()); for (String strategy : ReflectionUtils.DEFAULT_SCHEDULER_STRATEGIES) { - // The strategies themselves live in storm-server, so they cannot be loaded here, but they must pass the whitelist check. + // The strategies themselves live in storm-server, so they cannot be loaded here, but + // they must pass the whitelist check. RuntimeException notFound = assertThrows(RuntimeException.class, () -> ReflectionUtils.newSchedulerStrategyInstance(strategy, conf)); - assertTrue(notFound.getCause() instanceof ClassNotFoundException, "unexpected failure for " + strategy); + assertTrue(notFound.getCause() instanceof ClassNotFoundException, + "unexpected failure for " + strategy); } } @Test public void testSchedulerStrategyMessageNamesTheClassAndTheConfigToChange() { DisallowedStrategyException e = assertThrows(DisallowedStrategyException.class, - () -> ReflectionUtils.newSchedulerStrategyInstance("com.example.CustomStrategy", new HashMap<>())); + () -> ReflectionUtils.newSchedulerStrategyInstance("com.example.CustomStrategy", + new HashMap<>())); assertTrue(e.getMessage().contains("com.example.CustomStrategy"), e.getMessage()); - assertTrue(e.getMessage().contains(Config.NIMBUS_SCHEDULER_STRATEGY_CLASS_WHITELIST), e.getMessage()); + assertTrue(e.getMessage().contains(Config.NIMBUS_SCHEDULER_STRATEGY_CLASS_WHITELIST), e + .getMessage()); } @Test public void testSchedulerStrategyWithWhitelist() { Map conf = new HashMap<>(); - conf.put(Config.NIMBUS_SCHEDULER_STRATEGY_CLASS_WHITELIST, Collections.singletonList("java.util.HashMap")); + conf.put(Config.NIMBUS_SCHEDULER_STRATEGY_CLASS_WHITELIST, Collections + .singletonList("java.util.HashMap")); Object instance = ReflectionUtils.newSchedulerStrategyInstance("java.util.HashMap", conf); assertEquals(HashMap.class, instance.getClass()); conf.put(Config.NIMBUS_SCHEDULER_STRATEGY_CLASS_WHITELIST, Collections.emptyList()); - assertThrows(DisallowedStrategyException.class, () -> ReflectionUtils.newSchedulerStrategyInstance("java.util.HashMap", conf)); + assertThrows(DisallowedStrategyException.class, () -> ReflectionUtils + .newSchedulerStrategyInstance("java.util.HashMap", conf)); } @Test public void testDefaultsYamlMatchesTheShippedStrategies() { List fromDefaults = - (List) Utils.readDefaultConfig().get(Config.NIMBUS_SCHEDULER_STRATEGY_CLASS_WHITELIST); + (List) Utils.readDefaultConfig() + .get(Config.NIMBUS_SCHEDULER_STRATEGY_CLASS_WHITELIST); assertEquals(ReflectionUtils.DEFAULT_SCHEDULER_STRATEGIES, fromDefaults); } } diff --git a/storm-client/test/jvm/org/apache/storm/utils/SecurityUtilsTest.java b/storm-client/test/jvm/org/apache/storm/utils/SecurityUtilsTest.java index 4a006a83342..f76195a8646 100644 --- a/storm-client/test/jvm/org/apache/storm/utils/SecurityUtilsTest.java +++ b/storm-client/test/jvm/org/apache/storm/utils/SecurityUtilsTest.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

      - * http://www.apache.org/licenses/LICENSE-2.0 - *

      - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

      http://www.apache.org/licenses/LICENSE-2.0 + * + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -21,16 +27,19 @@ public class SecurityUtilsTest { public void testInferKeyStoreTypeFromPath_Pkcs12Extension() { Assertions.assertEquals("PKCS12", SecurityUtils.inferKeyStoreTypeFromPath("keystore.p12")); Assertions.assertEquals("PKCS12", SecurityUtils.inferKeyStoreTypeFromPath("mykeys.P12")); - Assertions.assertEquals("PKCS12", SecurityUtils.inferKeyStoreTypeFromPath("path/to/keystore.pkcs12")); + Assertions.assertEquals("PKCS12", SecurityUtils + .inferKeyStoreTypeFromPath("path/to/keystore.pkcs12")); Assertions.assertEquals("PKCS12", SecurityUtils.inferKeyStoreTypeFromPath("mykeys.pKCS12")); - Assertions.assertEquals("PKCS12", SecurityUtils.inferKeyStoreTypeFromPath("another/path/to/keystore.pfx")); + Assertions.assertEquals("PKCS12", SecurityUtils + .inferKeyStoreTypeFromPath("another/path/to/keystore.pfx")); } @Test public void testInferKeyStoreTypeFromPath_JksExtension() { Assertions.assertEquals("JKS", SecurityUtils.inferKeyStoreTypeFromPath("keystore.jks")); Assertions.assertEquals("JKS", SecurityUtils.inferKeyStoreTypeFromPath("mykeys.JKS")); - Assertions.assertEquals("JKS", SecurityUtils.inferKeyStoreTypeFromPath("path/to/keystore.jKs")); + Assertions.assertEquals("JKS", SecurityUtils + .inferKeyStoreTypeFromPath("path/to/keystore.jKs")); } @Test @@ -38,7 +47,8 @@ public void testInferKeyStoreTypeFromPath_UnsupportedExtension() { Assertions.assertNull(SecurityUtils.inferKeyStoreTypeFromPath("keystore.pem")); Assertions.assertNull(SecurityUtils.inferKeyStoreTypeFromPath("certificate.crt")); Assertions.assertNull(SecurityUtils.inferKeyStoreTypeFromPath("path/to/keystore.txt")); - Assertions.assertNull(SecurityUtils.inferKeyStoreTypeFromPath("another/path/to/keystore.pem")); + Assertions.assertNull(SecurityUtils + .inferKeyStoreTypeFromPath("another/path/to/keystore.pem")); } @Test diff --git a/storm-client/test/jvm/org/apache/storm/utils/ShellBoltMessageQueueTest.java b/storm-client/test/jvm/org/apache/storm/utils/ShellBoltMessageQueueTest.java index c8f0c525ffc..b08e8ad75ed 100644 --- a/storm-client/test/jvm/org/apache/storm/utils/ShellBoltMessageQueueTest.java +++ b/storm-client/test/jvm/org/apache/storm/utils/ShellBoltMessageQueueTest.java @@ -1,17 +1,26 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.utils; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + import java.util.ArrayList; import java.util.List; import java.util.concurrent.TimeUnit; @@ -19,10 +28,6 @@ import org.apache.storm.shade.com.google.common.collect.Lists; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; - public class ShellBoltMessageQueueTest { @Test public void testPollTaskIdsFirst() throws InterruptedException { @@ -50,7 +55,8 @@ public void testPollWhileThereAreNoDataAvailable() throws InterruptedException { long waitDuration = finish - start; assertNull(msg); - assertTrue(waitDuration >= 1000, "wait duration should be equal or greater than 1000, current: " + waitDuration); + assertTrue(waitDuration >= 1000, + "wait duration should be equal or greater than 1000, current: " + waitDuration); } @Test diff --git a/storm-client/test/jvm/org/apache/storm/utils/ShellUtilsTest.java b/storm-client/test/jvm/org/apache/storm/utils/ShellUtilsTest.java index 746bca2d8ed..fc3942d3483 100644 --- a/storm-client/test/jvm/org/apache/storm/utils/ShellUtilsTest.java +++ b/storm-client/test/jvm/org/apache/storm/utils/ShellUtilsTest.java @@ -1,17 +1,25 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.utils; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; + import java.util.HashMap; import java.util.Map; import org.apache.storm.Config; @@ -19,9 +27,6 @@ import org.apache.storm.task.TopologyContext; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertSame; -import static org.junit.jupiter.api.Assertions.assertThrows; - public class ShellUtilsTest { private Map configureLogHandler(String className) { @@ -80,7 +85,8 @@ public void getLogHandler_notAShellLogHandler() { */ @Test public void getLogHandler_customHandler() { - Map conf = configureLogHandler("org.apache.storm.utils.ShellUtilsTest$CustomShellLogHandler"); + Map conf = + configureLogHandler("org.apache.storm.utils.ShellUtilsTest$CustomShellLogHandler"); ShellLogHandler logHandler = ShellUtils.getLogHandler(conf); assertSame(logHandler.getClass(), CustomShellLogHandler.class); } @@ -94,4 +100,4 @@ public void setUpContext(Class owner, ShellProcess process, TopologyContext c public void log(ShellMsg msg) { } } -} \ No newline at end of file +} diff --git a/storm-client/test/jvm/org/apache/storm/utils/SimpleVersionTest.java b/storm-client/test/jvm/org/apache/storm/utils/SimpleVersionTest.java index 2949ff016fe..9668777afe5 100644 --- a/storm-client/test/jvm/org/apache/storm/utils/SimpleVersionTest.java +++ b/storm-client/test/jvm/org/apache/storm/utils/SimpleVersionTest.java @@ -19,10 +19,10 @@ package org.apache.storm.utils; -import org.junit.jupiter.api.Test; - import static org.junit.jupiter.api.Assertions.assertEquals; +import org.junit.jupiter.api.Test; + public class SimpleVersionTest { @Test @@ -67,4 +67,4 @@ public void testParseStorm0xSnapshot() { assertEquals(10, version.getMinor()); } -} \ No newline at end of file +} diff --git a/storm-client/test/jvm/org/apache/storm/utils/StormBoundedExponentialBackoffRetryTest.java b/storm-client/test/jvm/org/apache/storm/utils/StormBoundedExponentialBackoffRetryTest.java index 36dfacc8059..efc95af5521 100644 --- a/storm-client/test/jvm/org/apache/storm/utils/StormBoundedExponentialBackoffRetryTest.java +++ b/storm-client/test/jvm/org/apache/storm/utils/StormBoundedExponentialBackoffRetryTest.java @@ -1,25 +1,31 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.utils; +import static org.junit.jupiter.api.Assertions.assertTrue; + import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.junit.jupiter.api.Assertions.assertTrue; - public class StormBoundedExponentialBackoffRetryTest { - private static final Logger LOG = LoggerFactory.getLogger(StormBoundedExponentialBackoffRetryTest.class); + private static final Logger LOG = LoggerFactory + .getLogger(StormBoundedExponentialBackoffRetryTest.class); @Test public void testExponentialSleepLargeRetries() { @@ -63,35 +69,36 @@ public void testExponentialSleepSmallMaxTries() { } private void validateSleepTimes(int baseSleepMs, int maxSleepMs, int maxRetries) { - StormBoundedExponentialBackoffRetry retryPolicy = new StormBoundedExponentialBackoffRetry(baseSleepMs, maxSleepMs, maxRetries); + StormBoundedExponentialBackoffRetry retryPolicy = + new StormBoundedExponentialBackoffRetry(baseSleepMs, maxSleepMs, maxRetries); int retryCount = 0; long prevSleepMs = 0; - LOG.info("The baseSleepMs [" + baseSleepMs + "] the maxSleepMs [" + maxSleepMs + - "] the maxRetries [" + maxRetries + "]"); + LOG.info("The baseSleepMs [" + baseSleepMs + "] the maxSleepMs [" + maxSleepMs + + "] the maxRetries [" + maxRetries + "]"); while (retryCount <= maxRetries) { long currSleepMs = retryPolicy.getSleepTimeMs(retryCount, 0); - LOG.info("For retryCount [" + retryCount + "] the previousSleepMs [" + prevSleepMs + - "] the currentSleepMs [" + currSleepMs + "]"); + LOG.info("For retryCount [" + retryCount + "] the previousSleepMs [" + prevSleepMs + + "] the currentSleepMs [" + currSleepMs + "]"); assertTrue((prevSleepMs < currSleepMs) || (currSleepMs == maxSleepMs), - "For retryCount [" + retryCount + "] the previousSleepMs [" + prevSleepMs + - "] is not less than currentSleepMs [" + currSleepMs + "]"); + "For retryCount [" + retryCount + "] the previousSleepMs [" + prevSleepMs + + "] is not less than currentSleepMs [" + currSleepMs + "]"); assertTrue((baseSleepMs <= currSleepMs) || (currSleepMs == maxSleepMs), - "For retryCount [" + retryCount + "] the currentSleepMs [" + currSleepMs + - "] is less than baseSleepMs [" + baseSleepMs + "]."); + "For retryCount [" + retryCount + "] the currentSleepMs [" + currSleepMs + + "] is less than baseSleepMs [" + baseSleepMs + "]."); assertTrue(maxSleepMs >= currSleepMs, - "For retryCount [" + retryCount + "] the currentSleepMs [" + currSleepMs + - "] is greater than maxSleepMs [" + maxSleepMs + "]"); + "For retryCount [" + retryCount + "] the currentSleepMs [" + currSleepMs + + "] is greater than maxSleepMs [" + maxSleepMs + "]"); prevSleepMs = currSleepMs; retryCount++; } int badRetryCount = maxRetries + 10; long currSleepMs = retryPolicy.getSleepTimeMs(badRetryCount, 0); - LOG.info("For badRetryCount [" + badRetryCount + "] the previousSleepMs [" + prevSleepMs + - "] the currentSleepMs [" + currSleepMs + "]"); + LOG.info("For badRetryCount [" + badRetryCount + "] the previousSleepMs [" + prevSleepMs + + "] the currentSleepMs [" + currSleepMs + "]"); assertTrue(maxSleepMs >= currSleepMs, - "For the badRetryCount [" + badRetryCount + "] that's greater than maxRetries [" + - maxRetries + "]the currentSleepMs [" + currSleepMs + "] " + - "is greater than maxSleepMs [" + maxSleepMs + "]"); + "For the badRetryCount [" + badRetryCount + "] that's greater than maxRetries [" + + maxRetries + "]the currentSleepMs [" + currSleepMs + "] " + + "is greater than maxSleepMs [" + maxSleepMs + "]"); } } diff --git a/storm-client/test/jvm/org/apache/storm/utils/ThriftTopologyUtilsTest.java b/storm-client/test/jvm/org/apache/storm/utils/ThriftTopologyUtilsTest.java index 7181da984e9..9d6fc8c6823 100644 --- a/storm-client/test/jvm/org/apache/storm/utils/ThriftTopologyUtilsTest.java +++ b/storm-client/test/jvm/org/apache/storm/utils/ThriftTopologyUtilsTest.java @@ -18,6 +18,10 @@ package org.apache.storm.utils; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + import java.nio.ByteBuffer; import java.util.Set; import org.apache.storm.generated.Bolt; @@ -30,10 +34,6 @@ import org.apache.storm.shade.com.google.common.collect.ImmutableSet; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - public class ThriftTopologyUtilsTest { @Test public void testIsWorkerHook() { @@ -74,13 +74,14 @@ public void testGetComponentIdsWithoutWorkerHook() { ImmutableSet.of("bolt-1", "spout-1"), componentIds, "We expect to get the IDs of the components sans the Worker Hook" - ); + ); } @Test public void testGetComponentCommonWithWorkerHook() { StormTopology stormTopology = genereateStormTopology(true); - ComponentCommon componentCommon = ThriftTopologyUtils.getComponentCommon(stormTopology, "bolt-1"); + ComponentCommon componentCommon = ThriftTopologyUtils.getComponentCommon(stormTopology, + "bolt-1"); assertEquals( new Bolt().get_common(), componentCommon, @@ -90,12 +91,13 @@ public void testGetComponentCommonWithWorkerHook() { @Test public void testGetComponentCommonWithoutWorkerHook() { StormTopology stormTopology = genereateStormTopology(false); - ComponentCommon componentCommon = ThriftTopologyUtils.getComponentCommon(stormTopology, "bolt-1"); + ComponentCommon componentCommon = ThriftTopologyUtils.getComponentCommon(stormTopology, + "bolt-1"); assertEquals( new Bolt().get_common(), componentCommon, "We expect to get bolt-1's common" - ); + ); } private StormTopology genereateStormTopology(boolean withWorkerHook) { diff --git a/storm-client/test/jvm/org/apache/storm/utils/TimeTest.java b/storm-client/test/jvm/org/apache/storm/utils/TimeTest.java index eca418a3d11..750fe1e4bb2 100644 --- a/storm-client/test/jvm/org/apache/storm/utils/TimeTest.java +++ b/storm-client/test/jvm/org/apache/storm/utils/TimeTest.java @@ -1,25 +1,31 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.utils; -import org.apache.storm.utils.Time.SimulatedTime; -import org.junit.jupiter.api.Test; - import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import org.apache.storm.utils.Time.SimulatedTime; +import org.junit.jupiter.api.Test; + public class TimeTest { @Test diff --git a/storm-client/test/jvm/org/apache/storm/utils/UtilsTest.java b/storm-client/test/jvm/org/apache/storm/utils/UtilsTest.java index ed0c82cfe2c..d48fb07d8c0 100644 --- a/storm-client/test/jvm/org/apache/storm/utils/UtilsTest.java +++ b/storm-client/test/jvm/org/apache/storm/utils/UtilsTest.java @@ -18,6 +18,18 @@ package org.apache.storm.utils; +import static org.apache.storm.utils.Utils.handleUncaughtException; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + import java.io.IOException; import java.net.SocketException; import java.nio.charset.StandardCharsets; @@ -32,7 +44,6 @@ import java.util.Set; import java.util.concurrent.ThreadLocalRandom; import java.util.stream.Collectors; - import org.apache.storm.Config; import org.apache.storm.generated.StormTopology; import org.apache.storm.shade.com.google.common.collect.ImmutableList; @@ -48,9 +59,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.apache.storm.utils.Utils.handleUncaughtException; -import static org.junit.jupiter.api.Assertions.*; - public class UtilsTest { public static final Logger LOG = LoggerFactory.getLogger(UtilsTest.class); @@ -95,7 +103,8 @@ private void doParseJvmHeapMemByChildOptsTest(String message, String opt, double doParseJvmHeapMemByChildOptsTest(message, Collections.singletonList(opt), expected); } - private void doParseJvmHeapMemByChildOptsTest(String message, List opts, double expected) { + private void doParseJvmHeapMemByChildOptsTest(String message, List opts, + double expected) { assertEquals(expected, Utils.parseJvmHeapMemByChildOpts(opts, 123.0), 0, message); } @@ -131,7 +140,8 @@ public void parseJvmHeapMemByChildOptsTestNoMatch() { @Test public void parseJvmHeapMemByChildOptsTestNulls() { doParseJvmHeapMemByChildOptsTest("Null value results in default", (String) null, 123.0); - doParseJvmHeapMemByChildOptsTest("Null list results in default", (List) null, 123.0); + doParseJvmHeapMemByChildOptsTest("Null list results in default", (List) null, + 123.0); } @Test @@ -143,7 +153,8 @@ public void parseJvmHeapMemByChildOptsTestExtraChars() { @Test public void parseJvmHeapMemByChildOptsTestFirstMatch() { doParseJvmHeapMemByChildOptsTest("First valid match is used", - Arrays.asList(null, "Xmx1t", "Xmx1g", "Xms1024k Xmx1024k", "Xmx100m"), + Arrays.asList(null, "Xmx1t", "Xmx1g", "Xms1024k Xmx1024k", + "Xmx100m"), 1024.0); } @@ -191,25 +202,30 @@ public void testIsValidConfEmpty() { @Test public void testIsValidConfIdentical() { - Map map1 = ImmutableMap.of("k0", ImmutableList.of(1L, 2L), "k1", ImmutableSet.of('s', 'f'), + Map map1 = ImmutableMap.of("k0", ImmutableList.of(1L, 2L), "k1", + ImmutableSet.of('s', 'f'), "k2", "as"); assertTrue(Utils.isValidConf(map1, map1)); } @Test public void testIsValidConfEqual() { - Map map1 = ImmutableMap.of("k0", ImmutableList.of(1L, 2L), "k1", ImmutableSet.of('s', 'f'), + Map map1 = ImmutableMap.of("k0", ImmutableList.of(1L, 2L), "k1", + ImmutableSet.of('s', 'f'), "k2", "as"); - Map map2 = ImmutableMap.of("k0", ImmutableList.of(1L, 2L), "k1", ImmutableSet.of('s', 'f'), + Map map2 = ImmutableMap.of("k0", ImmutableList.of(1L, 2L), "k1", + ImmutableSet.of('s', 'f'), "k2", "as"); assertTrue(Utils.isValidConf(map1, map2)); // test deep equal } @Test public void testIsValidConfNotEqual() { - Map map1 = ImmutableMap.of("k0", ImmutableList.of(1L, 2L), "k1", ImmutableSet.of('s', 'f'), + Map map1 = ImmutableMap.of("k0", ImmutableList.of(1L, 2L), "k1", + ImmutableSet.of('s', 'f'), "k2", "as"); - Map map3 = ImmutableMap.of("k0", ImmutableList.of(1L, 2L), "k1", ImmutableSet.of('s', 't'), + Map map3 = ImmutableMap.of("k0", ImmutableList.of(1L, 2L), "k1", + ImmutableSet.of('s', 't'), "k2", "as"); assertFalse(Utils.isValidConf(map1, map3)); } @@ -236,7 +252,8 @@ public void checkVersionInfo() { versions.put(key, System.getProperty("java.class.path")); Map conf = new HashMap<>(); conf.put(Config.SUPERVISOR_WORKER_VERSION_CLASSPATH_MAP, versions); - NavigableMap alternativeVersions = Utils.getAlternativeVersionsMap(conf); + NavigableMap alternativeVersions = Utils + .getAlternativeVersionsMap(conf); assertEquals(1, alternativeVersions.size()); IVersionInfo found = alternativeVersions.get(key); assertNotNull(found); @@ -258,7 +275,8 @@ class CycleDetectionScenario { expectedCycles = 0; } - CycleDetectionScenario(String testName, String testDescription, StormTopology topology, int expectedCycles) { + CycleDetectionScenario(String testName, String testDescription, StormTopology topology, + int expectedCycles) { this.testName = testName.replace(' ', '-'); this.testDescription = testDescription; this.topology = topology; @@ -301,7 +319,8 @@ public List createTestScenarios() { tb.setBolt("bolt21", new TestWordCounter(), 10).shuffleGrouping("bolt2"); tb.setBolt("bolt22", new TestWordCounter(), 10).shuffleGrouping("bolt2"); // loop bolt 3 (also connect bolt3 to spout 1) - tb.setBolt("bolt3", new TestWordCounter(), 10).shuffleGrouping("spout1").shuffleGrouping("bolt3"); + tb.setBolt("bolt3", new TestWordCounter(), 10).shuffleGrouping("spout1") + .shuffleGrouping("bolt3"); ret.add(new CycleDetectionScenario(String.format("(%d) One Loop", testNo), "Three level component hierarchy with 1 cycle in bolt3", tb.createTopology(), @@ -320,7 +339,8 @@ public List createTestScenarios() { tb.setBolt("bolt21", new TestWordCounter(), 10).shuffleGrouping("bolt2"); tb.setBolt("bolt22", new TestWordCounter(), 10).shuffleGrouping("bolt2"); // loop bolt 3 -> 4 -> 5 -> 3 (also connect bolt3 to spout1) - tb.setBolt("bolt3", new TestWordCounter(), 10).shuffleGrouping("spout1").shuffleGrouping("bolt5"); + tb.setBolt("bolt3", new TestWordCounter(), 10).shuffleGrouping("spout1") + .shuffleGrouping("bolt5"); tb.setBolt("bolt4", new TestWordCounter(), 10).shuffleGrouping("bolt3"); tb.setBolt("bolt5", new TestWordCounter(), 10).shuffleGrouping("bolt4"); ret.add(new CycleDetectionScenario(String.format("(%d) One Loop", testNo), @@ -341,13 +361,16 @@ public List createTestScenarios() { tb.setBolt("bolt21", new TestWordCounter(), 10).shuffleGrouping("bolt2"); tb.setBolt("bolt22", new TestWordCounter(), 10).shuffleGrouping("bolt2"); // loop bolt 3 -> 4 -> 5 -> 3 (also connect bolt3 to spout1) - tb.setBolt("bolt3", new TestWordCounter(), 10).shuffleGrouping("spout1").shuffleGrouping("bolt5"); + tb.setBolt("bolt3", new TestWordCounter(), 10).shuffleGrouping("spout1") + .shuffleGrouping("bolt5"); tb.setBolt("bolt4", new TestWordCounter(), 10).shuffleGrouping("bolt3"); tb.setBolt("bolt5", new TestWordCounter(), 10).shuffleGrouping("bolt4"); // loop bolt 6 (also connect bolt6 to spout 1) - tb.setBolt("bolt6", new TestWordCounter(), 10).shuffleGrouping("spout1").shuffleGrouping("bolt6"); + tb.setBolt("bolt6", new TestWordCounter(), 10).shuffleGrouping("spout1") + .shuffleGrouping("bolt6"); ret.add(new CycleDetectionScenario(String.format("(%d) Two Loops", testNo), - "Four level component hierarchy with 2 cycles in bolt3,bolt4,bolt5 and bolt6", + "Four level component hierarchy with 2 cycles in bolt3,bolt4,bolt5 " + + "and bolt6", tb.createTopology(), 2)); } @@ -359,12 +382,17 @@ public List createTestScenarios() { tb = new TopologyBuilder(); tb.setSpout("spout1", new TestWordSpout(), 10); tb.setSpout("spout2", new TestWordSpout(), 10); - tb.setBolt("bolt1", new TestWordCounter(), 10).shuffleGrouping("spout1").shuffleGrouping("bolt4"); + tb.setBolt("bolt1", new TestWordCounter(), 10).shuffleGrouping("spout1") + .shuffleGrouping("bolt4"); tb.setBolt("bolt2", new TestWordCounter(), 10).shuffleGrouping("bolt1"); - tb.setBolt("bolt3", new TestWordCounter(), 10).shuffleGrouping("bolt2").shuffleGrouping("bolt4"); - tb.setBolt("bolt4", new TestWordCounter(), 10).shuffleGrouping("bolt3").shuffleGrouping("spout2"); - ret.add(new CycleDetectionScenario(String.format("(%d) Complex Loops#1", testNo), - "Complex cycle (S1 -> B1 -> B2 -> B3 -> B4 <- S2), (B4 -> B3), (B4 -> B1)", + tb.setBolt("bolt3", new TestWordCounter(), 10).shuffleGrouping("bolt2") + .shuffleGrouping("bolt4"); + tb.setBolt("bolt4", new TestWordCounter(), 10).shuffleGrouping("bolt3") + .shuffleGrouping("spout2"); + ret.add(new CycleDetectionScenario(String.format("(%d) Complex Loops#1", + testNo), + "Complex cycle (S1 -> B1 -> B2 -> B3 -> B4 <- S2), (B4 -> B3), (B4 -> " + + "B1)", tb.createTopology(), 1)); } @@ -375,12 +403,16 @@ public List createTestScenarios() { tb = new TopologyBuilder(); tb.setSpout("spout1", new TestWordSpout(), 10); tb.setSpout("spout2", new TestWordSpout(), 10); - tb.setBolt("bolt1", new TestWordCounter(), 10).shuffleGrouping("spout1").shuffleGrouping("bolt4").shuffleGrouping("bolt2"); + tb.setBolt("bolt1", new TestWordCounter(), 10).shuffleGrouping("spout1") + .shuffleGrouping("bolt4").shuffleGrouping("bolt2"); tb.setBolt("bolt2", new TestWordCounter(), 10).shuffleGrouping("bolt1"); - tb.setBolt("bolt3", new TestWordCounter(), 10).shuffleGrouping("bolt2").shuffleGrouping("bolt4"); + tb.setBolt("bolt3", new TestWordCounter(), 10).shuffleGrouping("bolt2") + .shuffleGrouping("bolt4"); tb.setBolt("bolt4", new TestWordCounter(), 10).shuffleGrouping("spout2"); - ret.add(new CycleDetectionScenario(String.format("(%d) Complex Loops#2", testNo), - "Complex cycle 2 (S1 -> B1 <-> B2 -> B3 ), (S2 -> B4 -> B3), (B4 -> B1)", + ret.add(new CycleDetectionScenario(String.format("(%d) Complex Loops#2", + testNo), + "Complex cycle 2 (S1 -> B1 <-> B2 -> B3 ), (S2 -> B4 -> B3), (B4 -> " + + "B1)", tb.createTopology(), 1)); } @@ -389,11 +421,14 @@ public List createTestScenarios() { { testNo++; tb = new TopologyBuilder(); - tb.setBolt("bolt1", new TestWordCounter(), 10).shuffleGrouping("bolt4").shuffleGrouping("bolt2"); + tb.setBolt("bolt1", new TestWordCounter(), 10).shuffleGrouping("bolt4") + .shuffleGrouping("bolt2"); tb.setBolt("bolt2", new TestWordCounter(), 10).shuffleGrouping("bolt1"); - tb.setBolt("bolt3", new TestWordCounter(), 10).shuffleGrouping("bolt2").shuffleGrouping("bolt4"); + tb.setBolt("bolt3", new TestWordCounter(), 10).shuffleGrouping("bolt2") + .shuffleGrouping("bolt4"); tb.setBolt("bolt4", new TestWordCounter(), 10); - ret.add(new CycleDetectionScenario(String.format("(%d) No spout complex loops", testNo), + ret.add(new CycleDetectionScenario(String.format("(%d) No spout complex loops", + testNo), "No Spouts, but with cycles (B1 <-> B2 -> B3 ), (B4 -> B3), (B4 -> B1)", tb.createTopology(), 0)); @@ -410,30 +445,35 @@ public List createTestScenarios() { // topology component and connection counts int spoutCnt = ThreadLocalRandom.current().nextInt(0, maxSpouts) + 1; int boltCnt = ThreadLocalRandom.current().nextInt(0, maxBolts) + 1; - int spoutToBoltConnectionCnt = ThreadLocalRandom.current().nextInt(spoutCnt * boltCnt) + 1; - int boltToBoltConnectionCnt = ThreadLocalRandom.current().nextInt(boltCnt * boltCnt) + 1; + int spoutToBoltConnectionCnt = ThreadLocalRandom.current() + .nextInt(spoutCnt * boltCnt) + 1; + int boltToBoltConnectionCnt = ThreadLocalRandom.current() + .nextInt(boltCnt * boltCnt) + 1; Map boltDeclarers = new HashMap<>(); - for (int iSpout = 0 ; iSpout < spoutCnt ; iSpout++) { + for (int iSpout = 0; iSpout < spoutCnt; iSpout++) { tb.setSpout("spout" + iSpout, new TestWordSpout(), 10); } - for (int iBolt = 0 ; iBolt < boltCnt ; iBolt++) { - boltDeclarers.put(iBolt, tb.setBolt("bolt" + iBolt, new TestWordCounter(), 10)); + for (int iBolt = 0; iBolt < boltCnt; iBolt++) { + boltDeclarers.put(iBolt, tb.setBolt("bolt" + iBolt, new TestWordCounter(), + 10)); } // spout to bolt connections - for (int i = 0 ; i < spoutToBoltConnectionCnt ; i++) { + for (int i = 0; i < spoutToBoltConnectionCnt; i++) { int iSpout = ThreadLocalRandom.current().nextInt(0, spoutCnt); int iBolt = ThreadLocalRandom.current().nextInt(0, boltCnt); boltDeclarers.get(iBolt).shuffleGrouping("spout" + iSpout); } // bolt to bolt connections - for (int i = 0 ; i < boltToBoltConnectionCnt ; i++) { + for (int i = 0; i < boltToBoltConnectionCnt; i++) { int iBolt1 = ThreadLocalRandom.current().nextInt(0, boltCnt); int iBolt2 = ThreadLocalRandom.current().nextInt(0, boltCnt); boltDeclarers.get(iBolt2).shuffleGrouping("bolt" + iBolt1); } - ret.add(new CycleDetectionScenario(String.format("(%d) Random Topo#%d", testNo, iRandTest), - String.format("Random topology #%d, spouts=%d, bolts=%d, connections: fromSpouts=%d/fromBolts=%d", + ret.add(new CycleDetectionScenario(String.format("(%d) Random Topo#%d", testNo, + iRandTest), + String.format("Random topology #%d, spouts=%d, bolts=%d, connections: " + + "fromSpouts=%d/fromBolts=%d", iRandTest, spoutCnt, boltCnt, spoutToBoltConnectionCnt, boltToBoltConnectionCnt), tb.createTopology(), -1)); @@ -442,10 +482,12 @@ public List createTestScenarios() { return ret; } } + List testFailures = new ArrayList<>(); new CycleDetectionScenario().createTestScenarios().forEach(x -> { - LOG.info("==================== Running Test Scenario: {} =======================", x.testName); + LOG.info("==================== Running Test Scenario: {} =======================", + x.testName); LOG.info("{}: {}", x.testName, x.testDescription); List> loops = Utils.findComponentCycles(x.topology, x.testName); @@ -459,7 +501,8 @@ public List createTestScenarios() { } if (loops.size() != x.expectedCycles) { testFailures.add( - String.format("Test \"%s\" failed, detected cycles=%d does not match expected=%d for \"%s\"", + String.format("Test \"%s\" failed, detected cycles=%d does not match " + + "expected=%d for \"%s\"", x.testName, loops.size(), x.expectedCycles, x.testDescription)); if (!loops.isEmpty()) { testFailures.add( @@ -483,39 +526,44 @@ public List createTestScenarios() { @Test public void testHandleUncaughtExceptionSwallowsCausedAndDerivedExceptions() { - Set> allowedExceptions = new HashSet<>(Arrays.asList(new Class[]{ IOException.class })); + Set> allowedExceptions = new HashSet<>(Arrays + .asList(new Class[]{ IOException.class })); try { handleUncaughtException(new IOException(), allowedExceptions, false); - } catch(Throwable unexpected) { + } catch (Throwable unexpected) { fail("Should have swallowed IOException!", unexpected); } try { handleUncaughtException(new SocketException(), allowedExceptions, false); - } catch(Throwable unexpected) { + } catch (Throwable unexpected) { fail("Should have swallowed Throwable derived from IOException!", unexpected); } try { - handleUncaughtException(new TTransportException(new IOException()), allowedExceptions, false); - } catch(Throwable unexpected) { + handleUncaughtException(new TTransportException(new IOException()), allowedExceptions, + false); + } catch (Throwable unexpected) { fail("Should have swallowed Throwable caused by an IOException!", unexpected); } try { - handleUncaughtException(new TTransportException(new SocketException()), allowedExceptions, false); - } catch(Throwable unexpected) { - fail("Should have swallowed Throwable caused by a Throwable derived from IOException!", unexpected); + handleUncaughtException(new TTransportException(new SocketException()), + allowedExceptions, false); + } catch (Throwable unexpected) { + fail("Should have swallowed Throwable caused by a Throwable derived from IOException!", + unexpected); } Throwable t = new NullPointerException(); - String expectationMessage = "Should have thrown an Error() with a cause of NullPointerException"; + String expectationMessage = + "Should have thrown an Error() with a cause of NullPointerException"; try { handleUncaughtException(t, allowedExceptions, false); fail(expectationMessage); - } catch(Error expected) { + } catch (Error expected) { assertEquals(expected.getCause(), t, expectationMessage); - } catch(Throwable unexpected) { + } catch (Throwable unexpected) { fail(expectationMessage, unexpected); } } @@ -796,7 +844,7 @@ public void findOneValidIPredicateValidCollectionShouldPass() { IPredicate mockPredicate = Mockito.mock(IPredicate.class); Mockito.when(mockPredicate.test(Mockito.any())).thenReturn(true); - Map map = Map.of(1,1); + Map map = Map.of(1, 1); Integer integer = 1; assertEquals(integer, Utils.findOne(mockPredicate, map)); } @@ -811,7 +859,7 @@ public void findOneValidIPredicateValidCollection2ShouldPass() { Integer integer2 = 2; Mockito.lenient().when(mockPredicate.test(integer)).thenReturn(false); Mockito.lenient().when(mockPredicate.test(integer2)).thenReturn(true); - Map map = Map.of(1,1,2,2); + Map map = Map.of(1, 1, 2, 2); assertEquals(integer2, Utils.findOne(mockPredicate, map)); } @@ -822,7 +870,7 @@ public void findOneNotCorrectIPredicateValidCollectionShouldPass() { IPredicate mockPredicate = Mockito.mock(IPredicate.class); Mockito.when(mockPredicate.test(Mockito.any())).thenReturn(false); - Map map = Map.of(1,1); + Map map = Map.of(1, 1); assertNull(Utils.findOne(mockPredicate, map)); } } diff --git a/storm-client/test/jvm/org/apache/storm/windowing/WaterMarkEventGeneratorTest.java b/storm-client/test/jvm/org/apache/storm/windowing/WaterMarkEventGeneratorTest.java index 5479ab8d6d5..ff7720deb3e 100644 --- a/storm-client/test/jvm/org/apache/storm/windowing/WaterMarkEventGeneratorTest.java +++ b/storm-client/test/jvm/org/apache/storm/windowing/WaterMarkEventGeneratorTest.java @@ -1,17 +1,27 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.windowing; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; @@ -22,12 +32,8 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - /** - * Unit tests for {@link WaterMarkEventGenerator} + * Unit tests for {@link WaterMarkEventGenerator}. */ public class WaterMarkEventGeneratorTest { WaterMarkEventGenerator waterMarkEventGenerator; @@ -48,7 +54,8 @@ public void add(Event event) { }; // set watermark interval to a high value and trigger manually to fix timing issues waterMarkEventGenerator = new WaterMarkEventGenerator<>(windowManager, 100000, 5, - Collections.singleton(streamId("s1"))); + Collections + .singleton(streamId("s1"))); waterMarkEventGenerator.start(); } diff --git a/storm-client/test/jvm/org/apache/storm/windowing/WindowManagerTest.java b/storm-client/test/jvm/org/apache/storm/windowing/WindowManagerTest.java index 7d0cae96c7b..93bebbd405b 100644 --- a/storm-client/test/jvm/org/apache/storm/windowing/WindowManagerTest.java +++ b/storm-client/test/jvm/org/apache/storm/windowing/WindowManagerTest.java @@ -1,17 +1,31 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.windowing; +import static org.apache.storm.topology.base.BaseWindowedBolt.Duration; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.empty; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -23,16 +37,8 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import static org.apache.storm.topology.base.BaseWindowedBolt.Duration; -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.empty; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - /** - * Unit tests for {@link WindowManager} + * Unit tests for {@link WindowManager}. */ public class WindowManagerTest { private WindowManager windowManager; @@ -52,7 +58,8 @@ public void tearDown() { @Test public void testCountBasedWindow() { EvictionPolicy evictionPolicy = new CountEvictionPolicy<>(5); - TriggerPolicy triggerPolicy = new CountTriggerPolicy<>(2, windowManager, evictionPolicy); + TriggerPolicy triggerPolicy = new CountTriggerPolicy<>(2, windowManager, + evictionPolicy); triggerPolicy.start(); windowManager.setEvictionPolicy(evictionPolicy); windowManager.setTriggerPolicy(triggerPolicy); @@ -94,7 +101,8 @@ public void testExpireThreshold() { int threshold = WindowManager.EXPIRE_EVENTS_THRESHOLD; int windowLength = 5; windowManager.setEvictionPolicy(new CountEvictionPolicy<>(5)); - TriggerPolicy triggerPolicy = new TimeTriggerPolicy<>(new Duration(1, TimeUnit.HOURS).value, windowManager); + TriggerPolicy triggerPolicy = new TimeTriggerPolicy<>(new Duration(1, + TimeUnit.HOURS).value, windowManager); triggerPolicy.start(); windowManager.setTriggerPolicy(triggerPolicy); for (int i : seq(1, 5)) { @@ -112,59 +120,72 @@ public void testExpireThreshold() { assertEquals(seq(1, threshold - windowLength), listener.onExpiryEvents); } - private void testEvictBeforeWatermarkForWatermarkEvictionPolicy(EvictionPolicy watermarkEvictionPolicy, int windowLength) { + private void testEvictBeforeWatermarkForWatermarkEvictionPolicy(EvictionPolicy watermarkEvictionPolicy, int windowLength) { /* - * The watermark eviction policy must not evict tuples until the first watermark has been received. - * The policies can't make a meaningful decision prior to the first watermark, so the safe decision + * The watermark eviction policy must not evict tuples until the first watermark has been + * received. + * The policies can't make a meaningful decision prior to the first watermark, so the safe + * decision * is to postpone eviction. */ int threshold = WindowManager.EXPIRE_EVENTS_THRESHOLD; windowManager.setEvictionPolicy(watermarkEvictionPolicy); - WatermarkCountTriggerPolicy triggerPolicy = new WatermarkCountTriggerPolicy<>(windowLength, windowManager, + WatermarkCountTriggerPolicy triggerPolicy = + new WatermarkCountTriggerPolicy<>(windowLength, windowManager, watermarkEvictionPolicy, windowManager); triggerPolicy.start(); windowManager.setTriggerPolicy(triggerPolicy); for (int i : seq(1, threshold)) { windowManager.add(i, i); } - assertThat("The watermark eviction policies should never evict events before the first watermark is received", + assertThat("The watermark eviction policies should never evict events before the first " + + "watermark is received", listener.onExpiryEvents, is(empty())); windowManager.add(new WaterMarkEvent<>(threshold)); // The events should be put in a window when the first watermark is received assertEquals(seq(1, threshold), listener.onActivationEvents); - //Now add some more events and a new watermark, and check that the previous events are expired + // Now add some more events and a new watermark, and check that the previous events are + // expired for (int i : seq(threshold + 1, threshold * 2)) { windowManager.add(i, i); } windowManager.add(new WaterMarkEvent<>(threshold + windowLength + 1)); - //All the events should be expired when the next watermark is received - assertThat("All the events should be expired after the second watermark", listener.onExpiryEvents, equalTo(seq(1, threshold))); + // All the events should be expired when the next watermark is received + assertThat("All the events should be expired after the second watermark", + listener.onExpiryEvents, equalTo(seq(1, threshold))); } @Test public void testExpireThresholdWithWatermarkCountEvictionPolicy() { int windowLength = WindowManager.EXPIRE_EVENTS_THRESHOLD; - EvictionPolicy watermarkCountEvictionPolicy = new WatermarkCountEvictionPolicy<>(windowLength); - testEvictBeforeWatermarkForWatermarkEvictionPolicy(watermarkCountEvictionPolicy, windowLength); + EvictionPolicy watermarkCountEvictionPolicy = + new WatermarkCountEvictionPolicy<>(windowLength); + testEvictBeforeWatermarkForWatermarkEvictionPolicy(watermarkCountEvictionPolicy, + windowLength); } @Test public void testExpireThresholdWithWatermarkTimeEvictionPolicy() { int windowLength = WindowManager.EXPIRE_EVENTS_THRESHOLD; - EvictionPolicy watermarkTimeEvictionPolicy = new WatermarkTimeEvictionPolicy<>(windowLength); - testEvictBeforeWatermarkForWatermarkEvictionPolicy(watermarkTimeEvictionPolicy, windowLength); + EvictionPolicy watermarkTimeEvictionPolicy = + new WatermarkTimeEvictionPolicy<>(windowLength); + testEvictBeforeWatermarkForWatermarkEvictionPolicy(watermarkTimeEvictionPolicy, + windowLength); } @Test public void testTimeBasedWindow() { - EvictionPolicy evictionPolicy = new TimeEvictionPolicy<>(new Duration(1, TimeUnit.SECONDS).value); + EvictionPolicy evictionPolicy = new TimeEvictionPolicy<>(new Duration(1, + TimeUnit.SECONDS).value); windowManager.setEvictionPolicy(evictionPolicy); /* * Don't wait for Timetrigger to fire since this could lead to timing issues in unit tests. * Set it to a large value and trigger manually. */ TriggerPolicy triggerPolicy = - new TimeTriggerPolicy<>(new Duration(1, TimeUnit.DAYS).value, windowManager, evictionPolicy); + new TimeTriggerPolicy<>(new Duration(1, TimeUnit.DAYS).value, windowManager, + evictionPolicy); triggerPolicy.start(); windowManager.setTriggerPolicy(triggerPolicy); long now = System.currentTimeMillis(); @@ -182,7 +203,8 @@ public void testTimeBasedWindow() { assertEquals(50, listener.onExpiryEvents.size()); // add more events with past ts - for (int i : seq(WindowManager.EXPIRE_EVENTS_THRESHOLD + 1, WindowManager.EXPIRE_EVENTS_THRESHOLD + 100)) { + for (int i : seq(WindowManager.EXPIRE_EVENTS_THRESHOLD + 1, + WindowManager.EXPIRE_EVENTS_THRESHOLD + 100)) { windowManager.add(i, now - 1000); } // simulate the time trigger by setting the reference time and invoking onTrigger() manually @@ -191,19 +213,24 @@ public void testTimeBasedWindow() { // 100 events with past ts should expire assertEquals(100, listener.onExpiryEvents.size()); - assertEquals(seq(WindowManager.EXPIRE_EVENTS_THRESHOLD + 1, WindowManager.EXPIRE_EVENTS_THRESHOLD + 100), + assertEquals(seq(WindowManager.EXPIRE_EVENTS_THRESHOLD + 1, + WindowManager.EXPIRE_EVENTS_THRESHOLD + 100), listener.onExpiryEvents); List activationsEvents = seq(51, WindowManager.EXPIRE_EVENTS_THRESHOLD); assertEquals(seq(51, WindowManager.EXPIRE_EVENTS_THRESHOLD), listener.onActivationEvents); - assertEquals(seq(51, WindowManager.EXPIRE_EVENTS_THRESHOLD), listener.onActivationNewEvents); - // activation expired list should contain even the ones expired due to EXPIRE_EVENTS_THRESHOLD + assertEquals(seq(51, WindowManager.EXPIRE_EVENTS_THRESHOLD), + listener.onActivationNewEvents); + // activation expired list should contain even the ones expired due to + // EXPIRE_EVENTS_THRESHOLD List expiredList = seq(1, 50); - expiredList.addAll(seq(WindowManager.EXPIRE_EVENTS_THRESHOLD + 1, WindowManager.EXPIRE_EVENTS_THRESHOLD + 100)); + expiredList.addAll(seq(WindowManager.EXPIRE_EVENTS_THRESHOLD + 1, + WindowManager.EXPIRE_EVENTS_THRESHOLD + 100)); assertEquals(expiredList, listener.onActivationExpiredEvents); listener.clear(); // add more events with current ts - List newEvents = seq(WindowManager.EXPIRE_EVENTS_THRESHOLD + 101, WindowManager.EXPIRE_EVENTS_THRESHOLD + 200); + List newEvents = seq(WindowManager.EXPIRE_EVENTS_THRESHOLD + 101, + WindowManager.EXPIRE_EVENTS_THRESHOLD + 200); for (int i : newEvents) { windowManager.add(i, now); } @@ -219,13 +246,15 @@ public void testTimeBasedWindow() { @Test public void testTimeBasedWindowExpiry() { - EvictionPolicy evictionPolicy = new TimeEvictionPolicy<>(new Duration(100, TimeUnit.MILLISECONDS).value); + EvictionPolicy evictionPolicy = new TimeEvictionPolicy<>(new Duration(100, + TimeUnit.MILLISECONDS).value); windowManager.setEvictionPolicy(evictionPolicy); /* * Don't wait for Timetrigger to fire since this could lead to timing issues in unit tests. * Set it to a large value and trigger manually. */ - TriggerPolicy triggerPolicy = new TimeTriggerPolicy<>(new Duration(1, TimeUnit.DAYS).value, windowManager); + TriggerPolicy triggerPolicy = new TimeTriggerPolicy<>(new Duration(1, + TimeUnit.DAYS).value, windowManager); triggerPolicy.start(); windowManager.setTriggerPolicy(triggerPolicy); long now = System.currentTimeMillis(); @@ -258,7 +287,8 @@ public void testTimeBasedWindowExpiry() { public void testTumblingWindow() { EvictionPolicy evictionPolicy = new CountEvictionPolicy<>(3); windowManager.setEvictionPolicy(evictionPolicy); - TriggerPolicy triggerPolicy = new CountTriggerPolicy<>(3, windowManager, evictionPolicy); + TriggerPolicy triggerPolicy = new CountTriggerPolicy<>(3, windowManager, + evictionPolicy); triggerPolicy.start(); windowManager.setTriggerPolicy(triggerPolicy); windowManager.add(1); @@ -287,7 +317,8 @@ public void testTumblingWindow() { public void testEventTimeBasedWindow() { EvictionPolicy evictionPolicy = new WatermarkTimeEvictionPolicy<>(20); windowManager.setEvictionPolicy(evictionPolicy); - TriggerPolicy triggerPolicy = new WatermarkTimeTriggerPolicy<>(10, windowManager, evictionPolicy, windowManager); + TriggerPolicy triggerPolicy = new WatermarkTimeTriggerPolicy<>(10, + windowManager, evictionPolicy, windowManager); triggerPolicy.start(); windowManager.setTriggerPolicy(triggerPolicy); @@ -353,7 +384,8 @@ public void testEventTimeBasedWindow() { public void testCountBasedWindowWithEventTs() { EvictionPolicy evictionPolicy = new WatermarkCountEvictionPolicy<>(3); windowManager.setEvictionPolicy(evictionPolicy); - TriggerPolicy triggerPolicy = new WatermarkTimeTriggerPolicy<>(10, windowManager, evictionPolicy, windowManager); + TriggerPolicy triggerPolicy = new WatermarkTimeTriggerPolicy<>(10, + windowManager, evictionPolicy, windowManager); triggerPolicy.start(); windowManager.setTriggerPolicy(triggerPolicy); @@ -392,7 +424,8 @@ public void testCountBasedWindowWithEventTs() { public void testCountBasedTriggerWithEventTs() { EvictionPolicy evictionPolicy = new WatermarkTimeEvictionPolicy<>(20); windowManager.setEvictionPolicy(evictionPolicy); - TriggerPolicy triggerPolicy = new WatermarkCountTriggerPolicy<>(3, windowManager, evictionPolicy, windowManager); + TriggerPolicy triggerPolicy = new WatermarkCountTriggerPolicy<>(3, + windowManager, evictionPolicy, windowManager); triggerPolicy.start(); windowManager.setTriggerPolicy(triggerPolicy); @@ -432,7 +465,8 @@ public void testCountBasedTriggerWithEventTs() { public void testCountBasedTumblingWithSameEventTs() { EvictionPolicy evictionPolicy = new WatermarkCountEvictionPolicy<>(2); windowManager.setEvictionPolicy(evictionPolicy); - TriggerPolicy triggerPolicy = new WatermarkCountTriggerPolicy<>(2, windowManager, evictionPolicy, windowManager); + TriggerPolicy triggerPolicy = new WatermarkCountTriggerPolicy<>(2, + windowManager, evictionPolicy, windowManager); triggerPolicy.start(); windowManager.setTriggerPolicy(triggerPolicy); @@ -460,7 +494,8 @@ public void testCountBasedTumblingWithSameEventTs() { public void testCountBasedSlidingWithSameEventTs() { EvictionPolicy evictionPolicy = new WatermarkCountEvictionPolicy<>(5); windowManager.setEvictionPolicy(evictionPolicy); - TriggerPolicy triggerPolicy = new WatermarkCountTriggerPolicy<>(2, windowManager, evictionPolicy, windowManager); + TriggerPolicy triggerPolicy = new WatermarkCountTriggerPolicy<>(2, + windowManager, evictionPolicy, windowManager); triggerPolicy.start(); windowManager.setTriggerPolicy(triggerPolicy); @@ -489,7 +524,8 @@ public void testCountBasedSlidingWithSameEventTs() { public void testEventTimeLag() { EvictionPolicy evictionPolicy = new WatermarkTimeEvictionPolicy<>(20, 5); windowManager.setEvictionPolicy(evictionPolicy); - TriggerPolicy triggerPolicy = new WatermarkTimeTriggerPolicy<>(10, windowManager, evictionPolicy, windowManager); + TriggerPolicy triggerPolicy = new WatermarkTimeTriggerPolicy<>(10, + windowManager, evictionPolicy, windowManager); triggerPolicy.start(); windowManager.setTriggerPolicy(triggerPolicy); @@ -514,7 +550,8 @@ public void testEventTimeLag() { @Test public void testScanStop() { final Set eventsScanned = new HashSet<>(); - EvictionPolicy evictionPolicy = new WatermarkTimeEvictionPolicy(20, 5) { + EvictionPolicy evictionPolicy = new WatermarkTimeEvictionPolicy(20, + 5) { @Override public Action evict(Event event) { @@ -524,7 +561,8 @@ public Action evict(Event event) { }; windowManager.setEvictionPolicy(evictionPolicy); - TriggerPolicy triggerPolicy = new WatermarkTimeTriggerPolicy<>(10, windowManager, evictionPolicy, windowManager); + TriggerPolicy triggerPolicy = new WatermarkTimeTriggerPolicy<>(10, + windowManager, evictionPolicy, windowManager); triggerPolicy.start(); windowManager.setTriggerPolicy(triggerPolicy); @@ -585,7 +623,8 @@ public void onExpiry(List events) { } @Override - public void onActivation(List events, List newEvents, List expired, Long timestamp) { + public void onActivation(List events, List newEvents, + List expired, Long timestamp) { onActivationEvents = events; allOnActivationEvents.add(events); onActivationNewEvents = newEvents; diff --git a/storm-client/test/jvm/org/apache/storm/windowing/persistence/WindowStateTest.java b/storm-client/test/jvm/org/apache/storm/windowing/persistence/WindowStateTest.java index 8f3e89c43a1..7e3b7f69de6 100644 --- a/storm-client/test/jvm/org/apache/storm/windowing/persistence/WindowStateTest.java +++ b/storm-client/test/jvm/org/apache/storm/windowing/persistence/WindowStateTest.java @@ -1,17 +1,27 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

      - * http://www.apache.org/licenses/LICENSE-2.0 - *

      - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

      http://www.apache.org/licenses/LICENSE-2.0 + * + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.windowing.persistence; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.mockito.AdditionalAnswers.returnsArgAt; + import java.util.ArrayList; import java.util.Collections; import java.util.Deque; @@ -33,12 +43,8 @@ import org.mockito.junit.jupiter.MockitoExtension; import org.mockito.stubbing.Answer; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.mockito.AdditionalAnswers.returnsArgAt; - /** - * Unit tests for {@link WindowState} + * Unit tests for {@link WindowState}. */ @ExtendWith(MockitoExtension.class) public class WindowStateTest { @@ -69,10 +75,12 @@ public void testAdd() throws Exception { ws.add(getEvent(i)); } // 5 partitions evicted to window state - Mockito.verify(windowState, Mockito.times(5)).put(longCaptor.capture(), windowValuesCaptor.capture()); + Mockito.verify(windowState, Mockito.times(5)).put(longCaptor.capture(), windowValuesCaptor + .capture()); assertEquals(5, longCaptor.getAllValues().size()); // each evicted partition has MAX_EVENTS_PER_PARTITION - windowValuesCaptor.getAllValues().forEach(wp -> assertEquals(WindowState.MAX_PARTITION_EVENTS, wp.size())); + windowValuesCaptor.getAllValues() + .forEach(wp -> assertEquals(WindowState.MAX_PARTITION_EVENTS, wp.size())); // last partition is not evicted assertFalse(longCaptor.getAllValues().contains(partitions - 1)); } @@ -207,4 +215,4 @@ private WindowState getWindowState(int maxEvents) { return new WindowState<>(windowState, partitionIdsState, systemState, supplier, maxEvents); } -} \ No newline at end of file +} diff --git a/storm-core/pom.xml b/storm-core/pom.xml index 1f0ef1d7421..3840e05d5d2 100644 --- a/storm-core/pom.xml +++ b/storm-core/pom.xml @@ -233,6 +233,16 @@ + + org.openrewrite.maven + rewrite-maven-plugin + + + + org.codehaus.gmavenplus + gmavenplus-plugin + + org.apache.maven.plugins maven-checkstyle-plugin diff --git a/storm-core/src/jvm/org/apache/storm/command/Activate.java b/storm-core/src/jvm/org/apache/storm/command/Activate.java index b619da027c5..d1930b3df81 100644 --- a/storm-core/src/jvm/org/apache/storm/command/Activate.java +++ b/storm-core/src/jvm/org/apache/storm/command/Activate.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-core/src/jvm/org/apache/storm/command/AdminCommands.java b/storm-core/src/jvm/org/apache/storm/command/AdminCommands.java index 83c77ffd85e..2fe8809cdcc 100644 --- a/storm-core/src/jvm/org/apache/storm/command/AdminCommands.java +++ b/storm-core/src/jvm/org/apache/storm/command/AdminCommands.java @@ -7,9 +7,9 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed + *

      Unless required by applicable law or agreed to in writing, software distributed * under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions @@ -71,13 +71,16 @@ public interface AdminCommand { private static class RemoveCorruptTopologies implements AdminCommand { @Override public void run(String[] args, Map conf, String command) throws Exception { - try (BlobStore nimbusBlobStore = ServerUtils.getNimbusBlobStore(conf, NimbusInfo.fromConf(conf), null)) { + try (BlobStore nimbusBlobStore = ServerUtils.getNimbusBlobStore(conf, NimbusInfo + .fromConf(conf), null)) { IStormClusterState stormClusterState = ClusterUtils.mkStormClusterState(conf, new ClusterStateContext(DaemonType.NIMBUS, conf)); - Set blobStoreTopologyIds = nimbusBlobStore.filterAndListKeys(key -> ConfigUtils.getIdFromBlobKey(key)); + Set blobStoreTopologyIds = nimbusBlobStore + .filterAndListKeys(key -> ConfigUtils.getIdFromBlobKey(key)); Set activeTopologyIds = new HashSet<>(stormClusterState.activeStorms()); - Sets.SetView diffTopology = Sets.difference(activeTopologyIds, blobStoreTopologyIds); + Sets.SetView diffTopology = Sets.difference(activeTopologyIds, + blobStoreTopologyIds); LOG.info("active-topology-ids [{}] blob-topology-ids [{}] diff-topology [{}]", activeTopologyIds, blobStoreTopologyIds, diffTopology); for (String corruptId : diffTopology) { @@ -97,7 +100,8 @@ private static class CredentialsDebug implements AdminCommand { @Override public void run(String[] args, Map conf, String command) throws Exception { // We are pretending to be nimbus here. - IStormClusterState state = ClusterUtils.mkStormClusterState(conf, new ClusterStateContext(DaemonType.NIMBUS, conf)); + IStormClusterState state = ClusterUtils.mkStormClusterState(conf, + new ClusterStateContext(DaemonType.NIMBUS, conf)); for (String topologyId : args) { System.out.println(topologyId + ":"); Credentials creds = state.credentials(topologyId, null); @@ -118,6 +122,7 @@ public void printCliHelp(String command, PrintStream out) { /** * Print value in a human readable format. + * * @param value what to print. * @return a human readable string */ @@ -167,7 +172,7 @@ private static String keyStr(String key) { } private static void prettyPrintKeyValue(String key, Object o, int depth, StringBuilder out) { - //Special cases for storm... + // Special cases for storm... if ("json_conf".equals(key) && o instanceof String) { try { o = Utils.parseJson((String) o); @@ -208,10 +213,12 @@ public void run(String[] args, Map conf, String command) throws File f = new File(arg); if (f.exists()) { topo = Utils.deserialize(FileUtils.readFileToByteArray(f), StormTopology.class); - } else { //assume it is a topology id + } else { // assume it is a topology id final String key = ConfigUtils.masterStormCodeKey(arg); - try (BlobStore store = ServerUtils.getNimbusBlobStore(conf, NimbusInfo.fromConf(conf), null)) { - topo = Utils.deserialize(store.readBlob(key, Nimbus.NIMBUS_SUBJECT), StormTopology.class); + try (BlobStore store = ServerUtils.getNimbusBlobStore(conf, NimbusInfo + .fromConf(conf), null)) { + topo = Utils.deserialize(store.readBlob(key, Nimbus.NIMBUS_SUBJECT), + StormTopology.class); } } @@ -229,7 +236,8 @@ public void printCliHelp(String command, PrintStream out) { private static class PrintSupervisors implements AdminCommand { @Override public void run(String[] args, Map conf, String command) throws Exception { - IStormClusterState stormClusterState = ClusterUtils.mkStormClusterState(conf, new ClusterStateContext(DaemonType.NIMBUS, conf)); + IStormClusterState stormClusterState = ClusterUtils.mkStormClusterState(conf, + new ClusterStateContext(DaemonType.NIMBUS, conf)); Map infos = stormClusterState.allSupervisorInfo(); if (args.length <= 0) { for (Map.Entry entry : infos.entrySet()) { @@ -247,14 +255,16 @@ public void run(String[] args, Map conf, String command) throws @Override public void printCliHelp(String command, PrintStream out) { out.println(command + " [supervisor_id]*:"); - out.println("\tPrint a human readable version of the supervisor info(s). Print all if no args"); + out.println("\tPrint a human readable version of the supervisor info(s). Print all if " + + "no args"); } } private static class PrintAssignments implements AdminCommand { @Override public void run(String[] args, Map conf, String command) throws Exception { - IStormClusterState stormClusterState = ClusterUtils.mkStormClusterState(conf, new ClusterStateContext(DaemonType.NIMBUS, conf)); + IStormClusterState stormClusterState = ClusterUtils.mkStormClusterState(conf, + new ClusterStateContext(DaemonType.NIMBUS, conf)); stormClusterState.syncRemoteAssignments(null); stormClusterState.syncRemoteIds(null); stormClusterState.setAssignmentsBackendSynchronized(); @@ -275,7 +285,8 @@ public void run(String[] args, Map conf, String command) throws @Override public void printCliHelp(String command, PrintStream out) { out.println(command + " [topology_id]*:"); - out.println("\tPrint a human readable version of the topologies assignment info(s). Print all if no args"); + out.println("\tPrint a human readable version of the topologies assignment info(s). " + + "Print all if no args"); } } @@ -299,7 +310,8 @@ public void run(String[] args, Map conf, String command) { @Override public void printCliHelp(String command, PrintStream out) { out.println(command + " [...]:"); - out.println("\tPrint a help message about one or more commands. If not commands are given, print all"); + out.println("\tPrint a help message about one or more commands. If not commands are " + + "given, print all"); } } diff --git a/storm-core/src/jvm/org/apache/storm/command/BasicDrpcClient.java b/storm-core/src/jvm/org/apache/storm/command/BasicDrpcClient.java index f995d93eaff..9c5f7e3c52b 100644 --- a/storm-core/src/jvm/org/apache/storm/command/BasicDrpcClient.java +++ b/storm-core/src/jvm/org/apache/storm/command/BasicDrpcClient.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -32,6 +38,7 @@ private static void runAndPrint(DRPCClient drpc, String func, String arg) throws /** * Main entry point for the basic DRPC client. + * * @param args command line arguments to be parsed * @throws Exception on errors */ @@ -45,7 +52,8 @@ public static void main(String[] args) throws Exception { try (DRPCClient drpc = DRPCClient.getConfiguredClient(conf)) { if (function == null) { if (funcAndArgs.size() % 2 != 0) { - LOG.error("If no -f is supplied arguments need to be in the form [function arg]. This has {} args", funcAndArgs.size()); + LOG.error("If no -f is supplied arguments need to be in the form [function " + + "arg]. This has {} args", funcAndArgs.size()); System.exit(-1); } for (int i = 0; i < funcAndArgs.size(); i += 2) { diff --git a/storm-core/src/jvm/org/apache/storm/command/Blobstore.java b/storm-core/src/jvm/org/apache/storm/command/Blobstore.java index d2bbb7ab51b..0ea4a55cb1c 100644 --- a/storm-core/src/jvm/org/apache/storm/command/Blobstore.java +++ b/storm-core/src/jvm/org/apache/storm/command/Blobstore.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -217,7 +223,8 @@ private static void setAclCli(String[] args) throws Exception { private static void replicationCli(String[] args) throws Exception { if (args.length == 0) { - throw new IllegalArgumentException("replication command needs at least subcommand as parameter."); + throw new IllegalArgumentException("replication command needs at least subcommand as " + + "parameter."); } final String subCommand = args[0]; final String[] newArgs = Arrays.copyOfRange(args, 1, args.length); @@ -228,7 +235,8 @@ public void run(ClientBlobStore blobStore) throws Exception { switch (subCommand) { case "--read": if (newArgs.length == 0) { - throw new IllegalArgumentException("replication --read needs key as parameter."); + throw new IllegalArgumentException("replication --read needs key as " + + "parameter."); } String key = newArgs[0]; @@ -241,11 +249,13 @@ public void run(ClientBlobStore blobStore) throws Exception { break; default: - throw new RuntimeException("" + subCommand + " is not a supported blobstore command"); + throw new RuntimeException("" + subCommand + + " is not a supported blobstore command"); } } - private void updateReplicationFactor(ClientBlobStore blobStore, String[] args) throws Exception { + private void updateReplicationFactor(ClientBlobStore blobStore, + String[] args) throws Exception { Map cl = CLI.opt("r", "replication-factor", null, CLI.AS_INT) .arg("key", CLI.FIRST_WINS) .parse(args); @@ -280,7 +290,8 @@ static void readBlob(final String key, final OutputStream os) throws Exception { }); } - static void createBlobFromStream(final String key, final InputStream is, final SettableBlobMeta meta) throws Exception { + static void createBlobFromStream(final String key, final InputStream is, + final SettableBlobMeta meta) throws Exception { ClientBlobStore.withConfiguredClient(blobStore -> { AtomicOutputStream os = blobStore.createBlob(key, meta); copyInputStreamToBlobOutputStream(is, os); @@ -294,7 +305,8 @@ static void updateBlobFromStream(final String key, final InputStream is) throws }); } - static void copyInputStreamToBlobOutputStream(InputStream is, AtomicOutputStream os) throws IOException { + static void copyInputStreamToBlobOutputStream(InputStream is, + AtomicOutputStream os) throws IOException { try { IOUtils.copy(is, os); os.close(); diff --git a/storm-core/src/jvm/org/apache/storm/command/CLI.java b/storm-core/src/jvm/org/apache/storm/command/CLI.java index d1112838f88..e6c025a4246 100644 --- a/storm-core/src/jvm/org/apache/storm/command/CLI.java +++ b/storm-core/src/jvm/org/apache/storm/command/CLI.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -70,9 +76,12 @@ public class CLI { /** * Add an option to be parsed. - * @param shortName the short single character name of the option (no `-` character proceeds it). + * + * @param shortName the short single character name of the option (no `-` character proceeds + * it). * @param longName the multi character name of the option (no `--` characters proceed it). - * @param defaultValue the value that will be returned of the command if none is given. null if none is given. + * @param defaultValue the value that will be returned of the command if none is given. null if + * none is given. * @return a builder to be used to continue creating the command line. */ public static CLIBuilder opt(String shortName, String longName, Object defaultValue) { @@ -81,32 +90,45 @@ public static CLIBuilder opt(String shortName, String longName, Object defaultVa /** * Add an option to be parsed. - * @param shortName the short single character name of the option (no `-` character proceeds it). + * + * @param shortName the short single character name of the option (no `-` character proceeds + * it). * @param longName the multi character name of the option (no `--` characters proceed it). - * @param defaultValue the value that will be returned of the command if none is given. null if none is given. - * @param parse an optional function to transform the string to something else. If null a NOOP is used. + * @param defaultValue the value that will be returned of the command if none is given. null if + * none is given. + * @param parse an optional function to transform the string to something else. If null a NOOP + * is used. * @return a builder to be used to continue creating the command line. */ - public static CLIBuilder opt(String shortName, String longName, Object defaultValue, Parse parse) { + public static CLIBuilder opt(String shortName, String longName, Object defaultValue, + Parse parse) { return new CLIBuilder().opt(shortName, longName, defaultValue, parse); } /** * Add an option to be parsed. - * @param shortName the short single character name of the option (no `-` character proceeds it). + * + * @param shortName the short single character name of the option (no `-` character proceeds + * it). * @param longName the multi character name of the option (no `--` characters proceed it). - * @param defaultValue the value that will be returned of the command if none is given. null if none is given. - * @param parse an optional function to transform the string to something else. If null a NOOP is used. - * @param assoc an association command to decide what to do if the option appears multiple times. If null LAST_WINS is used. + * @param defaultValue the value that will be returned of the command if none is given. null if + * none is given. + * @param parse an optional function to transform the string to something else. If null a NOOP + * is used. + * @param assoc an association command to decide what to do if the option appears multiple + * times. If null LAST_WINS is used. * @return a builder to be used to continue creating the command line. */ - public static CLIBuilder opt(String shortName, String longName, Object defaultValue, Parse parse, Assoc assoc) { + public static CLIBuilder opt(String shortName, String longName, Object defaultValue, + Parse parse, Assoc assoc) { return new CLIBuilder().opt(shortName, longName, defaultValue, parse, assoc); } /** * Add a boolean option that enables something. - * @param shortName the short single character name of the option (no `-` character proceeds it). + * + * @param shortName the short single character name of the option (no `-` character proceeds + * it). * @param longName the multi character name of the option (no `--` characters proceed it). * @return a builder to be used to continue creating the command line. */ @@ -116,6 +138,7 @@ public static CLIBuilder boolOpt(String shortName, String longName) { /** * Add a named argument. + * * @param name the name of the argument. * @return a builder to be used to continue creating the command line. */ @@ -125,8 +148,10 @@ public static CLIBuilder arg(String name) { /** * Add a named argument. + * * @param name the name of the argument. - * @param assoc an association command to decide what to do if the argument appears multiple times. If null INTO_LIST is used. + * @param assoc an association command to decide what to do if the argument appears multiple + * times. If null INTO_LIST is used. * @return a builder to be used to continue creating the command line. */ public static CLIBuilder arg(String name, Assoc assoc) { @@ -135,8 +160,10 @@ public static CLIBuilder arg(String name, Assoc assoc) { /** * Add a named argument. + * * @param name the name of the argument. - * @param parse an optional function to transform the string to something else. If null a NOOP is used. + * @param parse an optional function to transform the string to something else. If null a NOOP + * is used. * @return a builder to be used to continue creating the command line. */ public static CLIBuilder arg(String name, Parse parse) { @@ -145,9 +172,12 @@ public static CLIBuilder arg(String name, Parse parse) { /** * Add a named argument. + * * @param name the name of the argument. - * @param parse an optional function to transform the string to something else. If null a NOOP is used. - * @param assoc an association command to decide what to do if the argument appears multiple times. If null INTO_LIST is used. + * @param parse an optional function to transform the string to something else. If null a NOOP + * is used. + * @param assoc an association command to decide what to do if the argument appears multiple + * times. If null INTO_LIST is used. * @return a builder to be used to continue creating the command line. */ public static CLIBuilder arg(String name, Parse parse, Assoc assoc) { @@ -156,6 +186,7 @@ public static CLIBuilder arg(String name, Parse parse, Assoc assoc) { /** * Add a named argument that is optional. + * * @param name the name of the argument. * @return a builder to be used to continue creating the command line. */ @@ -165,8 +196,10 @@ public static CLIBuilder optionalArg(String name) { /** * Add a named argument that is optional. + * * @param name the name of the argument. - * @param assoc an association command to decide what to do if the argument appears multiple times. If null INTO_LIST is used. + * @param assoc an association command to decide what to do if the argument appears multiple + * times. If null INTO_LIST is used. * @return a builder to be used to continue creating the command line. */ public static CLIBuilder optionalArg(String name, Assoc assoc) { @@ -175,8 +208,10 @@ public static CLIBuilder optionalArg(String name, Assoc assoc) { /** * Add a named argument that is optional. + * * @param name the name of the argument. - * @param parse an optional function to transform the string to something else. If null a NOOP is used. + * @param parse an optional function to transform the string to something else. If null a NOOP + * is used. * @return a builder to be used to continue creating the command line. */ public static CLIBuilder optionalArg(String name, Parse parse) { @@ -185,9 +220,12 @@ public static CLIBuilder optionalArg(String name, Parse parse) { /** * Add a named argument that is optional. + * * @param name the name of the argument. - * @param parse an optional function to transform the string to something else. If null a NOOP is used. - * @param assoc an association command to decide what to do if the argument appears multiple times. If null INTO_LIST is used. + * @param parse an optional function to transform the string to something else. If null a NOOP + * is used. + * @param assoc an association command to decide what to do if the argument appears multiple + * times. If null INTO_LIST is used. * @return a builder to be used to continue creating the command line. */ public static CLIBuilder optionalArg(String name, Parse parse, Assoc assoc) { @@ -197,6 +235,7 @@ public static CLIBuilder optionalArg(String name, Parse parse, Assoc assoc) { public interface Parse { /** * Parse a String to the type you want it to be. + * * @param value the String to parse * @return the parsed value */ @@ -206,6 +245,7 @@ public interface Parse { public interface Assoc { /** * Associate a value into something else. + * * @param current what to put value into, will be null if no values have been added yet. * @param value what to add * @return the result of combining the two @@ -221,7 +261,8 @@ private static class Opt { final Assoc assoc; final boolean noValue; - Opt(String shortName, String longName, Object defaultValue, Parse parse, Assoc assoc, boolean noValue) { + Opt(String shortName, String longName, Object defaultValue, Parse parse, Assoc assoc, + boolean noValue) { this.shortName = shortName; this.longName = longName; this.defaultValue = defaultValue; @@ -259,9 +300,12 @@ public static class CLIBuilder { /** * Add an option to be parsed. - * @param shortName the short single character name of the option (no `-` character proceeds it). + * + * @param shortName the short single character name of the option (no `-` character proceeds + * it). * @param longName the multi character name of the option (no `--` characters proceed it). - * @param defaultValue the value that will be returned of the command if none is given. null if none is given. + * @param defaultValue the value that will be returned of the command if none is given. null + * if none is given. * @return a builder to be used to continue creating the command line. */ public CLIBuilder opt(String shortName, String longName, Object defaultValue) { @@ -270,10 +314,14 @@ public CLIBuilder opt(String shortName, String longName, Object defaultValue) { /** * Add an option to be parsed. - * @param shortName the short single character name of the option (no `-` character proceeds it). + * + * @param shortName the short single character name of the option (no `-` character proceeds + * it). * @param longName the multi character name of the option (no `--` characters proceed it). - * @param defaultValue the value that will be returned of the command if none is given. null if none is given. - * @param parse an optional function to transform the string to something else. If null a NOOP is used. + * @param defaultValue the value that will be returned of the command if none is given. null + * if none is given. + * @param parse an optional function to transform the string to something else. If null a + * NOOP is used. * @return a builder to be used to continue creating the command line. */ public CLIBuilder opt(String shortName, String longName, Object defaultValue, Parse parse) { @@ -282,21 +330,29 @@ public CLIBuilder opt(String shortName, String longName, Object defaultValue, Pa /** * Add an option to be parsed. - * @param shortName the short single character name of the option (no `-` character proceeds it). + * + * @param shortName the short single character name of the option (no `-` character proceeds + * it). * @param longName the multi character name of the option (no `--` characters proceed it). - * @param defaultValue the value that will be returned of the command if none is given. null if none is given. - * @param parse an optional function to transform the string to something else. If null a NOOP is used. - * @param assoc an association command to decide what to do if the option appears multiple times. If null LAST_WINS is used. + * @param defaultValue the value that will be returned of the command if none is given. null + * if none is given. + * @param parse an optional function to transform the string to something else. If null a + * NOOP is used. + * @param assoc an association command to decide what to do if the option appears multiple + * times. If null LAST_WINS is used. * @return a builder to be used to continue creating the command line. */ - public CLIBuilder opt(String shortName, String longName, Object defaultValue, Parse parse, Assoc assoc) { + public CLIBuilder opt(String shortName, String longName, Object defaultValue, Parse parse, + Assoc assoc) { opts.add(new Opt(shortName, longName, defaultValue, parse, assoc, false)); return this; } /** * Add a boolean option that enables something. - * @param shortName the short single character name of the option (no `-` character proceeds it). + * + * @param shortName the short single character name of the option (no `-` character proceeds + * it). * @param longName the multi character name of the option (no `--` characters proceed it). * @return a builder to be used to continue creating the command line. */ @@ -307,6 +363,7 @@ public CLIBuilder boolOpt(String shortName, String longName) { /** * Add a named argument. + * * @param name the name of the argument. * @return a builder to be used to continue creating the command line. */ @@ -316,8 +373,10 @@ public CLIBuilder arg(String name) { /** * Add a named argument. + * * @param name the name of the argument. - * @param assoc an association command to decide what to do if the argument appears multiple times. If null INTO_LIST is used. + * @param assoc an association command to decide what to do if the argument appears multiple + * times. If null INTO_LIST is used. * @return a builder to be used to continue creating the command line. */ public CLIBuilder arg(String name, Assoc assoc) { @@ -326,8 +385,10 @@ public CLIBuilder arg(String name, Assoc assoc) { /** * Add a named argument. + * * @param name the name of the argument. - * @param parse an optional function to transform the string to something else. If null a NOOP is used. + * @param parse an optional function to transform the string to something else. If null a + * NOOP is used. * @return a builder to be used to continue creating the command line. */ public CLIBuilder arg(String name, Parse parse) { @@ -336,14 +397,18 @@ public CLIBuilder arg(String name, Parse parse) { /** * Add a named argument. + * * @param name the name of the argument. - * @param parse an optional function to transform the string to something else. If null a NOOP is used. - * @param assoc an association command to decide what to do if the argument appears multiple times. If null INTO_LIST is used. + * @param parse an optional function to transform the string to something else. If null a + * NOOP is used. + * @param assoc an association command to decide what to do if the argument appears multiple + * times. If null INTO_LIST is used. * @return a builder to be used to continue creating the command line. */ public CLIBuilder arg(String name, Parse parse, Assoc assoc) { if (!optionalArgs.isEmpty()) { - throw new IllegalStateException("Cannot have a required argument after adding in an optional argument"); + throw new IllegalStateException("Cannot have a required argument after adding in " + + "an optional argument"); } args.add(new Arg(name, parse, assoc)); return this; @@ -351,6 +416,7 @@ public CLIBuilder arg(String name, Parse parse, Assoc assoc) { /** * Add a named argument that is optional. + * * @param name the name of the argument. * @return a builder to be used to continue creating the command line. */ @@ -360,8 +426,10 @@ public CLIBuilder optionalArg(String name) { /** * Add a named argument that is optional. + * * @param name the name of the argument. - * @param assoc an association command to decide what to do if the argument appears multiple times. If null INTO_LIST is used. + * @param assoc an association command to decide what to do if the argument appears multiple + * times. If null INTO_LIST is used. * @return a builder to be used to continue creating the command line. */ public CLIBuilder optionalArg(String name, Assoc assoc) { @@ -370,8 +438,10 @@ public CLIBuilder optionalArg(String name, Assoc assoc) { /** * Add a named argument that is optional. + * * @param name the name of the argument. - * @param parse an optional function to transform the string to something else. If null a NOOP is used. + * @param parse an optional function to transform the string to something else. If null a + * NOOP is used. * @return a builder to be used to continue creating the command line. */ public CLIBuilder optionalArg(String name, Parse parse) { @@ -380,9 +450,12 @@ public CLIBuilder optionalArg(String name, Parse parse) { /** * Add a named argument that is optional. + * * @param name the name of the argument. - * @param parse an optional function to transform the string to something else. If null a NOOP is used. - * @param assoc an association command to decide what to do if the argument appears multiple times. If null INTO_LIST is used. + * @param parse an optional function to transform the string to something else. If null a + * NOOP is used. + * @param assoc an association command to decide what to do if the argument appears multiple + * times. If null INTO_LIST is used. * @return a builder to be used to continue creating the command line. */ public CLIBuilder optionalArg(String name, Parse parse, Assoc assoc) { @@ -392,20 +465,25 @@ public CLIBuilder optionalArg(String name, Parse parse, Assoc assoc) { /** * Parse the command line arguments. + * * @param rawArgs the string arguments to be parsed. * @return The parsed command line. * opts will be stored under the short argument name. - * args will be stored under the argument name, unless no arguments are configured, and then they will be stored under "ARGS". - * The last argument configured is greedy and is used to process all remaining command line arguments. + * args will be stored under the argument name, unless no arguments are configured, and + * then they will be stored under "ARGS". + * The last argument configured is greedy and is used to process all remaining command + * line arguments. * @throws Exception on any error. */ public Map parse(String... rawArgs) throws Exception { Options options = new Options(); for (Opt opt : opts) { if (opt.noValue) { - options.addOption(Option.builder(opt.shortName).longOpt(opt.longName).hasArg(false).build()); + options.addOption(Option.builder(opt.shortName).longOpt(opt.longName) + .hasArg(false).build()); } else { - options.addOption(Option.builder(opt.shortName).longOpt(opt.longName).hasArg().build()); + options.addOption(Option.builder(opt.shortName).longOpt(opt.longName).hasArg() + .build()); } } DefaultParser parser = new DefaultParser(); diff --git a/storm-core/src/jvm/org/apache/storm/command/ConfigValue.java b/storm-core/src/jvm/org/apache/storm/command/ConfigValue.java index eb6e4510b61..290870c8ff0 100644 --- a/storm-core/src/jvm/org/apache/storm/command/ConfigValue.java +++ b/storm-core/src/jvm/org/apache/storm/command/ConfigValue.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -31,6 +37,7 @@ private ConfigValue() { /** * Read the topology config and return the value for the given key. + * * @param args - an array of length 1 containing the key to fetch. */ public static void main(final String[] args) { diff --git a/storm-core/src/jvm/org/apache/storm/command/Deactivate.java b/storm-core/src/jvm/org/apache/storm/command/Deactivate.java index b8bbf59ae0f..d45909fc8b8 100644 --- a/storm-core/src/jvm/org/apache/storm/command/Deactivate.java +++ b/storm-core/src/jvm/org/apache/storm/command/Deactivate.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-core/src/jvm/org/apache/storm/command/DevZookeeper.java b/storm-core/src/jvm/org/apache/storm/command/DevZookeeper.java index 52531c4f363..1ddf522afbc 100644 --- a/storm-core/src/jvm/org/apache/storm/command/DevZookeeper.java +++ b/storm-core/src/jvm/org/apache/storm/command/DevZookeeper.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-core/src/jvm/org/apache/storm/command/GetErrors.java b/storm-core/src/jvm/org/apache/storm/command/GetErrors.java index f6ff4146a53..f18e5202b8d 100644 --- a/storm-core/src/jvm/org/apache/storm/command/GetErrors.java +++ b/storm-core/src/jvm/org/apache/storm/command/GetErrors.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -27,6 +33,7 @@ public class GetErrors { /** * Only get errors for a topology. + * * @param args Used to accept the topology name. * @throws Exception on errors. */ @@ -55,9 +62,11 @@ public void run(Nimbus.Iface client) throws Exception { System.out.println(JSONValue.toJSONString(outputMap)); } - private Map getComponentErrors(Map> topologyErrors) { + private Map getComponentErrors(Map> topologyErrors) { Map componentErrorMap = new HashMap<>(); - for (Map.Entry> compNameToCompErrors : topologyErrors.entrySet()) { + for (Map.Entry> compNameToCompErrors : topologyErrors + .entrySet()) { String compName = compNameToCompErrors.getKey(); List compErrors = compNameToCompErrors.getValue(); if (compErrors != null && !compErrors.isEmpty()) { diff --git a/storm-core/src/jvm/org/apache/storm/command/HealthCheck.java b/storm-core/src/jvm/org/apache/storm/command/HealthCheck.java index 5a35fd302b1..083ca32c43b 100644 --- a/storm-core/src/jvm/org/apache/storm/command/HealthCheck.java +++ b/storm-core/src/jvm/org/apache/storm/command/HealthCheck.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-core/src/jvm/org/apache/storm/command/Heartbeats.java b/storm-core/src/jvm/org/apache/storm/command/Heartbeats.java index 8603155ac8c..2e5c3531b06 100644 --- a/storm-core/src/jvm/org/apache/storm/command/Heartbeats.java +++ b/storm-core/src/jvm/org/apache/storm/command/Heartbeats.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -72,7 +78,8 @@ private static void handleGetCommand(IStateStorage cluster, String path) { String message; byte[] hb = cluster.get_worker_hb(path, false); if (hb != null) { - Map heartbeatMap = StatsUtil.convertZkWorkerHb(Utils.deserialize(hb, ClusterWorkerHeartbeat.class)); + Map heartbeatMap = StatsUtil.convertZkWorkerHb(Utils.deserialize(hb, + ClusterWorkerHeartbeat.class)); message = JSONValue.toJSONString(heartbeatMap); } else { message = "No Heartbeats found"; diff --git a/storm-core/src/jvm/org/apache/storm/command/KillTopology.java b/storm-core/src/jvm/org/apache/storm/command/KillTopology.java index c193ab0cd12..4f19d637e8b 100644 --- a/storm-core/src/jvm/org/apache/storm/command/KillTopology.java +++ b/storm-core/src/jvm/org/apache/storm/command/KillTopology.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -55,7 +61,8 @@ public static void main(String[] args) throws Exception { } if (names.isEmpty()) { - throw new RuntimeException("Failed to successfully kill " + errorCount + " topologies."); + throw new RuntimeException("Failed to successfully kill " + errorCount + + " topologies."); } // Wait this many seconds after deactivating topology before killing @@ -74,7 +81,8 @@ public static void main(String[] args) throws Exception { } catch (Exception e) { errorCount += 1; if (continueOnError) { - LOG.error("Caught error killing topology '{}'; continuing as -i was passed.", name, e); + LOG.error("Caught error killing topology '{}'; continuing as -i was " + + "passed.", name, e); } else { throw e; } @@ -83,7 +91,8 @@ public static void main(String[] args) throws Exception { // If we failed to kill any topology, still exit with failure status if (errorCount > 0) { - throw new RuntimeException("Failed to successfully kill " + errorCount + " topologies."); + throw new RuntimeException("Failed to successfully kill " + errorCount + + " topologies."); } }); } diff --git a/storm-core/src/jvm/org/apache/storm/command/KillWorkers.java b/storm-core/src/jvm/org/apache/storm/command/KillWorkers.java index 11affbd8e05..7a0c53fd78b 100644 --- a/storm-core/src/jvm/org/apache/storm/command/KillWorkers.java +++ b/storm-core/src/jvm/org/apache/storm/command/KillWorkers.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -21,7 +27,8 @@ public class KillWorkers { public static void main(String[] args) throws Exception { Map conf = Utils.readStormConfig(); - try (Supervisor supervisor = new Supervisor(conf, null, new StandaloneSupervisor(), new StormMetricsRegistry())) { + try (Supervisor supervisor = new Supervisor(conf, null, new StandaloneSupervisor(), + new StormMetricsRegistry())) { supervisor.shutdownAllWorkers(null, null); } } diff --git a/storm-core/src/jvm/org/apache/storm/command/ListTopologies.java b/storm-core/src/jvm/org/apache/storm/command/ListTopologies.java index b25ddd8b99f..ee650a6dafb 100644 --- a/storm-core/src/jvm/org/apache/storm/command/ListTopologies.java +++ b/storm-core/src/jvm/org/apache/storm/command/ListTopologies.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -36,7 +42,9 @@ public static void main(String[] args) throws Exception { "Uptime_secs", "Topology_Id", "Owner"); - System.out.println("----------------------------------------------------------------------------------------"); + System.out + .println("----------------------------------------------------------------" + + "------------------------"); for (TopologySummary topology : topologies) { System.out.printf(MSG_FORMAT, topology.get_name(), topology.get_status(), topology.get_num_tasks(), topology.get_num_workers(), diff --git a/storm-core/src/jvm/org/apache/storm/command/Monitor.java b/storm-core/src/jvm/org/apache/storm/command/Monitor.java index 9e6972bf4fe..e4b774f2bd9 100644 --- a/storm-core/src/jvm/org/apache/storm/command/Monitor.java +++ b/storm-core/src/jvm/org/apache/storm/command/Monitor.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-core/src/jvm/org/apache/storm/command/Rebalance.java b/storm-core/src/jvm/org/apache/storm/command/Rebalance.java index 30eff08e575..59c0cb9f833 100644 --- a/storm-core/src/jvm/org/apache/storm/command/Rebalance.java +++ b/storm-core/src/jvm/org/apache/storm/command/Rebalance.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -24,7 +30,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; - public class Rebalance { private static final Logger LOG = LoggerFactory.getLogger(Rebalance.class); @@ -33,7 +38,8 @@ public static void main(String[] args) throws Exception { Map cl = CLI.opt("w", "wait", null, CLI.AS_INT) .opt("n", "num-workers", null, CLI.AS_INT) .opt("e", "executor", null, new ExecutorParser(), CLI.INTO_MAP) - .opt("r", "resources", null, new ResourcesParser(), CLI.INTO_MAP) + .opt("r", "resources", null, new ResourcesParser(), + CLI.INTO_MAP) .opt("t", "topology-conf", null, new ConfParser(), CLI.INTO_MAP) .arg("topologyName", CLI.FIRST_WINS) .parse(args); @@ -52,20 +58,23 @@ public static void main(String[] args) throws Exception { if (null != numExecutors) { rebalanceOptions.set_num_executors(numExecutors); } - Map> resourceOverrides = (Map>) cl.get("r"); + Map> resourceOverrides = (Map>) cl + .get("r"); if (null != resourceOverrides) { rebalanceOptions.set_topology_resources_overrides(resourceOverrides); } Map confOverrides = (Map) cl.get("t"); - Map jvmOpts = Utils.readCommandLineOpts(); // values in -Dstorm.options (originally -c in storm.py) + Map jvmOpts = Utils + .readCommandLineOpts(); // values in -Dstorm.options (originally -c in storm.py) if (jvmOpts != null && !jvmOpts.isEmpty()) { if (confOverrides == null) { confOverrides = jvmOpts; } else { confOverrides.putAll(jvmOpts); // override with values obtained from -Dstorm.options } - LOG.info("Rebalancing topology with overrides {}", JSONObject.toJSONString(confOverrides)); + LOG.info("Rebalancing topology with overrides {}", JSONObject + .toJSONString(confOverrides)); } if (null != confOverrides) { @@ -99,9 +108,10 @@ public Object parse(String value) { throw new RuntimeException("No arguments found for topology resources override!"); } try { - //This is a bit ugly The JSON we are expecting should be in the form + // This is a bit ugly The JSON we are expecting should be in the form // {"component": {"resource": value, ...}, ...} - // But because value is coming from JSON it is going to be a Number, and we want it to be a Double. + // But because value is coming from JSON it is going to be a Number, and we want it + // to be a Double. // So the goal is to go through each entry and update it accordingly Map> ret = new HashMap<>(); for (Map.Entry compEntry : Utils.parseJson(value).entrySet()) { @@ -132,7 +142,8 @@ public Object parse(String value) { return result; } catch (Throwable ex) { throw new IllegalArgumentException( - format("Failed to parse '%s' correctly. Expected in = format", value), ex); + format("Failed to parse '%s' correctly. Expected in = " + + "format", value), ex); } } } diff --git a/storm-core/src/jvm/org/apache/storm/command/SetLogLevel.java b/storm-core/src/jvm/org/apache/storm/command/SetLogLevel.java index dcbeceef99a..7e59fc06e0e 100644 --- a/storm-core/src/jvm/org/apache/storm/command/SetLogLevel.java +++ b/storm-core/src/jvm/org/apache/storm/command/SetLogLevel.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -29,8 +35,10 @@ public class SetLogLevel { private static final Logger LOG = LoggerFactory.getLogger(SetLogLevel.class); public static void main(String[] args) throws Exception { - Map cl = CLI.opt("l", "log-setting", null, new LogLevelsParser(LogLevelAction.UPDATE), CLI.INTO_MAP) - .opt("r", "remove-log-setting", null, new LogLevelsParser(LogLevelAction.REMOVE), CLI.INTO_MAP) + Map cl = CLI.opt("l", "log-setting", null, + new LogLevelsParser(LogLevelAction.UPDATE), CLI.INTO_MAP) + .opt("r", "remove-log-setting", null, + new LogLevelsParser(LogLevelAction.REMOVE), CLI.INTO_MAP) .arg("topologyName", CLI.FIRST_WINS) .parse(args); final String topologyName = (String) cl.get("topologyName"); diff --git a/storm-core/src/jvm/org/apache/storm/command/ShellSubmission.java b/storm-core/src/jvm/org/apache/storm/command/ShellSubmission.java index 9c5564cabae..ac88b3048ca 100644 --- a/storm-core/src/jvm/org/apache/storm/command/ShellSubmission.java +++ b/storm-core/src/jvm/org/apache/storm/command/ShellSubmission.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -14,7 +20,6 @@ import java.util.Arrays; import java.util.Map; - import org.apache.commons.lang3.ArrayUtils; import org.apache.storm.StormSubmitter; import org.apache.storm.generated.NimbusSummary; @@ -38,7 +43,8 @@ public static void main(String[] args) throws Exception { String host = ns.get_host(); int port = ns.get_port(); String jarPath = StormSubmitter.submitJar(conf, args[0]); - String[] newArgs = (String[]) ArrayUtils.addAll(Arrays.copyOfRange(args, 1, args.length), + String[] newArgs = (String[]) ArrayUtils.addAll(Arrays.copyOfRange(args, 1, + args.length), new String[]{host, String.valueOf(port), jarPath}); ServerUtils.execCommand(newArgs); } diff --git a/storm-core/src/jvm/org/apache/storm/command/UploadCredentials.java b/storm-core/src/jvm/org/apache/storm/command/UploadCredentials.java index 12d76cefebb..6bb2d3baa5a 100644 --- a/storm-core/src/jvm/org/apache/storm/command/UploadCredentials.java +++ b/storm-core/src/jvm/org/apache/storm/command/UploadCredentials.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -36,6 +42,7 @@ public class UploadCredentials { /** * Uploads credentials for a topology. + * * @param args To accept topology name. * @throws Exception on errors. */ @@ -71,18 +78,20 @@ public static void main(String[] args) throws Exception { } Map topologyConf = new HashMap<>(); - //Try to get the topology conf from nimbus, so we can reuse it. + // Try to get the topology conf from nimbus, so we can reuse it. try (NimbusClient nc = NimbusClient.Builder.withConf(new HashMap<>()).build()) { Nimbus.Iface client = nc.getClient(); TopologySummary topo = client.getTopologySummaryByName(topologyName); - //We found the topology, lets get the conf + // We found the topology, lets get the conf String topologyId = topo.get_id(); - topologyConf = (Map) JSONValue.parse(client.getTopologyConf(topologyId)); + topologyConf = (Map) JSONValue.parse(client + .getTopologyConf(topologyId)); LOG.info("Using topology conf from {} as basis for getting new creds", topologyId); Map commandLine = Utils.readCommandLineOpts(); List clCreds = (List) commandLine.get(Config.TOPOLOGY_AUTO_CREDENTIALS); - List topoCreds = (List) topologyConf.get(Config.TOPOLOGY_AUTO_CREDENTIALS); + List topoCreds = (List) topologyConf + .get(Config.TOPOLOGY_AUTO_CREDENTIALS); if (clCreds != null) { Set extra = new HashSet<>(clCreds); @@ -90,16 +99,18 @@ public static void main(String[] args) throws Exception { extra.removeAll(topoCreds); } if (!extra.isEmpty()) { - LOG.warn("The topology {} is not using {} but they were included here.", topologyId, extra); + LOG.warn("The topology {} is not using {} but they were included here.", + topologyId, extra); } - //Now check for autoCreds that are missing from the command line, but only if the + // Now check for autoCreds that are missing from the command line, but only if the // command line is used. if (topoCreds != null) { Set missing = new HashSet<>(topoCreds); missing.removeAll(clCreds); if (!missing.isEmpty()) { - LOG.warn("The topology {} is using {} but they were not included here.", topologyId, missing); + LOG.warn("The topology {} is using {} but they were not included here.", + topologyId, missing); } } } @@ -114,12 +125,14 @@ public static void main(String[] args) throws Exception { */ topologyConf.remove("java.security.auth.login.config"); topologyConf.remove(Config.NIMBUS_THRIFT_CLIENT_USE_TLS); - // Nimbus masks credentials before serving a conf, so these entries hold no usable value here. + // Nimbus masks credentials before serving a conf, so these entries hold no usable value + // here. // Dropping them lets the client's own configuration supply them, e.g. TLS store passwords. topologyConf.keySet().removeIf(ConfigUtils::isCredentialKey); boolean throwExceptionForEmptyCreds = (boolean) cl.get("e"); - boolean hasCreds = StormSubmitter.pushCredentials(topologyName, topologyConf, credentialsMap, (String) cl.get("u")); + boolean hasCreds = StormSubmitter.pushCredentials(topologyName, topologyConf, + credentialsMap, (String) cl.get("u")); if (!hasCreds && throwExceptionForEmptyCreds) { String message = "No credentials were uploaded for " + topologyName; LOG.error(message); diff --git a/storm-core/src/jvm/org/apache/storm/planner/CompoundSpout.java b/storm-core/src/jvm/org/apache/storm/planner/CompoundSpout.java index 6db0de74538..8af4c42ada0 100644 --- a/storm-core/src/jvm/org/apache/storm/planner/CompoundSpout.java +++ b/storm-core/src/jvm/org/apache/storm/planner/CompoundSpout.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-core/src/jvm/org/apache/storm/planner/CompoundTask.java b/storm-core/src/jvm/org/apache/storm/planner/CompoundTask.java index c5b4b0b2e51..2139e6a2b14 100644 --- a/storm-core/src/jvm/org/apache/storm/planner/CompoundTask.java +++ b/storm-core/src/jvm/org/apache/storm/planner/CompoundTask.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-core/src/jvm/org/apache/storm/planner/TaskBundle.java b/storm-core/src/jvm/org/apache/storm/planner/TaskBundle.java index 6052b6de70b..b4858a8ef33 100644 --- a/storm-core/src/jvm/org/apache/storm/planner/TaskBundle.java +++ b/storm-core/src/jvm/org/apache/storm/planner/TaskBundle.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -15,7 +21,6 @@ import java.io.Serializable; import org.apache.storm.task.IBolt; - public class TaskBundle implements Serializable { public IBolt task; public int componentId; diff --git a/storm-core/src/jvm/org/apache/storm/shade/org/apache/zookeeper/ZkCli.java b/storm-core/src/jvm/org/apache/storm/shade/org/apache/zookeeper/ZkCli.java index bba06931b94..ffe011a5769 100644 --- a/storm-core/src/jvm/org/apache/storm/shade/org/apache/zookeeper/ZkCli.java +++ b/storm-core/src/jvm/org/apache/storm/shade/org/apache/zookeeper/ZkCli.java @@ -7,16 +7,16 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed + *

      Unless required by applicable law or agreed to in writing, software distributed * under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions * and limitations under the License. */ -//This is a hack to allow ZooKeeperMain to be called by this command. +// This is a hack to allow ZooKeeperMain to be called by this command. package org.apache.storm.shade.org.apache.zookeeper; @@ -38,11 +38,13 @@ public void run(String[] args, Map conf, String command) throws int port = ObjectReader.getInt(conf.get(Config.STORM_ZOOKEEPER_PORT)); String root = (String) conf.get(Config.STORM_ZOOKEEPER_ROOT); Map cl = CLI.opt("s", "server", null, CLI.AS_STRING, CLI.LAST_WINS) - .opt("t", "time-out", ObjectReader.getInt(conf.get(Config.STORM_ZOOKEEPER_SESSION_TIMEOUT)), + .opt("t", "time-out", ObjectReader.getInt(conf + .get(Config.STORM_ZOOKEEPER_SESSION_TIMEOUT)), CLI.AS_INT, CLI.LAST_WINS) .boolOpt("w", "write") .boolOpt("n", "no-root") - .opt("j", "jaas", conf.get("java.security.auth.login.config"), CLI.AS_STRING, CLI.LAST_WINS) + .opt("j", "jaas", conf.get("java.security.auth.login.config"), CLI.AS_STRING, + CLI.LAST_WINS) .boolOpt("h", "help") .parse(args); @@ -77,9 +79,9 @@ public void run(String[] args, Map conf, String command) throws int timeout = (Integer) cl.get("t"); ZooKeeper zk; if (readOnly) { - zk = new ReadOnlyZookeeper(connectionString, timeout, watchedEvent -> { }); + zk = new ReadOnlyZookeeper(connectionString, timeout, watchedEvent -> {}); } else { - zk = new ZooKeeper(connectionString, timeout, watchedEvent -> { }); + zk = new ZooKeeper(connectionString, timeout, watchedEvent -> {}); } ZooKeeperMain main = new ZooKeeperMain(zk); main.run(); @@ -90,26 +92,34 @@ public void printCliHelp(String command, PrintStream out) { out.println(command + " []:"); out.println("\tStart a zookeeper shell"); out.println(); - out.println("\t-s --server : Set the connection string to use, defaults to storm connection string."); - out.println("\t-t --time-out : Set the timeout to use, defaults to storm zookeeper timeout."); - out.println("\t-w --write: Allow for writes, defaults to read only, we don't want to cause problems."); - out.println("\t-n --no-root: Don't include the storm root on the default connection string."); - out.println("\t-j --jaas : Include a jaas file that should be used when authenticating with\n\t\t" + out.println("\t-s --server : Set the connection string to use, " + + "defaults to storm connection string."); + out.println("\t-t --time-out : Set the timeout to use, defaults to storm " + + "zookeeper timeout."); + out.println("\t-w --write: Allow for writes, defaults to read only, " + + "we don't want to cause problems."); + out.println("\t-n --no-root: Don't include the storm root on the " + + "default connection string."); + out.println("\t-j --jaas : Include a jaas file that should be used " + + "when authenticating with\n\t\t" + "ZK defaults to the java.security.auth.login.config conf."); } private static class ReadOnlyZookeeper extends ZooKeeper { - ReadOnlyZookeeper(String connectionString, int timeout, Watcher watcher) throws IOException { + ReadOnlyZookeeper(String connectionString, int timeout, + Watcher watcher) throws IOException { super(connectionString, timeout, watcher); } @Override - public String create(String path, byte[] data, List acl, CreateMode createMode) throws KeeperException { + public String create(String path, byte[] data, List acl, + CreateMode createMode) throws KeeperException { throw KeeperException.create(KeeperException.Code.NOTREADONLY, path); } @Override - public void create(String path, byte[] data, List acl, CreateMode createMode, AsyncCallback.StringCallback cb, Object ctx) { + public void create(String path, byte[] data, List acl, CreateMode createMode, + AsyncCallback.StringCallback cb, Object ctx) { throw new IllegalArgumentException("In Read Only Mode"); } @@ -139,7 +149,8 @@ public Stat setData(String path, byte[] data, int version) throws KeeperExceptio } @Override - public void setData(String path, byte[] data, int version, AsyncCallback.StatCallback cb, Object ctx) { + public void setData(String path, byte[] data, int version, AsyncCallback.StatCallback cb, + Object ctx) { throw new IllegalArgumentException("In Read Only Mode"); } @@ -149,7 +160,8 @@ public Stat setACL(String path, List acl, int version) throws KeeperExcepti } @Override - public void setACL(String path, List acl, int version, AsyncCallback.StatCallback cb, Object ctx) { + public void setACL(String path, List acl, int version, AsyncCallback.StatCallback cb, + Object ctx) { throw new IllegalArgumentException("In Read Only Mode"); } } diff --git a/storm-core/src/jvm/org/apache/storm/testing/MockLeaderElector.java b/storm-core/src/jvm/org/apache/storm/testing/MockLeaderElector.java index 2599b4eb5f9..2f98e9ddd4e 100644 --- a/storm-core/src/jvm/org/apache/storm/testing/MockLeaderElector.java +++ b/storm-core/src/jvm/org/apache/storm/testing/MockLeaderElector.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -38,17 +44,17 @@ public MockLeaderElector(boolean isLeader, String host, int port) { @Override public void prepare(Map conf) { - //NOOP + // NOOP } @Override public void addToLeaderLockQueue() throws Exception { - //NOOP + // NOOP } @Override public void quitElectionFor(int delayMs) throws Exception { - //NOOP + // NOOP } @Override @@ -73,6 +79,6 @@ public List getAllNimbuses() throws Exception { @Override public void close() { - //NOOP + // NOOP } } diff --git a/storm-core/src/jvm/org/apache/storm/testing/staticmocking/MockedCluster.java b/storm-core/src/jvm/org/apache/storm/testing/staticmocking/MockedCluster.java index 34bfbc3d85c..0a171b2ead1 100644 --- a/storm-core/src/jvm/org/apache/storm/testing/staticmocking/MockedCluster.java +++ b/storm-core/src/jvm/org/apache/storm/testing/staticmocking/MockedCluster.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-core/src/jvm/org/apache/storm/testing/staticmocking/MockedSupervisorUtils.java b/storm-core/src/jvm/org/apache/storm/testing/staticmocking/MockedSupervisorUtils.java index f806bf7737d..d55fed027e1 100644 --- a/storm-core/src/jvm/org/apache/storm/testing/staticmocking/MockedSupervisorUtils.java +++ b/storm-core/src/jvm/org/apache/storm/testing/staticmocking/MockedSupervisorUtils.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-core/src/jvm/org/apache/storm/testing/staticmocking/MockedZookeeper.java b/storm-core/src/jvm/org/apache/storm/testing/staticmocking/MockedZookeeper.java index 6207123f8c8..887134ba47d 100644 --- a/storm-core/src/jvm/org/apache/storm/testing/staticmocking/MockedZookeeper.java +++ b/storm-core/src/jvm/org/apache/storm/testing/staticmocking/MockedZookeeper.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-core/src/jvm/org/apache/storm/utils/Monitor.java b/storm-core/src/jvm/org/apache/storm/utils/Monitor.java index d82f2916ce2..8b84a30851a 100644 --- a/storm-core/src/jvm/org/apache/storm/utils/Monitor.java +++ b/storm-core/src/jvm/org/apache/storm/utils/Monitor.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -72,7 +78,8 @@ public void metrics(Nimbus.Iface client) throws Exception { if (!WATCH_TRANSFERRED.equals(watch) && !WATCH_EMITTED.equals(watch)) { throw new IllegalArgumentException("watch item must either be transferred or emitted"); } - System.out.println("topology\tcomponent\tparallelism\tstream\ttime-diff ms\t" + watch + "\tthroughput (Kt/s)"); + System.out.println("topology\tcomponent\tparallelism\tstream\ttime-diff ms\t" + watch + + "\tthroughput (Kt/s)"); long pollMs = interval * 1000; long now = System.currentTimeMillis(); @@ -102,7 +109,8 @@ public void metrics(Nimbus.Iface client, long now, MetricsState state) throws Ex componentParallelism++; ExecutorStats stats = es.get_stats(); if (stats != null) { - Map> statted = WATCH_EMITTED.equals(watch) ? stats.get_emitted() : stats.get_transferred(); + Map> statted = WATCH_EMITTED.equals(watch) ? stats + .get_emitted() : stats.get_transferred(); if (statted != null) { Map e2 = statted.get(":all-time"); if (e2 != null) { @@ -135,7 +143,8 @@ public void metrics(Nimbus.Iface client, long now, MetricsState state) throws Ex long stattedDelta = totalStatted - state.getLastStatted(); state.setLastTime(now); state.setLastStatted(totalStatted); - double throughput = (stattedDelta == 0 || timeDelta == 0) ? 0.0 : ((double) stattedDelta / (double) timeDelta); + double throughput = (stattedDelta == 0 || timeDelta == 0) + ? 0.0 : ((double) stattedDelta / (double) timeDelta); System.out.println(topology + "\t" + component + "\t" + componentParallelism + "\t" diff --git a/storm-core/src/jvm/org/apache/storm/utils/TopologySpoutLag.java b/storm-core/src/jvm/org/apache/storm/utils/TopologySpoutLag.java index 8875e9e52e5..e34f74a138e 100644 --- a/storm-core/src/jvm/org/apache/storm/utils/TopologySpoutLag.java +++ b/storm-core/src/jvm/org/apache/storm/utils/TopologySpoutLag.java @@ -9,8 +9,10 @@ * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -18,7 +20,6 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Strings; - import java.io.File; import java.io.FileOutputStream; import java.io.IOException; @@ -51,12 +52,17 @@ public class TopologySpoutLag { private static final String GROUPID_CONFIG = CONFIG_KEY_PREFIX + "groupid"; private static final String BOOTSTRAP_CONFIG = CONFIG_KEY_PREFIX + "bootstrap.servers"; private static final String SECURITY_PROTOCOL_CONFIG = CONFIG_KEY_PREFIX + "security.protocol"; - private static final Set ALL_CONFIGS = new HashSet<>(Arrays.asList(TOPICS_CONFIG, GROUPID_CONFIG, + private static final Set ALL_CONFIGS = new HashSet<>(Arrays.asList(TOPICS_CONFIG, + GROUPID_CONFIG, BOOTSTRAP_CONFIG, SECURITY_PROTOCOL_CONFIG)); - // The spout json_conf comes from the submitted topology, while storm-kafka-monitor runs on the UI host. - // Only the connection settings the monitor actually needs to reach the brokers are forwarded to it; anything - // else (deserializers, interceptor.classes, metric.reporters, sasl.jaas.config, callback handler classes, ...) - // is dropped, so the monitor keeps using its own defaults rather than classes named by the topology. + // The spout json_conf comes from the submitted topology, while storm-kafka-monitor runs on the + // UI host. + // Only the connection settings the monitor actually needs to reach the brokers are forwarded to + // it; anything + // else (deserializers, interceptor.classes, metric.reporters, sasl.jaas.config, callback + // handler classes, ...) + // is dropped, so the monitor keeps using its own defaults rather than classes named by the + // topology. private static final Set ALLOWED_EXTRA_PROPERTIES = new HashSet<>(Arrays.asList( "client.id", "request.timeout.ms", @@ -82,7 +88,8 @@ public class TopologySpoutLag { // hint at most once to avoid spamming the UI logs, which poll the lag endpoint periodically. private static volatile boolean warnedMonitorMissing = false; - public static Map> lag(StormTopology stormTopology, Map topologyConf) { + public static Map> lag(StormTopology stormTopology, Map topologyConf) { Map> result = new HashMap<>(); Map spouts = stormTopology.get_spouts(); for (Map.Entry spout : spouts.entrySet()) { @@ -115,7 +122,8 @@ private static boolean isKafkaMonitorInstalled() { return jars != null && jars.length > 0; } - private static List getCommandLineOptionsForNewKafkaSpout(Map jsonConf) { + private static List getCommandLineOptionsForNewKafkaSpout(Map jsonConf) { LOGGER.debug("json configuration keys: {}", jsonConf.keySet()); List commands = new ArrayList<>(); @@ -139,7 +147,8 @@ static File createExtraPropertiesFile(Map jsonConf) { Map extraProperties = new HashMap<>(); List droppedProperties = new ArrayList<>(); for (Map.Entry conf : jsonConf.entrySet()) { - if (conf.getKey().startsWith(CONFIG_KEY_PREFIX) && !ALL_CONFIGS.contains(conf.getKey())) { + if (conf.getKey().startsWith(CONFIG_KEY_PREFIX) && !ALL_CONFIGS.contains(conf + .getKey())) { String consumerKey = conf.getKey().substring(CONFIG_KEY_PREFIX.length()); if (ALLOWED_EXTRA_PROPERTIES.contains(consumerKey)) { extraProperties.put(consumerKey, conf.getValue().toString()); @@ -148,9 +157,11 @@ static File createExtraPropertiesFile(Map jsonConf) { } } } - // The UI polls the lag endpoint, so log the dropped keys once per call rather than one line each. + // The UI polls the lag endpoint, so log the dropped keys once per call rather than one line + // each. if (!droppedProperties.isEmpty()) { - LOGGER.info("Not passing consumer properties {} from the topology to the Kafka spout lag monitor, " + LOGGER.info("Not passing consumer properties {} from the topology to the Kafka spout " + + "lag monitor, " + "only these properties are passed on: {}", droppedProperties, ALLOWED_EXTRA_PROPERTIES); } if (!extraProperties.isEmpty()) { @@ -169,7 +180,8 @@ static File createExtraPropertiesFile(Map jsonConf) { return file; } - private static void addLagResultForKafkaSpout(Map> finalResult, String spoutId, SpoutSpec spoutSpec) + private static void addLagResultForKafkaSpout(Map> finalResult, + String spoutId, SpoutSpec spoutSpec) throws IOException { ComponentCommon componentCommon = spoutSpec.get_common(); String json = componentCommon.get_json_conf(); @@ -189,7 +201,8 @@ private static void addLagResultForKafkaSpout(Map> f } } - private static Map getLagResultForKafka(String spoutId, SpoutSpec spoutSpec) throws IOException { + private static Map getLagResultForKafka(String spoutId, + SpoutSpec spoutSpec) throws IOException { ComponentCommon componentCommon = spoutSpec.get_common(); String json = componentCommon.get_json_conf(); Map result = null; @@ -207,7 +220,8 @@ private static Map getLagResultForKafka(String spoutId, SpoutSpe if (stormHomeDir != null && !stormHomeDir.endsWith("/")) { stormHomeDir += File.separator; } - commands.add(stormHomeDir != null ? stormHomeDir + "bin" + File.separator + "storm-kafka-monitor" : "storm-kafka-monitor"); + commands.add(stormHomeDir != null ? stormHomeDir + "bin" + File.separator + + "storm-kafka-monitor" : "storm-kafka-monitor"); Map jsonMap = null; try { jsonMap = (Map) JSONValue.parseWithException(json); @@ -223,11 +237,15 @@ private static Map getLagResultForKafka(String spoutId, SpoutSpe } LOGGER.debug("Command to run: {}", commands); - // if commands contains one or more null value, spout is compiled with lower version of storm-kafka-client + // if commands contains one or more null value, spout is compiled with lower version of + // storm-kafka-client if (!commands.contains(null) && !isKafkaMonitorInstalled()) { - errorMsg = "Kafka spout lag monitoring is unavailable because the storm-kafka-monitor " - + "jars are not installed. They ship only in the full binary distribution; on the " - + "lite distribution run 'bin/storm-kafka-monitor-fetch' on the UI host (and restart " + errorMsg = + "Kafka spout lag monitoring is unavailable because the storm-kafka-monitor " + + "jars are not installed. They ship only in the full binary distribution; on " + + "the " + + "lite distribution run 'bin/storm-kafka-monitor-fetch' on the UI host (and " + + "restart " + "the UI) to enable it."; if (!warnedMonitorMissing) { warnedMonitorMissing = true; @@ -238,20 +256,25 @@ private static Map getLagResultForKafka(String spoutId, SpoutSpe } } else if (!commands.contains(null)) { try { - String resultFromMonitor = new ShellCommandRunnerImpl().execCommand(commands.toArray(new String[0])); + String resultFromMonitor = new ShellCommandRunnerImpl().execCommand(commands + .toArray(new String[0])); try { Object parsed = JSONValue.parseWithException(resultFromMonitor); if (parsed instanceof Map) { result = (Map) parsed; } else { - // json-smart parses unquoted plain text leniently as a String, so we can land here - // when the monitor printed an error message instead of JSON; surface it as the error. - LOGGER.debug("Monitor returned non-JSON output, treating as error: {}", resultFromMonitor); + // json-smart parses unquoted plain text leniently as a String, so we + // can land here + // when the monitor printed an error message instead of JSON; surface it + // as the error. + LOGGER.debug("Monitor returned non-JSON output, treating as error: {}", + resultFromMonitor); errorMsg = resultFromMonitor; } } catch (ParseException e) { - LOGGER.debug("JSON parsing failed, assuming message as error message: {}", resultFromMonitor); + LOGGER.debug("JSON parsing failed, assuming message as error message: {}", + resultFromMonitor); errorMsg = resultFromMonitor; } } finally { @@ -275,7 +298,8 @@ private static Map getLagResultForKafka(String spoutId, SpoutSpe return kafkaSpoutLagInfo; } - private static Map getLagResultForNewKafkaSpout(String spoutId, SpoutSpec spoutSpec) throws IOException { + private static Map getLagResultForNewKafkaSpout(String spoutId, + SpoutSpec spoutSpec) throws IOException { return getLagResultForKafka(spoutId, spoutSpec); } } diff --git a/storm-core/test/jvm/org/apache/storm/MockAutoCred.java b/storm-core/test/jvm/org/apache/storm/MockAutoCred.java index 5eede6303e9..c0da77e565f 100644 --- a/storm-core/test/jvm/org/apache/storm/MockAutoCred.java +++ b/storm-core/test/jvm/org/apache/storm/MockAutoCred.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -19,7 +25,8 @@ import org.apache.storm.security.auth.ICredentialsRenewer; /** - * mock implementation of INimbusCredentialPlugin,IAutoCredentials and ICredentialsRenewer for testing only. + * Mock implementation of INimbusCredentialPlugin,IAutoCredentials and ICredentialsRenewer for + * testing only. */ public class MockAutoCred implements INimbusCredentialPlugin, IAutoCredentials, ICredentialsRenewer { public static final String NIMBUS_CRED_KEY = "nimbusCredTestKey"; @@ -51,7 +58,8 @@ public void updateSubject(Subject subject, Map credentials) { } @Override - public void renew(Map credentials, Map topologyConf, String ownerPrincipal) { + public void renew(Map credentials, Map topologyConf, + String ownerPrincipal) { credentials.put(NIMBUS_CRED_KEY, NIMBUS_CRED_RENEW_VAL); credentials.put(GATEWAY_CRED_KEY, GATEWAY_CRED_RENEW_VAL); } diff --git a/storm-core/test/jvm/org/apache/storm/SubmitterTest.java b/storm-core/test/jvm/org/apache/storm/SubmitterTest.java index 64b50383d0e..f58063d0ea7 100644 --- a/storm-core/test/jvm/org/apache/storm/SubmitterTest.java +++ b/storm-core/test/jvm/org/apache/storm/SubmitterTest.java @@ -16,14 +16,13 @@ package org.apache.storm; -import org.apache.commons.lang3.StringUtils; -import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import java.util.HashMap; import java.util.Map; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; +import org.apache.commons.lang3.StringUtils; +import org.junit.jupiter.api.Test; public class SubmitterTest { @@ -60,7 +59,7 @@ public void testMd5DigestSecretGeneration03() { Map result = StormSubmitter.prepareZookeeperAuthentication(conf); Object actualPayload = result.get(Config.STORM_ZOOKEEPER_TOPOLOGY_AUTH_PAYLOAD); Object actualScheme = result.get(Config.STORM_ZOOKEEPER_TOPOLOGY_AUTH_SCHEME); - assertFalse(StringUtils.isBlank((String)actualPayload)); + assertFalse(StringUtils.isBlank((String) actualPayload)); assertEquals("digest", actualScheme); } @@ -75,7 +74,7 @@ public void testMd5DigestSecretGeneration04() { Object actualPayload = result.get(Config.STORM_ZOOKEEPER_TOPOLOGY_AUTH_PAYLOAD); Object actualScheme = result.get(Config.STORM_ZOOKEEPER_TOPOLOGY_AUTH_SCHEME); assertFalse(StormSubmitter.validateZKDigestPayload(bogusPayload)); - assertFalse(StringUtils.isBlank((String)actualPayload)); + assertFalse(StringUtils.isBlank((String) actualPayload)); assertEquals("digest", actualScheme); } @@ -88,7 +87,7 @@ public void testMd5DigestSecretGeneration05() { Map result = StormSubmitter.prepareZookeeperAuthentication(conf); Object actualPayload = result.get(Config.STORM_ZOOKEEPER_TOPOLOGY_AUTH_PAYLOAD); Object actualScheme = result.get(Config.STORM_ZOOKEEPER_TOPOLOGY_AUTH_SCHEME); - assertFalse(StringUtils.isBlank((String)actualPayload)); + assertFalse(StringUtils.isBlank((String) actualPayload)); assertEquals("digest", actualScheme); } @@ -101,7 +100,7 @@ public void testMd5DigestSecretGeneration06() { Map result = StormSubmitter.prepareZookeeperAuthentication(conf); Object actualPayload = result.get(Config.STORM_ZOOKEEPER_TOPOLOGY_AUTH_PAYLOAD); Object actualScheme = result.get(Config.STORM_ZOOKEEPER_TOPOLOGY_AUTH_SCHEME); - assertFalse(StringUtils.isBlank((String)actualPayload)); + assertFalse(StringUtils.isBlank((String) actualPayload)); assertEquals("digest", actualScheme); } } diff --git a/storm-core/test/jvm/org/apache/storm/command/RebalanceTest.java b/storm-core/test/jvm/org/apache/storm/command/RebalanceTest.java index 3eb84ab7f97..3a9662ced82 100644 --- a/storm-core/test/jvm/org/apache/storm/command/RebalanceTest.java +++ b/storm-core/test/jvm/org/apache/storm/command/RebalanceTest.java @@ -1,30 +1,35 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ - package org.apache.storm.command; -import java.util.Map; -import org.junit.jupiter.api.Test; - import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import java.util.Map; +import org.junit.jupiter.api.Test; + public class RebalanceTest { @Test public void testParser() { Rebalance.ExecutorParser executorParser = new Rebalance.ExecutorParser(); - Map componentParallelism = (Map) executorParser.parse("comp1=3"); + Map componentParallelism = (Map) executorParser + .parse("comp1=3"); assertEquals(3, (int) componentParallelism.get("comp1")); } diff --git a/storm-core/test/jvm/org/apache/storm/command/SetLogLevelTest.java b/storm-core/test/jvm/org/apache/storm/command/SetLogLevelTest.java index 4c6c7effeb3..2eb509272b0 100644 --- a/storm-core/test/jvm/org/apache/storm/command/SetLogLevelTest.java +++ b/storm-core/test/jvm/org/apache/storm/command/SetLogLevelTest.java @@ -1,49 +1,62 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.command; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + import java.util.Map; import org.apache.storm.generated.LogLevel; import org.apache.storm.generated.LogLevelAction; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; - public class SetLogLevelTest { @Test public void testUpdateLogLevelParser() { - SetLogLevel.LogLevelsParser logLevelsParser = new SetLogLevel.LogLevelsParser(LogLevelAction.UPDATE); - LogLevel logLevel = ((Map) logLevelsParser.parse("com.foo.one=warn")).get("com.foo.one"); + SetLogLevel.LogLevelsParser logLevelsParser = new SetLogLevel + .LogLevelsParser(LogLevelAction.UPDATE); + LogLevel logLevel = ((Map) logLevelsParser.parse("com.foo.one=warn")) + .get("com.foo.one"); assertEquals(0, logLevel.get_reset_log_level_timeout_secs()); assertEquals("WARN", logLevel.get_target_log_level()); - logLevel = ((Map) logLevelsParser.parse("com.foo.two=DEBUG:10")).get("com.foo.two"); + logLevel = ((Map) logLevelsParser.parse("com.foo.two=DEBUG:10")) + .get("com.foo.two"); assertEquals(10, logLevel.get_reset_log_level_timeout_secs()); assertEquals("DEBUG", logLevel.get_target_log_level()); } @Test public void testInvalidTimeout() { - SetLogLevel.LogLevelsParser logLevelsParser = new SetLogLevel.LogLevelsParser(LogLevelAction.UPDATE); - assertThrows(NumberFormatException.class, () -> logLevelsParser.parse("com.foo.bar=warn:NaN")); + SetLogLevel.LogLevelsParser logLevelsParser = new SetLogLevel + .LogLevelsParser(LogLevelAction.UPDATE); + assertThrows(NumberFormatException.class, () -> logLevelsParser + .parse("com.foo.bar=warn:NaN")); } @Test public void testInvalidLogLevel() { - SetLogLevel.LogLevelsParser logLevelsParser = new SetLogLevel.LogLevelsParser(LogLevelAction.UPDATE); - assertThrows(IllegalArgumentException.class, () -> logLevelsParser.parse("com.foo.bar=CRITICAL")); + SetLogLevel.LogLevelsParser logLevelsParser = new SetLogLevel + .LogLevelsParser(LogLevelAction.UPDATE); + assertThrows(IllegalArgumentException.class, () -> logLevelsParser + .parse("com.foo.bar=CRITICAL")); } } diff --git a/storm-core/test/jvm/org/apache/storm/command/TestCLI.java b/storm-core/test/jvm/org/apache/storm/command/TestCLI.java index 8521424d82c..92839f8d719 100644 --- a/storm-core/test/jvm/org/apache/storm/command/TestCLI.java +++ b/storm-core/test/jvm/org/apache/storm/command/TestCLI.java @@ -1,26 +1,32 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.command; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.fail; + import java.util.HashMap; import java.util.List; import java.util.Map; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.fail; - public class TestCLI { @Test @@ -33,9 +39,10 @@ public void testSimple() throws Exception { .opt("f", "ff", null, new PairParse(), CLI.INTO_MAP) .arg("A") .arg("B", CLI.AS_INT) - .parse("-a100", "--aa", "200", "-c2", "-b", "50", "--cc", "100", "A-VALUE", "1", "2", "3", "-b40", - "-d1", "-d2", "-d3" - , "-f", "key1=value1", "-f", "key2=value2"); + .parse("-a100", "--aa", "200", "-c2", "-b", "50", "--cc", + "100", "A-VALUE", "1", "2", "3", "-b40", + "-d1", "-d2", "-d3", + "-f", "key1=value1", "-f", "key2=value2"); assertEquals(8, values.size()); assertEquals("200", values.get("a")); assertEquals((Integer) 40, (Integer) values.get("b")); @@ -64,7 +71,6 @@ public void testSimple() throws Exception { assertEquals("value2", f.get("key2")); } - @Test public void testOptional() throws Exception { Map values = CLI.optionalArg("A", CLI.LAST_WINS) @@ -103,7 +109,7 @@ public void argAfterOptional() { fail("Expected an exception to be thrown by now"); } catch (IllegalStateException is) { - //Expected + // Expected } } diff --git a/storm-core/test/jvm/org/apache/storm/integration/AckEveryOtherBolt.java b/storm-core/test/jvm/org/apache/storm/integration/AckEveryOtherBolt.java index 6d9db7f2a65..2da68ce073b 100644 --- a/storm-core/test/jvm/org/apache/storm/integration/AckEveryOtherBolt.java +++ b/storm-core/test/jvm/org/apache/storm/integration/AckEveryOtherBolt.java @@ -33,7 +33,8 @@ public void declareOutputFields(OutputFieldsDeclarer declarer) { } @Override - public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) { + public void prepare(Map topoConf, TopologyContext context, + OutputCollector collector) { this.collector = collector; } diff --git a/storm-core/test/jvm/org/apache/storm/integration/AggBolt.java b/storm-core/test/jvm/org/apache/storm/integration/AggBolt.java index 99b355cda4d..18c4805a74b 100644 --- a/storm-core/test/jvm/org/apache/storm/integration/AggBolt.java +++ b/storm-core/test/jvm/org/apache/storm/integration/AggBolt.java @@ -43,7 +43,8 @@ public void declareOutputFields(OutputFieldsDeclarer declarer) { } @Override - public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) { + public void prepare(Map topoConf, TopologyContext context, + OutputCollector collector) { this.collector = collector; } diff --git a/storm-core/test/jvm/org/apache/storm/integration/IdentityBolt.java b/storm-core/test/jvm/org/apache/storm/integration/IdentityBolt.java index 38aad5b8015..7bb9df9b7fb 100644 --- a/storm-core/test/jvm/org/apache/storm/integration/IdentityBolt.java +++ b/storm-core/test/jvm/org/apache/storm/integration/IdentityBolt.java @@ -34,7 +34,8 @@ public void declareOutputFields(OutputFieldsDeclarer declarer) { } @Override - public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) { + public void prepare(Map topoConf, TopologyContext context, + OutputCollector collector) { this.collector = collector; } diff --git a/storm-core/test/jvm/org/apache/storm/integration/TestingTest.java b/storm-core/test/jvm/org/apache/storm/integration/TestingTest.java index e96d3f35cc5..ff0f393ea74 100644 --- a/storm-core/test/jvm/org/apache/storm/integration/TestingTest.java +++ b/storm-core/test/jvm/org/apache/storm/integration/TestingTest.java @@ -40,8 +40,8 @@ import org.apache.storm.tuple.Fields; import org.apache.storm.tuple.Tuple; import org.apache.storm.tuple.Values; -import org.apache.storm.utils.Time; import org.apache.storm.utils.Time.SimulatedTime; +import org.apache.storm.utils.Time; import org.apache.storm.utils.Utils; import org.junit.jupiter.api.Test; @@ -50,7 +50,7 @@ public class TestingTest { @Test public void testSimulatedTime() throws Exception { assertThat(Time.isSimulating(), is(false)); - try(SimulatedTime time = new SimulatedTime()) { + try (SimulatedTime time = new SimulatedTime()) { assertThat(Time.isSimulating(), is(true)); } } @@ -92,15 +92,22 @@ public void testWithTrackedCluster() throws Exception { spoutMap.put("1", Thrift.prepareSpoutDetails(feeder.getSpout())); Map boltMap = new HashMap<>(); - boltMap.put("2", Thrift.prepareBoltDetails(Collections.singletonMap(Utils.getGlobalStreamId("1", null), Thrift.prepareShuffleGrouping()), new IdentityBolt())); - boltMap.put("3", Thrift.prepareBoltDetails(Collections.singletonMap(Utils.getGlobalStreamId("1", null), Thrift.prepareShuffleGrouping()), new IdentityBolt())); + boltMap.put("2", Thrift.prepareBoltDetails(Collections.singletonMap(Utils + .getGlobalStreamId("1", null), Thrift + .prepareShuffleGrouping()), new IdentityBolt())); + boltMap.put("3", Thrift.prepareBoltDetails(Collections.singletonMap(Utils + .getGlobalStreamId("1", null), Thrift + .prepareShuffleGrouping()), new IdentityBolt())); Map aggregatorInputs = new HashMap<>(); - aggregatorInputs.put(Utils.getGlobalStreamId("2", null), Thrift.prepareShuffleGrouping()); - aggregatorInputs.put(Utils.getGlobalStreamId("3", null), Thrift.prepareShuffleGrouping()); + aggregatorInputs.put(Utils.getGlobalStreamId("2", null), Thrift + .prepareShuffleGrouping()); + aggregatorInputs.put(Utils.getGlobalStreamId("3", null), Thrift + .prepareShuffleGrouping()); boltMap.put("4", Thrift.prepareBoltDetails(aggregatorInputs, new AggBolt(4))); - TrackedTopology tracked = new TrackedTopology(Thrift.buildTopology(spoutMap, boltMap), cluster);; + TrackedTopology tracked = new TrackedTopology(Thrift.buildTopology(spoutMap, boltMap), + cluster);; cluster.submitTopology("test-acking2", new Config(), tracked); @@ -130,7 +137,9 @@ public void testAdvanceClusterTime() throws Exception { spoutMap.put("1", Thrift.prepareSpoutDetails(feeder)); Map boltMap = new HashMap<>(); - boltMap.put("2", Thrift.prepareBoltDetails(Collections.singletonMap(Utils.getGlobalStreamId("1", null), Thrift.prepareShuffleGrouping()), new AckEveryOtherBolt())); + boltMap.put("2", Thrift.prepareBoltDetails(Collections.singletonMap(Utils + .getGlobalStreamId("1", null), Thrift + .prepareShuffleGrouping()), new AckEveryOtherBolt())); StormTopology topology = Thrift.buildTopology(spoutMap, boltMap); @@ -166,7 +175,9 @@ public void testDisableTupleTimeout() throws Exception { spoutMap.put("1", Thrift.prepareSpoutDetails(feeder)); Map boltMap = new HashMap<>(); - boltMap.put("2", Thrift.prepareBoltDetails(Collections.singletonMap(Utils.getGlobalStreamId("1", null), Thrift.prepareShuffleGrouping()), new AckEveryOtherBolt())); + boltMap.put("2", Thrift.prepareBoltDetails(Collections.singletonMap(Utils + .getGlobalStreamId("1", null), Thrift + .prepareShuffleGrouping()), new AckEveryOtherBolt())); StormTopology topology = Thrift.buildTopology(spoutMap, boltMap); diff --git a/storm-core/test/jvm/org/apache/storm/integration/TopologyIntegrationTest.java b/storm-core/test/jvm/org/apache/storm/integration/TopologyIntegrationTest.java index 7926c1b7199..f7b033c64a5 100644 --- a/storm-core/test/jvm/org/apache/storm/integration/TopologyIntegrationTest.java +++ b/storm-core/test/jvm/org/apache/storm/integration/TopologyIntegrationTest.java @@ -36,7 +36,6 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; - import org.apache.storm.Config; import org.apache.storm.LocalCluster; import org.apache.storm.Testing; @@ -89,7 +88,8 @@ public void testBasicTopology(boolean useLocalMessaging) throws Exception { try (LocalCluster cluster = new LocalCluster.Builder() .withSimulatedTime() .withSupervisors(4) - .withDaemonConf(Collections.singletonMap(Config.STORM_LOCAL_MODE_ZMQ, !useLocalMessaging)) + .withDaemonConf(Collections.singletonMap(Config.STORM_LOCAL_MODE_ZMQ, + !useLocalMessaging)) .build()) { TopologyBuilder builder = new TopologyBuilder(); @@ -107,13 +107,15 @@ public void testBasicTopology(boolean useLocalMessaging) throws Exception { .map(value -> new FixedTuple(new Values(value))) .collect(Collectors.toList()); - MockedSources mockedSources = new MockedSources(Collections.singletonMap("1", testTuples)); + MockedSources mockedSources = new MockedSources(Collections.singletonMap("1", + testTuples)); CompleteTopologyParam completeTopologyParams = new CompleteTopologyParam(); completeTopologyParams.setMockedSources(mockedSources); completeTopologyParams.setStormConf(stormConf); - Map> results = Testing.completeTopology(cluster, topology, completeTopologyParams); + Map> results = Testing.completeTopology(cluster, topology, + completeTopologyParams); assertThat(Testing.readTuples(results, "1"), containsInAnyOrder( new Values("nathan"), @@ -152,7 +154,8 @@ public void declareOutputFields(OutputFieldsDeclarer declarer) { } @Override - public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) { + public void prepare(Map topoConf, TopologyContext context, + OutputCollector collector) { this.collector = collector; this.taskIndex = context.getThisTaskIndex(); } @@ -177,12 +180,14 @@ public void testMultiTasksPerCluster() throws Exception { .addConfigurations(Collections.singletonMap(Config.TOPOLOGY_TASKS, 6)); StormTopology topology = builder.createTopology(); - MockedSources mockedSources = new MockedSources(Collections.singletonMap("1", Collections.singletonList(new FixedTuple(new Values("a"))))); + MockedSources mockedSources = new MockedSources(Collections.singletonMap("1", + Collections.singletonList(new FixedTuple(new Values("a"))))); CompleteTopologyParam completeTopologyParams = new CompleteTopologyParam(); completeTopologyParams.setMockedSources(mockedSources); - Map> results = Testing.completeTopology(cluster, topology, completeTopologyParams); + Map> results = Testing.completeTopology(cluster, topology, + completeTopologyParams); assertThat(Testing.readTuples(results, "2"), containsInAnyOrder( new Values(0), @@ -211,7 +216,8 @@ public void testTimeout() throws Exception { builder.setBolt("2", new AckEveryOtherBolt()).globalGrouping("1"); StormTopology topology = builder.createTopology(); - cluster.submitTopology("timeout-tester", Collections.singletonMap(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, 10), topology); + cluster.submitTopology("timeout-tester", Collections + .singletonMap(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, 10), topology); cluster.advanceClusterTime(11); feeder.feed(new Values("a"), 1); @@ -236,7 +242,8 @@ public void declareOutputFields(OutputFieldsDeclarer declarer) { } @Override - public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) { + public void prepare(Map topoConf, TopologyContext context, + OutputCollector collector) { this.collector = collector; } @@ -272,28 +279,32 @@ public void testResetTimeout() throws Exception { builder.setBolt("2", new ResetTimeoutBolt()).globalGrouping("1"); StormTopology topology = builder.createTopology(); - cluster.submitTopology("reset-timeout-tester", Collections.singletonMap(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, 10), topology); + cluster.submitTopology("reset-timeout-tester", Collections + .singletonMap(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, 10), topology); - //The first tuple wil be used to check timeout reset + // The first tuple wil be used to check timeout reset feeder.feed(new Values("a"), 1); - //The second tuple is used to wait for the spout to rotate its pending map + // The second tuple is used to wait for the spout to rotate its pending map feeder.feed(new Values("b"), 2); cluster.advanceClusterTime(9); - //The other tuples are used to reset the first tuple's timeout, - //and to wait for the message to get through to the spout (acks use the same path as timeout resets) + // The other tuples are used to reset the first tuple's timeout, + // and to wait for the message to get through to the spout (acks use the same path as + // timeout resets) feeder.feed(new Values("c"), 3); assertAcked(tracker, 3); cluster.advanceClusterTime(9); feeder.feed(new Values("d"), 4); assertAcked(tracker, 4); cluster.advanceClusterTime(2); - //The time is now twice the message timeout, the second tuple should expire since it was not acked - //Waiting for this also ensures that the first tuple gets failed if reset-timeout doesn't work + // The time is now twice the message timeout, the second tuple should expire since it + // was not acked + // Waiting for this also ensures that the first tuple gets failed if reset-timeout + // doesn't work assertFailed(tracker, 2); - //Put in a tuple to cause the first tuple to be acked + // Put in a tuple to cause the first tuple to be acked feeder.feed(new Values("e"), 5); assertAcked(tracker, 5); - //The first tuple should be acked, and should not have failed + // The first tuple should be acked, and should not have failed assertThat(tracker.isFailed(1), is(false)); assertAcked(tracker, 1); } @@ -316,26 +327,31 @@ private StormTopology mkInvalidateTopology1() { private StormTopology mkInvalidateTopology2() { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("1", new TestWordSpout(true), 3); - builder.setBolt("2", new TestWordCounter(), 4).fieldsGrouping("1", new Fields("non-exists-field")); + builder.setBolt("2", new TestWordCounter(), 4).fieldsGrouping("1", + new Fields("non-exists-field")); return builder.createTopology(); } private StormTopology mkInvalidateTopology3() { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("1", new TestWordSpout(true), 3); - builder.setBolt("2", new TestWordCounter(), 4).fieldsGrouping("1", "non-exists-stream", new Fields("word")); + builder.setBolt("2", new TestWordCounter(), 4).fieldsGrouping("1", "non-exists-stream", + new Fields("word")); return builder.createTopology(); } - private boolean tryCompleteWordCountTopology(LocalCluster cluster, StormTopology topology) throws Exception { + private boolean tryCompleteWordCountTopology(LocalCluster cluster, + StormTopology topology) throws Exception { try { List testTuples = Stream.of("nathan", "bob", "joey", "nathan") .map(value -> new FixedTuple(new Values(value))) .collect(Collectors.toList()); - MockedSources mockedSources = new MockedSources(Collections.singletonMap("1", testTuples)); + MockedSources mockedSources = new MockedSources(Collections.singletonMap("1", + testTuples)); CompleteTopologyParam completeTopologyParam = new CompleteTopologyParam(); completeTopologyParam.setMockedSources(mockedSources); - completeTopologyParam.setStormConf(Collections.singletonMap(Config.TOPOLOGY_WORKERS, 2)); + completeTopologyParam.setStormConf(Collections.singletonMap(Config.TOPOLOGY_WORKERS, + 2)); Testing.completeTopology(cluster, topology, completeTopologyParam); return false; } catch (InvalidTopologyException e) { @@ -358,7 +374,7 @@ public void testValidateTopologystructure() throws Exception { @Test public void testSystemStream() throws Exception { - //this test works because mocking a spout splits up the tuples evenly among the tasks + // this test works because mocking a spout splits up the tuples evenly among the tasks try (LocalCluster cluster = new LocalCluster.Builder() .withSimulatedTime() .build()) { @@ -376,13 +392,15 @@ public void testSystemStream() throws Exception { .map(value -> new FixedTuple(new Values(value))) .collect(Collectors.toList()); - MockedSources mockedSources = new MockedSources(Collections.singletonMap("1", testTuples)); + MockedSources mockedSources = new MockedSources(Collections.singletonMap("1", + testTuples)); CompleteTopologyParam completeTopologyParams = new CompleteTopologyParam(); completeTopologyParams.setMockedSources(mockedSources); completeTopologyParams.setStormConf(stormConf); - Map> results = Testing.completeTopology(cluster, topology, completeTopologyParams); + Map> results = Testing.completeTopology(cluster, topology, + completeTopologyParams); assertThat(Testing.readTuples(results, "2"), containsInAnyOrder( new Values("a"), @@ -407,7 +425,8 @@ public void declareOutputFields(OutputFieldsDeclarer declarer) { } @Override - public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) { + public void prepare(Map topoConf, TopologyContext context, + OutputCollector collector) { this.collector = collector; } @@ -428,7 +447,8 @@ public void declareOutputFields(OutputFieldsDeclarer declarer) { } @Override - public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) { + public void prepare(Map topoConf, TopologyContext context, + OutputCollector collector) { this.collector = collector; } @@ -455,7 +475,8 @@ public void testAcking() throws Exception { builder.setBolt("4", new BranchingBolt(2)).shuffleGrouping("1"); builder.setBolt("5", new BranchingBolt(4)).shuffleGrouping("2"); builder.setBolt("6", new BranchingBolt(1)).shuffleGrouping("3"); - builder.setBolt("7", new AggBolt(3)).shuffleGrouping("4").shuffleGrouping("5").shuffleGrouping("6"); + builder.setBolt("7", new AggBolt(3)).shuffleGrouping("4").shuffleGrouping("5") + .shuffleGrouping("6"); builder.setBolt("8", new BranchingBolt(2)).shuffleGrouping("7"); builder.setBolt("9", new AckBolt()).shuffleGrouping("8"); @@ -527,7 +548,8 @@ public void declareOutputFields(OutputFieldsDeclarer declarer) { } @Override - public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) { + public void prepare(Map topoConf, TopologyContext context, + OutputCollector collector) { this.collector = collector; } @@ -552,7 +574,8 @@ public void declareOutputFields(OutputFieldsDeclarer declarer) { } @Override - public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) { + public void prepare(Map topoConf, TopologyContext context, + OutputCollector collector) { this.collector = collector; } @@ -573,7 +596,8 @@ public void declareOutputFields(OutputFieldsDeclarer declarer) { } @Override - public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { + public void open(Map conf, TopologyContext context, + SpoutOutputCollector collector) { } @Override @@ -603,7 +627,9 @@ public void testSubmitInactiveTopology() throws Exception { StormTopology topology = builder.createTopology(); - cluster.submitTopologyWithOpts("test", Collections.singletonMap(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, 10), topology, new SubmitOptions(TopologyInitialStatus.INACTIVE)); + cluster.submitTopologyWithOpts("test", Collections + .singletonMap(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, + 10), topology, new SubmitOptions(TopologyInitialStatus.INACTIVE)); cluster.advanceClusterTime(11); feeder.feed(new Values("a"), 1); @@ -667,20 +693,25 @@ public void testKryoDecoratorsConfig() throws Exception { .build()) { TopologyBuilder topologyBuilder = new TopologyBuilder(); topologyBuilder.setSpout("1", new TestPlannerSpout(new Fields("conf"))); - topologyBuilder.setBolt("2", new TestConfBolt(Collections.singletonMap(Config.TOPOLOGY_KRYO_DECORATORS, Arrays.asList("one", "two")))) + topologyBuilder.setBolt("2", new TestConfBolt(Collections + .singletonMap(Config.TOPOLOGY_KRYO_DECORATORS, Arrays.asList("one", "two")))) .shuffleGrouping("1"); - List testTuples = Collections.singletonList(new Values(Config.TOPOLOGY_KRYO_DECORATORS)).stream() + List testTuples = Collections + .singletonList(new Values(Config.TOPOLOGY_KRYO_DECORATORS)).stream() .map(FixedTuple::new) .collect(Collectors.toList()); - MockedSources mockedSources = new MockedSources(Collections.singletonMap("1", testTuples)); + MockedSources mockedSources = new MockedSources(Collections.singletonMap("1", + testTuples)); CompleteTopologyParam completeTopologyParams = new CompleteTopologyParam(); completeTopologyParams.setMockedSources(mockedSources); - completeTopologyParams.setStormConf(Collections.singletonMap(Config.TOPOLOGY_KRYO_DECORATORS, Arrays.asList("one", "three"))); + completeTopologyParams.setStormConf(Collections + .singletonMap(Config.TOPOLOGY_KRYO_DECORATORS, Arrays.asList("one", "three"))); - Map> results = Testing.completeTopology(cluster, topologyBuilder.createTopology(), completeTopologyParams); + Map> results = Testing.completeTopology(cluster, + topologyBuilder.createTopology(), completeTopologyParams); List concatValues = Testing.readTuples(results, "2").stream() .flatMap(Collection::stream) @@ -704,13 +735,16 @@ public void testComponentSpecificConfig() throws Exception { componentConf.put("fake.config", 123); componentConf.put(Config.TOPOLOGY_MAX_TASK_PARALLELISM, 20); componentConf.put(Config.TOPOLOGY_MAX_SPOUT_PENDING, 30); - componentConf.put(Config.TOPOLOGY_KRYO_REGISTER, Arrays.asList(Collections.singletonMap("fake.type", "bad.serializer"), Collections.singletonMap("fake.type2", "a.serializer"))); + componentConf.put(Config.TOPOLOGY_KRYO_REGISTER, Arrays.asList(Collections + .singletonMap("fake.type", "bad.serializer"), Collections + .singletonMap("fake.type2", "a.serializer"))); topologyBuilder.setBolt("2", new TestConfBolt(componentConf)) .shuffleGrouping("1") .setMaxTaskParallelism(2) .addConfiguration("fake.config2", 987); - List testTuples = Stream.of("fake.config", Config.TOPOLOGY_MAX_TASK_PARALLELISM, Config.TOPOLOGY_MAX_SPOUT_PENDING, "fake.config2", Config.TOPOLOGY_KRYO_REGISTER) + List testTuples = Stream.of("fake.config", + Config.TOPOLOGY_MAX_TASK_PARALLELISM, Config.TOPOLOGY_MAX_SPOUT_PENDING, "fake.config2", Config.TOPOLOGY_KRYO_REGISTER) .map(value -> new FixedTuple(new Values(value))) .collect(Collectors.toList()); Map kryoRegister = new HashMap<>(); @@ -719,13 +753,15 @@ public void testComponentSpecificConfig() throws Exception { Map stormConf = new HashMap<>(); stormConf.put(Config.TOPOLOGY_KRYO_REGISTER, Collections.singletonList(kryoRegister)); - MockedSources mockedSources = new MockedSources(Collections.singletonMap("1", testTuples)); + MockedSources mockedSources = new MockedSources(Collections.singletonMap("1", + testTuples)); CompleteTopologyParam completeTopologyParams = new CompleteTopologyParam(); completeTopologyParams.setMockedSources(mockedSources); completeTopologyParams.setStormConf(stormConf); - Map> results = Testing.completeTopology(cluster, topologyBuilder.createTopology(), completeTopologyParams); + Map> results = Testing.completeTopology(cluster, + topologyBuilder.createTopology(), completeTopologyParams); Map expectedValues = new HashMap<>(); expectedValues.put("fake.config", 123L); @@ -756,7 +792,8 @@ public void declareOutputFields(OutputFieldsDeclarer declarer) { } @Override - public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) { + public void prepare(Map topoConf, TopologyContext context, + OutputCollector collector) { this.collector = collector; context.addTaskHook(new BaseTaskHook() { @Override @@ -807,12 +844,14 @@ public void testHooks() throws Exception { .map(value -> new FixedTuple(new Values(value))) .collect(Collectors.toList()); - MockedSources mockedSources = new MockedSources(Collections.singletonMap("1", testTuples)); + MockedSources mockedSources = new MockedSources(Collections.singletonMap("1", + testTuples)); CompleteTopologyParam completeTopologyParams = new CompleteTopologyParam(); completeTopologyParams.setMockedSources(mockedSources); - Map> results = Testing.completeTopology(cluster, topology, completeTopologyParams); + Map> results = Testing.completeTopology(cluster, topology, + completeTopologyParams); List> expectedTuples = Arrays.asList( Arrays.asList(0, 0, 0, 0), @@ -835,8 +874,12 @@ private static class TestUserResource implements Serializable { @Override public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } TestUserResource that = (TestUserResource) o; return Objects.equals(id, that.id) && Objects.equals(name, that.name); } @@ -857,7 +900,8 @@ public ResourceInitializingWorkerHook(Map resourceMap) { @Override public void start(Map topoConf, WorkerUserContext context) { resourceMap.forEach((resourceKey, resourceValue) -> - context.setResource(resourceKey, new TestUserResource(resourceKey, resourceValue))); + context.setResource(resourceKey, new TestUserResource(resourceKey, + resourceValue))); } } @@ -866,7 +910,8 @@ private static class ResourceForwardingBolt extends BaseTickTupleAwareRichBolt { private transient OutputCollector collector; @Override - public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) { + public void prepare(Map topoConf, TopologyContext context, + OutputCollector collector) { this.context = context; this.collector = collector; } @@ -904,8 +949,10 @@ public void testUserResourcesAreVisibleToTasks() throws Exception { CompleteTopologyParam completeTopologyParams = new CompleteTopologyParam(); - Map> results = Testing.completeTopology(cluster, topology, completeTopologyParams); - List expectedTuple = Arrays.asList("resource-key1", new TestUserResource("resource-key1", "resource-value1")); + Map> results = Testing.completeTopology(cluster, topology, + completeTopologyParams); + List expectedTuple = Arrays.asList("resource-key1", + new TestUserResource("resource-key1", "resource-value1")); assertThat(Testing.readTuples(results, "2"), hasItem(expectedTuple)); } } diff --git a/storm-core/test/jvm/org/apache/storm/messaging/NettyIntegrationTest.java b/storm-core/test/jvm/org/apache/storm/messaging/NettyIntegrationTest.java index d581a81ccef..bad970f71bc 100644 --- a/storm-core/test/jvm/org/apache/storm/messaging/NettyIntegrationTest.java +++ b/storm-core/test/jvm/org/apache/storm/messaging/NettyIntegrationTest.java @@ -16,9 +16,17 @@ package org.apache.storm.messaging; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; import org.apache.storm.Config; -import org.apache.storm.LocalCluster; import org.apache.storm.LocalCluster.Builder; +import org.apache.storm.LocalCluster; import org.apache.storm.Testing; import org.apache.storm.generated.StormTopology; import org.apache.storm.testing.CompleteTopologyParam; @@ -31,15 +39,6 @@ import org.apache.storm.tuple.Values; import org.junit.jupiter.api.Test; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static org.junit.jupiter.api.Assertions.assertEquals; - @IntegrationTest public class NettyIntegrationTest { @@ -47,7 +46,8 @@ public class NettyIntegrationTest { public void testIntegration() throws Exception { Map daemonConf = new HashMap<>(); daemonConf.put(Config.STORM_LOCAL_MODE_ZMQ, true); - daemonConf.put(Config.STORM_MESSAGING_TRANSPORT, "org.apache.storm.messaging.netty.Context"); + daemonConf.put(Config.STORM_MESSAGING_TRANSPORT, + "org.apache.storm.messaging.netty.Context"); daemonConf.put(Config.STORM_MESSAGING_NETTY_AUTHENTICATION, false); daemonConf.put(Config.STORM_MESSAGING_NETTY_BUFFER_SIZE, 1024000); daemonConf.put(Config.STORM_MESSAGING_NETTY_MIN_SLEEP_MS, 1000); @@ -84,18 +84,23 @@ public void testIntegration() throws Exception { .map(value -> new FixedTuple(new Values(value))) .collect(Collectors.toList()); - MockedSources mockedSources = new MockedSources(Collections.singletonMap("1", testTuples)); + MockedSources mockedSources = new MockedSources(Collections.singletonMap("1", + testTuples)); CompleteTopologyParam completeTopologyParams = new CompleteTopologyParam(); - completeTopologyParams.setStormConf(Collections.singletonMap(Config.TOPOLOGY_WORKERS, 3)); + completeTopologyParams.setStormConf(Collections.singletonMap(Config.TOPOLOGY_WORKERS, + 3)); completeTopologyParams.setMockedSources(mockedSources); - Map> results = Testing.completeTopology(cluster, topology, completeTopologyParams); + Map> results = Testing.completeTopology(cluster, topology, + completeTopologyParams); List> tuplesRead = Testing.readTuples(results, "2"); String errMsg = "Tuples Read:\n\t" - + String.join("\n\t", tuplesRead.stream().map(Object::toString).collect(Collectors.toList())) + + String.join("\n\t", tuplesRead.stream().map(Object::toString) + .collect(Collectors.toList())) + "\nTuples Expected:\n\t" - + String.join("\n\t", testTuples.stream().map(FixedTuple::toString).collect(Collectors.toList())); + + String.join("\n\t", testTuples.stream().map(FixedTuple::toString) + .collect(Collectors.toList())); assertEquals(6 * 4, tuplesRead.size(), errMsg); } } diff --git a/storm-core/test/jvm/org/apache/storm/messaging/netty/NettyTest.java b/storm-core/test/jvm/org/apache/storm/messaging/netty/NettyTest.java index da284beb087..0160ea4a901 100644 --- a/storm-core/test/jvm/org/apache/storm/messaging/netty/NettyTest.java +++ b/storm-core/test/jvm/org/apache/storm/messaging/netty/NettyTest.java @@ -55,13 +55,17 @@ public class NettyTest { private static final Logger LOG = LoggerFactory.getLogger(NettyTest.class); - private final AtomicBoolean[] remoteBpStatus = new AtomicBoolean[]{new AtomicBoolean(), new AtomicBoolean()}; + private final AtomicBoolean[] remoteBpStatus = + new AtomicBoolean[]{new AtomicBoolean(), new AtomicBoolean()}; private final int taskId = 1; /** - * In a "real" cluster (or an integration test), Storm itself would ensure that a topology's workers would only be activated once all - * the workers' connections are ready. The tests in this file however launch Netty servers and clients directly, and thus we must ensure - * manually that the server and the client connections are ready before we commence testing. If we don't do this, then we will lose the + * In a "real" cluster (or an integration test), Storm itself would ensure that a topology's + * workers would only be activated once all + * the workers' connections are ready. The tests in this file however launch Netty servers and + * clients directly, and thus we must ensure + * manually that the server and the client connections are ready before we commence testing. If + * we don't do this, then we will lose the * first messages being sent between the client and the server, which will fail the tests. */ private void waitUntilReady(IConnection... connections) { @@ -94,8 +98,10 @@ private void doTestBasic(Map stormConf) throws Exception { IContext context = TransportFactory.makeContext(stormConf, null); try { AtomicReference response = new AtomicReference<>(); - try (IConnection server = context.bind(null, 0, mkConnectionCallback(response::set), null); - IConnection client = context.connect(null, "localhost", server.getPort(), remoteBpStatus)) { + try (IConnection server = context.bind(null, 0, mkConnectionCallback(response::set), + null); + IConnection client = context.connect(null, "localhost", server.getPort(), + remoteBpStatus)) { waitUntilReady(client, server); byte[] messageBytes = reqMessage.getBytes(StandardCharsets.UTF_8); @@ -125,8 +131,10 @@ private Map basicConf() { stormConf.put(Config.TOPOLOGY_BACKPRESSURE_WAIT_PROGRESSIVE_LEVEL1_COUNT, 1); stormConf.put(Config.TOPOLOGY_BACKPRESSURE_WAIT_PROGRESSIVE_LEVEL2_COUNT, 1000); stormConf.put(Config.TOPOLOGY_BACKPRESSURE_WAIT_PROGRESSIVE_LEVEL3_SLEEP_MILLIS, 1); - stormConf.put(Config.TOPOLOGY_KRYO_FACTORY, "org.apache.storm.serialization.DefaultKryoFactory"); - stormConf.put(Config.TOPOLOGY_TUPLE_SERIALIZER, "org.apache.storm.serialization.types.ListDelegateSerializer"); + stormConf.put(Config.TOPOLOGY_KRYO_FACTORY, + "org.apache.storm.serialization.DefaultKryoFactory"); + stormConf.put(Config.TOPOLOGY_TUPLE_SERIALIZER, + "org.apache.storm.serialization.types.ListDelegateSerializer"); stormConf.put(Config.TOPOLOGY_FALL_BACK_ON_JAVA_SERIALIZATION, false); stormConf.put(Config.TOPOLOGY_SKIP_MISSING_KRYO_REGISTRATIONS, false); stormConf.put(Config.STORM_MESSAGING_NETTY_FLUSH_TIMEOUT_MS, 600000); @@ -136,7 +144,8 @@ private Map basicConf() { private Map withSaslConf(Map stormConf) { stormConf.put(Config.STORM_MESSAGING_NETTY_AUTHENTICATION, true); stormConf.put(Config.TOPOLOGY_NAME, "topo1-netty-sasl"); - stormConf.put(Config.STORM_ZOOKEEPER_TOPOLOGY_AUTH_PAYLOAD, Utils.secureRandomLong() + ":" + Utils.secureRandomLong()); + stormConf.put(Config.STORM_ZOOKEEPER_TOPOLOGY_AUTH_PAYLOAD, Utils.secureRandomLong() + ":" + + Utils.secureRandomLong()); return stormConf; } @@ -156,20 +165,26 @@ private void doTestLoad(Map stormConf) throws Exception { IContext context = TransportFactory.makeContext(stormConf, null); try { AtomicReference response = new AtomicReference<>(); - try (IConnection server = context.bind(null, 0, mkConnectionCallback(response::set), null); - IConnection client = context.connect(null, "localhost", server.getPort(), remoteBpStatus)) { + try (IConnection server = context.bind(null, 0, mkConnectionCallback(response::set), + null); + IConnection client = context.connect(null, "localhost", server.getPort(), + remoteBpStatus)) { waitUntilReady(client, server); byte[] messageBytes = reqMessage.getBytes(StandardCharsets.UTF_8); send(client, taskId, messageBytes); /* - * This test sends a broadcast to all connected clients from the server, so we need to wait until the server has registered + * This test sends a broadcast to all connected clients from the server, so we need + * to wait until the server has registered * the client as connected before sending load metrics. * - * It's not enough to wait until the client reports that the channel is open, because the server event loop may not have - * finished running channelActive for the new channel. If we send metrics too early, the server will broadcast to no one. + * It's not enough to wait until the client reports that the channel is open, + * because the server event loop may not have + * finished running channelActive for the new channel. If we send metrics too early, + * the server will broadcast to no one. * - * By waiting for the response here, we ensure that the client will be registered at the server before we send load metrics. + * By waiting for the response here, we ensure that the client will be registered at + * the server before we send load metrics. */ waitForNotNull(response); @@ -211,8 +226,10 @@ private void doTestLargeMessage(Map stormConf) throws Exception IContext context = TransportFactory.makeContext(stormConf, null); try { AtomicReference response = new AtomicReference<>(); - try (IConnection server = context.bind(null, 0, mkConnectionCallback(response::set), null); - IConnection client = context.connect(null, "localhost", server.getPort(), remoteBpStatus)) { + try (IConnection server = context.bind(null, 0, mkConnectionCallback(response::set), + null); + IConnection client = context.connect(null, "localhost", server.getPort(), + remoteBpStatus)) { waitUntilReady(client, server); byte[] messageBytes = reqMessage.getBytes(StandardCharsets.UTF_8); @@ -257,7 +274,8 @@ private void doTestServerDelayed(Map stormConf) throws Exception CompletableFuture serverStart = CompletableFuture.runAsync(() -> { try { Thread.sleep(100); - server.set(context.bind(null, port, mkConnectionCallback(response::set), null)); + server.set(context.bind(null, port, mkConnectionCallback(response::set), + null)); waitUntilReady(client, server.get()); } catch (Exception e) { throw Utils.wrapInRuntime(e); @@ -295,20 +313,23 @@ public void testServerDelayedWithSasl() throws Exception { private void doTestBatch(Map stormConf) throws Exception { int numMessages = 100_000; - LOG.info("Should send and receive many messages (testing with " + numMessages + " messages)"); + LOG.info("Should send and receive many messages (testing with " + numMessages + + " messages)"); ArrayList responses = new ArrayList<>(); AtomicInteger received = new AtomicInteger(); IContext context = TransportFactory.makeContext(stormConf, null); try { try (IConnection server = context.bind(null, 0, mkConnectionCallback((message) -> { - responses.add(message); - received.incrementAndGet(); - }), null); - IConnection client = context.connect(null, "localhost", server.getPort(), remoteBpStatus)) { + responses.add(message); + received.incrementAndGet(); + }), null); + IConnection client = context.connect(null, "localhost", server.getPort(), + remoteBpStatus)) { waitUntilReady(client, server); IntStream.range(1, numMessages) - .forEach(i -> send(client, taskId, String.valueOf(i).getBytes(StandardCharsets.UTF_8))); + .forEach(i -> send(client, taskId, String.valueOf(i) + .getBytes(StandardCharsets.UTF_8))); Awaitility.await("all batch messages to be received") .atMost(Testing.TEST_TIMEOUT_MS, TimeUnit.MILLISECONDS) @@ -318,7 +339,8 @@ private void doTestBatch(Map stormConf) throws Exception { return responses.size() >= numMessages - 1; }); IntStream.range(1, numMessages) - .forEach(i -> assertThat(new String(responses.get(i - 1).message(), StandardCharsets.UTF_8), is(String.valueOf(i)))); + .forEach(i -> assertThat(new String(responses.get(i - 1).message(), + StandardCharsets.UTF_8), is(String.valueOf(i)))); } } finally { context.term(); @@ -351,7 +373,8 @@ private void doTestServerAlwaysReconnects(Map stormConf) throws try (IConnection client = context.connect(null, "localhost", port, remoteBpStatus)) { byte[] messageBytes = reqMessage.getBytes(StandardCharsets.UTF_8); send(client, taskId, messageBytes); - try (IConnection server = context.bind(null, port, mkConnectionCallback(response::set), null)) { + try (IConnection server = context.bind(null, port, + mkConnectionCallback(response::set), null)) { waitUntilReady(client, server); send(client, taskId, messageBytes); waitForNotNull(response); @@ -381,8 +404,10 @@ private void connectToFixedPort(Map stormConf, int port) throws IContext context = TransportFactory.makeContext(stormConf, null); try { AtomicReference response = new AtomicReference<>(); - try (IConnection server = context.bind(null, port, mkConnectionCallback(response::set), null); - IConnection client = context.connect(null, "localhost", server.getPort(), remoteBpStatus)) { + try (IConnection server = context.bind(null, port, mkConnectionCallback(response::set), + null); + IConnection client = context.connect(null, "localhost", server.getPort(), + remoteBpStatus)) { waitUntilReady(client, server); byte[] messageBytes = reqMessage.getBytes(StandardCharsets.UTF_8); @@ -402,11 +427,12 @@ private void connectToFixedPort(Map stormConf, int port) throws public void testRebind() throws Exception { for (int i = 0; i < 10; ++i) { final long startTime = System.nanoTime(); - LOG.info("Binding to port 6700 iter: " + (i+1)); + LOG.info("Binding to port 6700 iter: " + (i + 1)); connectToFixedPort(basicConf(), 6700); final long endTime = System.nanoTime(); - LOG.info("Expected time taken should be less than 5 sec, actual time is: " + (endTime-startTime)/1_000_000 + " ms"); - assertThat((endTime-startTime)/1_000_000, lessThan(5_000L)); + LOG.info("Expected time taken should be less than 5 sec, actual time is: " + + (endTime - startTime) / 1_000_000 + " ms"); + assertThat((endTime - startTime) / 1_000_000, lessThan(5_000L)); } } diff --git a/storm-core/test/jvm/org/apache/storm/metric/FakeMetricConsumer.java b/storm-core/test/jvm/org/apache/storm/metric/FakeMetricConsumer.java index 1468cf24a5d..b6260be8812 100644 --- a/storm-core/test/jvm/org/apache/storm/metric/FakeMetricConsumer.java +++ b/storm-core/test/jvm/org/apache/storm/metric/FakeMetricConsumer.java @@ -1,12 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

      * http://www.apache.org/licenses/LICENSE-2.0 *

      - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -27,21 +32,25 @@ public class FakeMetricConsumer implements IMetricsConsumer { - public static final Table> BUFFER = HashBasedTable.create(); + public static final Table> BUFFER = HashBasedTable + .create(); - public static Map> getTaskIdToBuckets(String componentName, String metricName) { + public static Map> getTaskIdToBuckets(String componentName, + String metricName) { synchronized (BUFFER) { Multimap taskIdToBuckets = BUFFER.get(componentName, metricName); if (taskIdToBuckets == null) { return null; } return taskIdToBuckets.asMap().entrySet().stream() - .collect(Collectors.toMap(entry -> entry.getKey(), entry -> new ArrayList<>(entry.getValue()))); + .collect(Collectors.toMap(entry -> entry.getKey(), entry -> new ArrayList<>(entry + .getValue()))); } } @Override - public void prepare(Map topoConf, Object registrationArgument, TopologyContext context, IErrorReporter errorReporter) { + public void prepare(Map topoConf, Object registrationArgument, + TopologyContext context, IErrorReporter errorReporter) { synchronized (BUFFER) { BUFFER.clear(); } @@ -53,7 +62,8 @@ public void handleDataPoints(TaskInfo taskInfo, Collection dataPoints for (DataPoint dp : dataPoints) { for (Map.Entry entry : expandComplexDataPoint(dp).entrySet()) { String metricName = entry.getKey(); - Multimap taskIdToBucket = BUFFER.get(taskInfo.srcComponentId, metricName); + Multimap taskIdToBucket = BUFFER.get(taskInfo.srcComponentId, + metricName); if (null == taskIdToBucket) { taskIdToBucket = ArrayListMultimap.create(); taskIdToBucket.put(taskInfo.srcTaskId, entry.getValue()); diff --git a/storm-core/test/jvm/org/apache/storm/metric/MetricsIntegrationTest.java b/storm-core/test/jvm/org/apache/storm/metric/MetricsIntegrationTest.java index 7074d23960e..b7c786b8cdf 100644 --- a/storm-core/test/jvm/org/apache/storm/metric/MetricsIntegrationTest.java +++ b/storm-core/test/jvm/org/apache/storm/metric/MetricsIntegrationTest.java @@ -24,7 +24,6 @@ import java.util.Map; import java.util.concurrent.Callable; import java.util.concurrent.TimeUnit; - import org.apache.storm.Config; import org.apache.storm.LocalCluster; import org.apache.storm.Testing; @@ -43,12 +42,10 @@ import org.hamcrest.CoreMatchers; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.*; - /** * Integration tests for custom metrics with FakeMetricConsumer. * - * Ported from storm-core/test/clj/org/apache/storm/metrics_test.clj + *

      Ported from storm-core/test/clj/org/apache/storm/metrics_test.clj */ public class MetricsIntegrationTest { @@ -60,7 +57,8 @@ static class CountAcksBolt extends BaseRichBolt { private CountMetric customMetric; @Override - public void prepare(Map conf, TopologyContext context, OutputCollector collector) { + public void prepare(Map conf, TopologyContext context, + OutputCollector collector) { this.collector = collector; this.customMetric = new CountMetric(); context.registerMetric("my-custom-metric", customMetric, 5); diff --git a/storm-core/test/jvm/org/apache/storm/nimbus/InMemoryTopologyActionNotifier.java b/storm-core/test/jvm/org/apache/storm/nimbus/InMemoryTopologyActionNotifier.java index 2328c751412..301804002d7 100644 --- a/storm-core/test/jvm/org/apache/storm/nimbus/InMemoryTopologyActionNotifier.java +++ b/storm-core/test/jvm/org/apache/storm/nimbus/InMemoryTopologyActionNotifier.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -19,14 +25,13 @@ public class InMemoryTopologyActionNotifier implements ITopologyActionNotifierPlugin { - //static to ensure eventhough the class is created using reflection we can still get - //the topology to actions + // static to ensure eventhough the class is created using reflection we can still get + // the topology to actions private static final Map> topologyToActions = new HashMap<>(); - @Override public void prepare(Map stormConf) { - //no-op + // no-op } @Override @@ -43,6 +48,6 @@ public List getTopologyActions(String topologyName) { @Override public void cleanup() { - //no-op + // no-op } } diff --git a/storm-core/test/jvm/org/apache/storm/serialization/SerializationTest.java b/storm-core/test/jvm/org/apache/storm/serialization/SerializationTest.java index 343588b957c..b22fb740024 100644 --- a/storm-core/test/jvm/org/apache/storm/serialization/SerializationTest.java +++ b/storm-core/test/jvm/org/apache/storm/serialization/SerializationTest.java @@ -18,6 +18,9 @@ package org.apache.storm.serialization; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + import com.google.common.collect.Lists; import java.io.IOException; import java.util.HashMap; @@ -28,8 +31,6 @@ import org.apache.storm.utils.Utils; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.*; - // FIXME: it should be moved to storm-client when serialization-test.clj can be removed public class SerializationTest { @@ -40,8 +41,8 @@ public void testJavaSerialization() throws IOException { Map conf = new HashMap<>(); conf.put(Config.TOPOLOGY_KRYO_REGISTER, new HashMap() {{ - put("org.apache.storm.testing.TestSerObject", null); - }}); + put("org.apache.storm.testing.TestSerObject", null); + }}); conf.put(Config.TOPOLOGY_FALL_BACK_ON_JAVA_SERIALIZATION, false); assertThrows(Exception.class, () -> roundtrip(vals, conf)); @@ -60,7 +61,8 @@ public void testKryoDecorator() throws IOException { assertThrows(Exception.class, () -> roundtrip(vals, conf), "Expected Exception not Thrown for config: " + conf); - conf.put(Config.TOPOLOGY_KRYO_DECORATORS, Lists.newArrayList("org.apache.storm.testing.TestKryoDecorator")); + conf.put(Config.TOPOLOGY_KRYO_DECORATORS, Lists + .newArrayList("org.apache.storm.testing.TestKryoDecorator")); assertEquals(vals, roundtrip(vals, conf)); } diff --git a/storm-core/test/jvm/org/apache/storm/stats/TestStatsUtil.java b/storm-core/test/jvm/org/apache/storm/stats/TestStatsUtil.java index 6973a302644..a5558977817 100644 --- a/storm-core/test/jvm/org/apache/storm/stats/TestStatsUtil.java +++ b/storm-core/test/jvm/org/apache/storm/stats/TestStatsUtil.java @@ -1,24 +1,39 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.stats; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; - import org.apache.storm.cluster.IStormClusterState; import org.apache.storm.generated.ComponentAggregateStats; import org.apache.storm.generated.ComponentType; @@ -29,26 +44,18 @@ import org.apache.storm.scheduler.WorkerSlot; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - public class TestStatsUtil { /* * aggWorkerStats tests */ private Map task2Component = new HashMap(); - private Map, Map> beats = new HashMap, Map>(); + private Map, Map> beats = + new HashMap, Map>(); private Map, List> exec2NodePort = new HashMap, List>(); private Map nodeHost = new HashMap(); - private Map worker2Resources = new HashMap(); + private Map worker2Resources = + new HashMap(); private List makeExecutorId(int firstTask, int lastTask) { return Arrays.asList(new Long(firstTask), new Long(lastTask)); @@ -114,6 +121,7 @@ public void makeTopoInfo() { /** * Utility method for creating a template for Bolt stats. + * * @return Empty template map for Bolt statistics. */ private Map createBeatBoltStats() { @@ -136,6 +144,7 @@ private Map createBeatBoltStats() { /** * Utility method for creating a template for Spout stats. + * * @return Empty template map for Spout statistics. */ private Map createBeatSpoutStats() { @@ -199,7 +208,8 @@ private void makeTopoInfoWithSpout() { beats.put(exec3, exec3Beat); } - private List checkWorkerStats(boolean includeSys, boolean userAuthorized, String filterSupervisor) { + private List checkWorkerStats(boolean includeSys, boolean userAuthorized, + String filterSupervisor) { List summaries = StatsUtil.aggWorkerStats("my-storm-id", "my-storm-name", task2Component, beats, exec2NodePort, nodeHost, worker2Resources, @@ -239,6 +249,8 @@ private List checkWorkerStats(boolean includeSys, boolean userAut assertEquals(32.0, ws.get_assigned_memoffheap(), 0.001); assertEquals(48.0, ws.get_assigned_cpu(), 0.001); break; + default: + break; } } @@ -247,7 +259,7 @@ private List checkWorkerStats(boolean includeSys, boolean userAut } private WorkerSummary getWorkerSummaryForPort(List summaries, int port) { - //iterate of WorkerSummary and find the one with the port + // iterate of WorkerSummary and find the one with the port for (WorkerSummary ws : summaries) { if (ws.get_port() == port) { return ws; @@ -348,22 +360,26 @@ public void aggWorkerStatsFilterSupervisorAndHideSystemComponents() { public void aggTopoExecsStats_boltAndSpoutsHaveLastErrorReported() { // Define inputs final String expectedBoltErrorMsg = "This is my test bolt error message"; - final int expectedBoltErrorTime = (int) TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis()); + final int expectedBoltErrorTime = (int) TimeUnit.MILLISECONDS.toSeconds(System + .currentTimeMillis()); final int expectedBoltErrorPort = 4321; final String expectedBoltErrorHost = "my.errored.host"; final String expectedSpoutErrorMsg = "This is my test spout error message"; - final int expectedSpoutErrorTime = (int) TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis()); + final int expectedSpoutErrorTime = (int) TimeUnit.MILLISECONDS.toSeconds(System + .currentTimeMillis()); final int expectedSpoutErrorPort = 1234; final String expectedSpoutErrorHost = "my.errored.host2"; // Define our Last Error for the bolt - final ErrorInfo expectedBoltLastError = new ErrorInfo(expectedBoltErrorMsg, expectedBoltErrorTime); + final ErrorInfo expectedBoltLastError = new ErrorInfo(expectedBoltErrorMsg, + expectedBoltErrorTime); expectedBoltLastError.set_port(expectedBoltErrorPort); expectedBoltLastError.set_host(expectedBoltErrorHost); // Define our Last Error for the spout - final ErrorInfo expectedSpoutLastError = new ErrorInfo(expectedSpoutErrorMsg, expectedSpoutErrorTime); + final ErrorInfo expectedSpoutLastError = new ErrorInfo(expectedSpoutErrorMsg, + expectedSpoutErrorTime); expectedSpoutLastError.set_port(expectedSpoutErrorPort); expectedSpoutLastError.set_host(expectedSpoutErrorHost); @@ -394,7 +410,8 @@ public void aggTopoExecsStats_boltAndSpoutsHaveLastErrorReported() { assertEquals("my-storm-id", topologyPageInfo.get_id()); assertEquals(8, topologyPageInfo.get_num_tasks(), "Should have 7 tasks."); assertEquals(2, topologyPageInfo.get_num_workers(), "Should have 2 workers."); - assertEquals(2, topologyPageInfo.get_num_executors(), "Should have only a single executor."); + assertEquals(2, topologyPageInfo.get_num_executors(), + "Should have only a single executor."); // Validate Spout aggregate statistics assertNotNull(topologyPageInfo.get_id_to_spout_agg_stats(), "Should be non-null"); @@ -403,7 +420,8 @@ public void aggTopoExecsStats_boltAndSpoutsHaveLastErrorReported() { assertTrue(topologyPageInfo.get_id_to_spout_agg_stats().containsKey("my-spout")); assertNotNull(topologyPageInfo.get_id_to_spout_agg_stats().get("my-spout")); - ComponentAggregateStats componentStats = topologyPageInfo.get_id_to_spout_agg_stats().get("my-spout"); + ComponentAggregateStats componentStats = topologyPageInfo.get_id_to_spout_agg_stats() + .get("my-spout"); assertEquals(ComponentType.SPOUT, componentStats.get_type(), "Should be of type spout"); assertNotNull(componentStats.get_last_error(), "Last error should not be null"); @@ -471,7 +489,8 @@ public void aggTopoExecsStats_boltAndSpoutsHaveNoLastErrorReported() { assertEquals("my-storm-id", topologyPageInfo.get_id()); assertEquals(8, topologyPageInfo.get_num_tasks(), "Should have 7 tasks."); assertEquals(2, topologyPageInfo.get_num_workers(), "Should have 2 workers."); - assertEquals(2, topologyPageInfo.get_num_executors(), "Should have only a single executor."); + assertEquals(2, topologyPageInfo.get_num_executors(), + "Should have only a single executor."); // Validate Spout aggregate statistics assertNotNull(topologyPageInfo.get_id_to_spout_agg_stats(), "Should be non-null"); @@ -480,7 +499,8 @@ public void aggTopoExecsStats_boltAndSpoutsHaveNoLastErrorReported() { assertTrue(topologyPageInfo.get_id_to_spout_agg_stats().containsKey("my-spout")); assertNotNull(topologyPageInfo.get_id_to_spout_agg_stats().get("my-spout")); - ComponentAggregateStats componentStats = topologyPageInfo.get_id_to_spout_agg_stats().get("my-spout"); + ComponentAggregateStats componentStats = topologyPageInfo.get_id_to_spout_agg_stats() + .get("my-spout"); assertEquals(ComponentType.SPOUT, componentStats.get_type(), "Should be of type spout"); assertNull(componentStats.get_last_error(), "Last error should not be null"); diff --git a/storm-core/test/jvm/org/apache/storm/trident/StateTest.java b/storm-core/test/jvm/org/apache/storm/trident/StateTest.java index 68f615d2a24..d0589e6c2f1 100644 --- a/storm-core/test/jvm/org/apache/storm/trident/StateTest.java +++ b/storm-core/test/jvm/org/apache/storm/trident/StateTest.java @@ -16,12 +16,17 @@ package org.apache.storm.trident; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; + +import java.util.Collections; +import java.util.List; import org.apache.storm.shade.org.apache.curator.framework.CuratorFramework; import org.apache.storm.shade.org.apache.curator.framework.api.CreateBuilder; import org.apache.storm.shade.org.apache.curator.framework.api.ProtectACLCreateModeStatPathAndBytesable; import org.apache.storm.shade.org.apache.zookeeper.CreateMode; -import org.apache.storm.shade.org.apache.zookeeper.data.ACL; import org.apache.storm.shade.org.apache.zookeeper.ZooDefs; +import org.apache.storm.shade.org.apache.zookeeper.data.ACL; import org.apache.storm.trident.operation.builtin.Count; import org.apache.storm.trident.state.CombinerValueUpdater; import org.apache.storm.trident.state.OpaqueValue; @@ -38,31 +43,25 @@ import org.mockito.ArgumentMatchers; import org.mockito.Mockito; -import java.util.Collections; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotEquals; - public class StateTest { - private void singleRemove(MemoryMapState map, Object key){ + private void singleRemove(MemoryMapState map, Object key) { List> keys = Collections.singletonList(Collections.singletonList(key)); map.multiRemove(keys); } - private void singlePut(MemoryMapState map, Object key, Object val){ + private void singlePut(MemoryMapState map, Object key, Object val) { List> keys = Collections.singletonList(Collections.singletonList(key)); List vals = Collections.singletonList(val); map.multiPut(keys, vals); } - private Object singleGet(MapState map, Object key){ + private Object singleGet(MapState map, Object key) { List> keys = Collections.singletonList(Collections.singletonList(key)); return map.multiGet(keys).get(0); } - private Object singleUpdate(MapState map, Object key, Long amt){ + private Object singleUpdate(MapState map, Object key, Long amt) { List> keys = Collections.singletonList(Collections.singletonList(key)); CombinerValueUpdater valueUpdater = new CombinerValueUpdater(new Count(), amt); List updaters = Collections.singletonList(valueUpdater); @@ -140,11 +139,13 @@ public void testCreateNodeAcl() throws Exception { // Creates ZooKeeper nodes with the correct ACLs CuratorFramework curator = Mockito.mock(CuratorFramework.class); CreateBuilder builder0 = Mockito.mock(CreateBuilder.class); - ProtectACLCreateModeStatPathAndBytesable builder1 = Mockito.mock(ProtectACLCreateModeStatPathAndBytesable.class); + ProtectACLCreateModeStatPathAndBytesable builder1 = Mockito + .mock(ProtectACLCreateModeStatPathAndBytesable.class); List expectedAcls = ZooDefs.Ids.CREATOR_ALL_ACL; Mockito.when(curator.create()).thenReturn(builder0); Mockito.when(builder0.creatingParentsIfNeeded()).thenReturn(builder1); - Mockito.when(builder1.withMode(ArgumentMatchers.isA(CreateMode.class))).thenReturn(builder1); + Mockito.when(builder1.withMode(ArgumentMatchers.isA(CreateMode.class))) + .thenReturn(builder1); Mockito.when(builder1.withACL(Mockito.anyList())).thenReturn(builder1); TestTransactionalState.createNode(curator, "", new byte[0], expectedAcls, null); Mockito.verify(builder1).withACL(expectedAcls); diff --git a/storm-core/test/jvm/org/apache/storm/trident/TridentTupleViewTest.java b/storm-core/test/jvm/org/apache/storm/trident/TridentTupleViewTest.java index 21b02e64734..903ddd8918b 100644 --- a/storm-core/test/jvm/org/apache/storm/trident/TridentTupleViewTest.java +++ b/storm-core/test/jvm/org/apache/storm/trident/TridentTupleViewTest.java @@ -24,14 +24,13 @@ import java.util.Arrays; import java.util.List; - import org.apache.storm.Testing; import org.apache.storm.trident.tuple.TridentTuple; -import org.apache.storm.trident.tuple.TridentTupleView; import org.apache.storm.trident.tuple.TridentTupleView.FreshOutputFactory; import org.apache.storm.trident.tuple.TridentTupleView.OperationOutputFactory; import org.apache.storm.trident.tuple.TridentTupleView.ProjectionFactory; import org.apache.storm.trident.tuple.TridentTupleView.RootFactory; +import org.apache.storm.trident.tuple.TridentTupleView; import org.apache.storm.tuple.Fields; import org.apache.storm.tuple.Tuple; import org.junit.jupiter.api.Test; @@ -54,8 +53,10 @@ void testFresh() { @Test void testProjection() { - FreshOutputFactory freshFactory = new FreshOutputFactory(new Fields("a", "b", "c", "d", "e")); - ProjectionFactory projectFactory = new ProjectionFactory(freshFactory, new Fields("d", "a")); + FreshOutputFactory freshFactory = new FreshOutputFactory(new Fields("a", "b", "c", "d", + "e")); + ProjectionFactory projectFactory = new ProjectionFactory(freshFactory, new Fields("d", + "a")); TridentTuple tt = freshFactory.create(Arrays.asList(3, 2, 1, 4, 5)); TridentTuple tt2 = freshFactory.create(Arrays.asList(9, 8, 7, 6, 10)); @@ -75,8 +76,10 @@ void testProjection() { @Test void testAppends() { FreshOutputFactory freshFactory = new FreshOutputFactory(new Fields("a", "b", "c")); - OperationOutputFactory appendFactory = new OperationOutputFactory(freshFactory, new Fields("d", "e")); - OperationOutputFactory appendFactory2 = new OperationOutputFactory(appendFactory, new Fields("f")); + OperationOutputFactory appendFactory = new OperationOutputFactory(freshFactory, + new Fields("d", "e")); + OperationOutputFactory appendFactory2 = new OperationOutputFactory(appendFactory, + new Fields("f")); TridentTupleView tt = (TridentTupleView) freshFactory.create(Arrays.asList(1, 2, 3)); TridentTupleView tt2 = (TridentTupleView) appendFactory.create(tt, Arrays.asList(4, 5)); @@ -98,7 +101,8 @@ void testRoot() { assertEquals("a", tt.getValueByField("a")); assertEquals(1, tt.getValueByField("b")); - OperationOutputFactory appendFactory = new OperationOutputFactory(rootFactory, new Fields("c")); + OperationOutputFactory appendFactory = new OperationOutputFactory(rootFactory, + new Fields("c")); TridentTuple tt2 = appendFactory.create(tt, Arrays.asList(3)); assertEquals(Arrays.asList("a", 1, 3), tt2.getValues()); @@ -110,10 +114,14 @@ void testRoot() { @Test void testComplex() { FreshOutputFactory freshFactory = new FreshOutputFactory(new Fields("a", "b", "c")); - OperationOutputFactory appendFactory1 = new OperationOutputFactory(freshFactory, new Fields("d")); - OperationOutputFactory appendFactory2 = new OperationOutputFactory(appendFactory1, new Fields("e", "f")); - ProjectionFactory projectFactory1 = new ProjectionFactory(appendFactory2, new Fields("a", "f", "b")); - OperationOutputFactory appendFactory3 = new OperationOutputFactory(projectFactory1, new Fields("c")); + OperationOutputFactory appendFactory1 = new OperationOutputFactory(freshFactory, + new Fields("d")); + OperationOutputFactory appendFactory2 = new OperationOutputFactory(appendFactory1, + new Fields("e", "f")); + ProjectionFactory projectFactory1 = new ProjectionFactory(appendFactory2, new Fields("a", + "f", "b")); + OperationOutputFactory appendFactory3 = new OperationOutputFactory(projectFactory1, + new Fields("c")); TridentTupleView tt = (TridentTupleView) freshFactory.create(Arrays.asList(1, 2, 3)); TridentTupleView tt2 = (TridentTupleView) appendFactory1.create(tt, Arrays.asList(4)); @@ -135,7 +143,8 @@ void testComplex() { @Test void testITupleInterface() { - TridentTuple tt = TridentTupleView.createFreshTuple(new Fields("a", "b", "c"), Arrays.asList(1, 2, 3)); + TridentTuple tt = TridentTupleView.createFreshTuple(new Fields("a", "b", "c"), Arrays + .asList(1, 2, 3)); assertEquals(Arrays.asList(1, 2, 3), tt.getValues()); assertEquals(Arrays.asList("a", "b", "c"), tt.getFields().toList()); diff --git a/storm-core/test/jvm/org/apache/storm/utils/TopologySpoutLagTest.java b/storm-core/test/jvm/org/apache/storm/utils/TopologySpoutLagTest.java index 86459687730..36aff9976cf 100644 --- a/storm-core/test/jvm/org/apache/storm/utils/TopologySpoutLagTest.java +++ b/storm-core/test/jvm/org/apache/storm/utils/TopologySpoutLagTest.java @@ -28,7 +28,6 @@ import java.util.HashMap; import java.util.Map; import java.util.Properties; - import org.junit.jupiter.api.Test; public class TopologySpoutLagTest { diff --git a/storm-core/test/jvm/org/apache/storm/utils/VersionedStoreTest.java b/storm-core/test/jvm/org/apache/storm/utils/VersionedStoreTest.java index d1252ed4a08..d4d43a99c05 100644 --- a/storm-core/test/jvm/org/apache/storm/utils/VersionedStoreTest.java +++ b/storm-core/test/jvm/org/apache/storm/utils/VersionedStoreTest.java @@ -22,7 +22,6 @@ import java.io.File; import java.io.IOException; - import org.apache.commons.io.FileUtils; import org.apache.storm.testing.TmpPath; import org.junit.jupiter.api.Test; diff --git a/storm-core/test/jvm/org/apache/storm/utils/staticmocking/package-info.java b/storm-core/test/jvm/org/apache/storm/utils/staticmocking/package-info.java index 00aa8a38d98..69ce7a06d0b 100644 --- a/storm-core/test/jvm/org/apache/storm/utils/staticmocking/package-info.java +++ b/storm-core/test/jvm/org/apache/storm/utils/staticmocking/package-info.java @@ -1,49 +1,63 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at +/* + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. * * Provides implementations for testing static methods. * * This package should not exist and is only necessary while we need to mock static methods. * - * To mock static methods in java, we use a singleton. The class to mock must implement setInstance static method that accepts + * To mock static methods in java, we use a singleton. The class to mock must implement + * setInstance static method that accepts * an instance of the selfsame class and returns the previous instance that was set. * * Example: * * - * public class MyClass { public static MyClass setInstance(MyClass c) { MyClass oldInstance = _instance; _instance = c; return oldInstance; + * public class MyClass { public static MyClass setInstance(MyClass c) { MyClass oldInstance = + * _instance; _instance = c; return oldInstance; * } * - * // Any method that we wish to mock must delegate to the singleton // instance's corresponding member method implementation public static + * // Any method that we wish to mock must delegate to the singleton // instance's corresponding + * member method implementation public static * int mockableFunction(String arg) { return _instance.mockableFunctionImpl(); } * * protected int mockableFunctionImpl(String arg) { return arg.size(); } } * * - * Each class that could be mocked should have an Installer class defined in this package that sets the instance on construction and + * Each class that could be mocked should have an Installer class defined in this package that sets + * the instance on construction and * implements the * close method of {@link java.lang.AutoCloseable}. * * Example: * * - * class MyClassInstaller implementes AutoCloseable { private MyClass _oldInstance; private MyClass _curInstance; + * class MyClassInstaller implementes AutoCloseable { private MyClass _oldInstance; private MyClass + * _curInstance; * - * MyClassInstaller(MyClass instance) { _oldInstance = MyClass.setInstance(instance); _curInstance = instance; } + * MyClassInstaller(MyClass instance) { _oldInstance = MyClass.setInstance(instance); _curInstance = + * instance; } * - * @Override public void close() throws Exception { if (MyClass.setInstance(_oldInstance) != _curInstance) { throw new - * IllegalStateException( "Instances of this resource must be closed in reverse order of opening."); } } } + * @Override public void close() throws Exception { if (MyClass.setInstance(_oldInstance) != + * _curInstance) { throw new + * IllegalStateException( "Instances of this resource must be closed in reverse order of + * opening."); } } } * * - * To write a test with the mocked class instantiate a child class that implements the close method, and use try-with-resources. For + * To write a test with the mocked class instantiate a child class that implements the close method, + * and use try-with-resources. For * example: * * @@ -114,7 +128,7 @@ * } * } * } - * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * To write a test with the mocked class instantiate a child class that * implements the close method, and use try-with-resources. For example: diff --git a/storm-multilang/javascript/pom.xml b/storm-multilang/javascript/pom.xml index edff4460c68..c063db86a40 100644 --- a/storm-multilang/javascript/pom.xml +++ b/storm-multilang/javascript/pom.xml @@ -33,6 +33,16 @@ + + org.openrewrite.maven + rewrite-maven-plugin + + + + org.codehaus.gmavenplus + gmavenplus-plugin + + org.apache.maven.plugins maven-checkstyle-plugin diff --git a/storm-multilang/ruby/pom.xml b/storm-multilang/ruby/pom.xml index 291a3cdcdaa..ae84aee3119 100644 --- a/storm-multilang/ruby/pom.xml +++ b/storm-multilang/ruby/pom.xml @@ -33,6 +33,16 @@ + + org.openrewrite.maven + rewrite-maven-plugin + + + + org.codehaus.gmavenplus + gmavenplus-plugin + + org.apache.maven.plugins maven-checkstyle-plugin diff --git a/storm-server/pom.xml b/storm-server/pom.xml index ae859c3f5f5..b1ec5c8f832 100644 --- a/storm-server/pom.xml +++ b/storm-server/pom.xml @@ -198,6 +198,16 @@ ${project.build.directory}/test-reports + + org.openrewrite.maven + rewrite-maven-plugin + + + + org.codehaus.gmavenplus + gmavenplus-plugin + + org.apache.maven.plugins maven-checkstyle-plugin diff --git a/storm-server/src/main/java/org/apache/storm/DaemonConfig.java b/storm-server/src/main/java/org/apache/storm/DaemonConfig.java index e535baf1baa..9b13efafe62 100644 --- a/storm-server/src/main/java/org/apache/storm/DaemonConfig.java +++ b/storm-server/src/main/java/org/apache/storm/DaemonConfig.java @@ -51,8 +51,10 @@ import org.apache.storm.validation.Validated; /** - * Storm configs are specified as a plain old map. This class provides constants for all the configurations possible on a Storm cluster. - * Each constant is paired with an annotation that defines the validity criterion of the corresponding field. Default values for these + * Storm configs are specified as a plain old map. This class provides constants for all the + * configurations possible on a Storm cluster. + * Each constant is paired with an annotation that defines the validity criterion of the + * corresponding field. Default values for these * configs can be found in defaults.yaml. * *

      This class extends {@link org.apache.storm.Config} for supporting Storm Daemons. @@ -60,38 +62,45 @@ public class DaemonConfig implements Validated { /** - * We check with this interval that whether the Netty channel is writable and try to write pending messages. + * We check with this interval that whether the Netty channel is writable and try to write + * pending messages. */ @IsInteger - public static final String STORM_NETTY_FLUSH_CHECK_INTERVAL_MS = "storm.messaging.netty.flush.check.interval.ms"; + public static final String STORM_NETTY_FLUSH_CHECK_INTERVAL_MS = + "storm.messaging.netty.flush.check.interval.ms"; /** * A list of daemon metrics reporter plugin class names. These plugins must implement {@link * org.apache.storm.daemon.metrics.reporters.PreparableReporter} interface. */ @IsStringList - public static final String STORM_DAEMON_METRICS_REPORTER_PLUGINS = "storm.daemon.metrics.reporter.plugins"; + public static final String STORM_DAEMON_METRICS_REPORTER_PLUGINS = + "storm.daemon.metrics.reporter.plugins"; /** * Specify the domain for daemon metrics reporter plugin to limit reporting to specific domain. */ @IsString - public static final String STORM_DAEMON_METRICS_REPORTER_PLUGIN_DOMAIN = "storm.daemon.metrics.reporter.plugin.domain"; + public static final String STORM_DAEMON_METRICS_REPORTER_PLUGIN_DOMAIN = + "storm.daemon.metrics.reporter.plugin.domain"; /** * We report the metrics with this interval period. */ @IsInteger - public static final String STORM_DAEMON_METRICS_REPORTER_INTERVAL_SECS = "storm.daemon.metrics.reporter.interval.secs"; + public static final String STORM_DAEMON_METRICS_REPORTER_INTERVAL_SECS = + "storm.daemon.metrics.reporter.interval.secs"; /** * Specify the csv reporter directory for CvsPreparableReporter daemon metrics reporter. */ @IsString - public static final String STORM_DAEMON_METRICS_REPORTER_CSV_LOG_DIR = "storm.daemon.metrics.reporter.csv.log.dir"; + public static final String STORM_DAEMON_METRICS_REPORTER_CSV_LOG_DIR = + "storm.daemon.metrics.reporter.csv.log.dir"; /** - * A directory that holds configuration files for log4j2. It can be either a relative or an absolute directory. If relative, it is + * A directory that holds configuration files for log4j2. It can be either a relative or an + * absolute directory. If relative, it is * relative to the storm's home directory. */ @IsString @@ -110,25 +119,29 @@ public class DaemonConfig implements Validated { */ @IsInteger @IsPositiveNumber - public static final String SCHEDULING_TIMEOUT_SECONDS_PER_TOPOLOGY = "scheduling.timeout.seconds.per.topology"; + public static final String SCHEDULING_TIMEOUT_SECONDS_PER_TOPOLOGY = + "scheduling.timeout.seconds.per.topology"; /** * The number of seconds that the blacklist scheduler will concern of bad slots or supervisors. */ @IsPositiveNumber - public static final String BLACKLIST_SCHEDULER_TOLERANCE_TIME = "blacklist.scheduler.tolerance.time.secs"; + public static final String BLACKLIST_SCHEDULER_TOLERANCE_TIME = + "blacklist.scheduler.tolerance.time.secs"; /** * The number of hit count that will trigger blacklist in tolerance time. */ @IsPositiveNumber - public static final String BLACKLIST_SCHEDULER_TOLERANCE_COUNT = "blacklist.scheduler.tolerance.count"; + public static final String BLACKLIST_SCHEDULER_TOLERANCE_COUNT = + "blacklist.scheduler.tolerance.count"; /** * The number of seconds that the blacklisted slots or supervisor will be resumed. */ @IsPositiveNumber - public static final String BLACKLIST_SCHEDULER_RESUME_TIME = "blacklist.scheduler.resume.time.secs"; + public static final String BLACKLIST_SCHEDULER_RESUME_TIME = + "blacklist.scheduler.resume.time.secs"; /** * Enables blacklisting support for supervisors with failed send assignment calls. @@ -155,19 +168,24 @@ public class DaemonConfig implements Validated { public static final String BLACKLIST_SCHEDULER_STRATEGY = "blacklist.scheduler.strategy"; /** - * Whether {@link org.apache.storm.scheduler.blacklist.BlacklistScheduler} will assume the supervisor is bad + * Whether {@link org.apache.storm.scheduler.blacklist.BlacklistScheduler} will assume the + * supervisor is bad * based on bad slots or not. - * A bad slot indicates the situation where the nimbus doesn't receive heartbeat from the worker in time, + * A bad slot indicates the situation where the nimbus doesn't receive heartbeat from the worker + * in time, * it's hard to differentiate if it's because of the supervisor node or the worker itself. - * If this is set to true, the scheduler will consider a supervisor is bad when seeing bad slots in it. - * Otherwise, the scheduler will assume a supervisor is bad only when it does not receive supervisor heartbeat in time. + * If this is set to true, the scheduler will consider a supervisor is bad when seeing bad slots + * in it. + * Otherwise, the scheduler will assume a supervisor is bad only when it does not receive + * supervisor heartbeat in time. */ @IsBoolean public static final String BLACKLIST_SCHEDULER_ASSUME_SUPERVISOR_BAD_BASED_ON_BAD_SLOT = "blacklist.scheduler.assume.supervisor.bad.based.on.bad.slot"; /** - * Whether we want to display all the resource capacity and scheduled usage on the UI page. You MUST have this variable set if you are + * Whether we want to display all the resource capacity and scheduled usage on the UI page. You + * MUST have this variable set if you are * using any kind of resource-related scheduler. *

      * If this is not set, we will not display resource capacity and usage on the UI. @@ -176,9 +194,12 @@ public class DaemonConfig implements Validated { public static final String SCHEDULER_DISPLAY_RESOURCE = "scheduler.display.resource"; /** - * If true, {@link org.apache.storm.scheduler.EvenScheduler} may move already-assigned workers onto non-blacklisted supervisors - * with no slot in use. This lets a freshly returned supervisor pick up workers instead of staying idle. The number of workers - * freed per topology in a single scheduling round is capped by {@link #NIMBUS_EVEN_REBALANCE_MAX_FREE_PER_TOPOLOGY}, so even + * If true, {@link org.apache.storm.scheduler.EvenScheduler} may move already-assigned workers + * onto non-blacklisted supervisors + * with no slot in use. This lets a freshly returned supervisor pick up workers instead of + * staying idle. The number of workers + * freed per topology in a single scheduling round is capped by {@link + * #NIMBUS_EVEN_REBALANCE_MAX_FREE_PER_TOPOLOGY}, so even * distribution is approached gradually rather than rebuilt from scratch. */ @IsBoolean @@ -186,10 +207,13 @@ public class DaemonConfig implements Validated { = "nimbus.even.rebalance.idle.supervisor.enabled"; /** - * Optional upper bound on the number of currently-assigned workers a single topology may release in one scheduling round - * when the idle-supervisor rebalance defined by {@link #NIMBUS_EVEN_REBALANCE_ON_IDLE_SUPERVISOR_ENABLED} kicks in. The + * Optional upper bound on the number of currently-assigned workers a single topology may + * release in one scheduling round + * when the idle-supervisor rebalance defined by {@link + * #NIMBUS_EVEN_REBALANCE_ON_IDLE_SUPERVISOR_ENABLED} kicks in. The * default budget already targets an even per-supervisor distribution (idle supervisors absorb roughly {@code numWorkers / - * numSupervisors} workers each in one round), capped by the idle side's free slot capacity. Setting this to a positive + * numSupervisors} workers each in one round), capped by the idle side's free slot capacity. + * Setting this to a positive * value tightens that budget; setting it to {@code 0} or a negative value leaves the even-distribution budget unbounded. */ @IsInteger @@ -197,8 +221,10 @@ public class DaemonConfig implements Validated { = "nimbus.even.rebalance.max.free.per.topology"; /** - * Minimum number of consecutive supervisor monitor rounds that a fully-idle supervisor must have been alive before - * {@link org.apache.storm.scheduler.EvenScheduler} can relocate workers onto it. A positive value avoids moving workers onto a + * Minimum number of consecutive supervisor monitor rounds that a fully-idle supervisor must + * have been alive before + * {@link org.apache.storm.scheduler.EvenScheduler} can relocate workers onto it. A positive + * value avoids moving workers onto a * supervisor that has only just returned and may still be flapping. Setting this to {@code 0} or a negative value disables the * uptime guard. */ @@ -213,7 +239,8 @@ public class DaemonConfig implements Validated { public static final String STORM_HEALTH_CHECK_DIR = "storm.health.check.dir"; /** - * The time to allow any given healthcheck script to run before it is marked failed due to timeout. + * The time to allow any given healthcheck script to run before it is marked failed due to + * timeout. */ @IsNumber public static final String STORM_HEALTH_CHECK_TIMEOUT_MS = "storm.health.check.timeout.ms"; @@ -222,24 +249,28 @@ public class DaemonConfig implements Validated { * Boolean setting to configure if health checks should fail when timeouts occur or not. */ @IsBoolean - public static final String STORM_HEALTH_CHECK_FAIL_ON_TIMEOUTS = "storm.health.check.fail.on.timeouts"; + public static final String STORM_HEALTH_CHECK_FAIL_ON_TIMEOUTS = + "storm.health.check.fail.on.timeouts"; /** - * This is the user that the Nimbus daemon process is running as. May be used when security is enabled to authorize actions in the + * This is the user that the Nimbus daemon process is running as. May be used when security is + * enabled to authorize actions in the * cluster. */ @IsString public static final String NIMBUS_DAEMON_USER = "nimbus.daemon.user"; /** - * This parameter is used by the storm-deploy project to configure the jvm options for the nimbus daemon. + * This parameter is used by the storm-deploy project to configure the jvm options for the + * nimbus daemon. */ @IsStringOrStringList public static final String NIMBUS_CHILDOPTS = "nimbus.childopts"; /** - * How long without heartbeating a task can go before nimbus will consider the task dead and reassign it to another location. + * How long without heartbeating a task can go before nimbus will consider the task dead and + * reassign it to another location. * Can be exceeded when {@link Config#TOPOLOGY_WORKER_TIMEOUT_SECS} is set. */ @IsInteger @@ -247,8 +278,10 @@ public class DaemonConfig implements Validated { public static final String NIMBUS_TASK_TIMEOUT_SECS = "nimbus.task.timeout.secs"; /** - * How often nimbus should wake up to check heartbeats and do reassignments. Note that if a machine ever goes down Nimbus will - * immediately wake up and take action. This parameter is for checking for failures when there's no explicit event like that occurring. + * How often nimbus should wake up to check heartbeats and do reassignments. Note that if a + * machine ever goes down Nimbus will + * immediately wake up and take action. This parameter is for checking for failures when there's + * no explicit event like that occurring. */ @IsInteger @IsPositiveNumber @@ -266,26 +299,34 @@ public class DaemonConfig implements Validated { /** * The length of time a jar file lives in the inbox before being deleted by the cleanup thread. * - *

      Probably keep this value greater than or equal to NIMBUS_CLEANUP_INBOX_JAR_EXPIRATION_SECS. Note that the time - * it takes to delete an inbox jar file is going to be somewhat more than NIMBUS_CLEANUP_INBOX_JAR_EXPIRATION_SECS + *

      Probably keep this value greater than or equal to + * NIMBUS_CLEANUP_INBOX_JAR_EXPIRATION_SECS. Note that the time + * it takes to delete an inbox jar file is going to be somewhat more than + * NIMBUS_CLEANUP_INBOX_JAR_EXPIRATION_SECS * (depending on how often NIMBUS_CLEANUP_FREQ_SECS is set to). * - *

      This is also how long a dependency blob uploaded by a client may go without any topology referring to it - * before nimbus deletes it from the blob store, which covers the time between uploading the dependencies of a + *

      This is also how long a dependency blob uploaded by a client may go without any topology + * referring to it + * before nimbus deletes it from the blob store, which covers the time between uploading the + * dependencies of a * topology and submitting it. * * @see #NIMBUS_CLEANUP_INBOX_FREQ_SECS */ @IsInteger - public static final String NIMBUS_INBOX_JAR_EXPIRATION_SECS = "nimbus.inbox.jar.expiration.secs"; + public static final String NIMBUS_INBOX_JAR_EXPIRATION_SECS = + "nimbus.inbox.jar.expiration.secs"; /** - * How long before a supervisor can go without heartbeating before nimbus considers it dead and stops assigning new work to it. + * How long before a supervisor can go without heartbeating before nimbus considers it dead and + * stops assigning new work to it. * * @deprecated Unused. Supervisor liveness is tracked via an ephemeral ZooKeeper node (see * {@code StormClusterState#supervisorHeartbeat}); when a supervisor dies its ZooKeeper session - * expires and the node disappears, so Nimbus detects the loss directly rather than by timing out - * heartbeats. No code reads this value. It is scheduled for removal; retained for now only for + * expires and the node disappears, so Nimbus detects the loss directly rather than by + * timing out + * heartbeats. No code reads this value. It is scheduled for removal; retained for now only + * for * backward compatibility. */ @Deprecated(forRemoval = true, since = "3.1.0") @@ -294,7 +335,8 @@ public class DaemonConfig implements Validated { public static final String NIMBUS_SUPERVISOR_TIMEOUT_SECS = "nimbus.supervisor.timeout.secs"; /** - * A special timeout used when a task is initially launched. During launch, this is the timeout used until the first heartbeat, + * A special timeout used when a task is initially launched. During launch, this is the timeout + * used until the first heartbeat, * overriding nimbus.task.timeout.secs. * *

      A separate timeout exists for launch because there can be quite a bit of overhead @@ -306,14 +348,17 @@ public class DaemonConfig implements Validated { public static final String NIMBUS_TASK_LAUNCH_SECS = "nimbus.task.launch.secs"; /** - * During upload/download with the master, how long an upload or download connection is idle before nimbus considers it dead and drops + * During upload/download with the master, how long an upload or download connection is idle + * before nimbus considers it dead and drops * the connection. */ @IsInteger - public static final String NIMBUS_FILE_COPY_EXPIRATION_SECS = "nimbus.file.copy.expiration.secs"; + public static final String NIMBUS_FILE_COPY_EXPIRATION_SECS = + "nimbus.file.copy.expiration.secs"; /** - * A custom class that implements ITopologyValidator that is run whenever a topology is submitted. Can be used to provide + * A custom class that implements ITopologyValidator that is run whenever a topology is + * submitted. Can be used to provide * business-specific logic for whether topologies are allowed to run or not. */ @IsString @@ -343,37 +388,45 @@ public class DaemonConfig implements Validated { */ @IsInteger @IsPositiveNumber - public static final String NIMBUS_CREDENTIAL_RENEW_FREQ_SECS = "nimbus.credential.renewers.freq.secs"; + public static final String NIMBUS_CREDENTIAL_RENEW_FREQ_SECS = + "nimbus.credential.renewers.freq.secs"; /** * FQCN of a class that implements {@code I} @see org.apache.storm.nimbus.ITopologyActionNotifierPlugin for details. */ @IsImplementationOfClass(implementsClass = ITopologyActionNotifierPlugin.class) - public static final String NIMBUS_TOPOLOGY_ACTION_NOTIFIER_PLUGIN = "nimbus.topology.action.notifier.plugin.class"; + public static final String NIMBUS_TOPOLOGY_ACTION_NOTIFIER_PLUGIN = + "nimbus.topology.action.notifier.plugin.class"; /** - * This controls the number of working threads for distributing master assignments to supervisors. + * This controls the number of working threads for distributing master assignments to + * supervisors. */ @IsInteger - public static final String NIMBUS_ASSIGNMENTS_SERVICE_THREADS = "nimbus.assignments.service.threads"; + public static final String NIMBUS_ASSIGNMENTS_SERVICE_THREADS = + "nimbus.assignments.service.threads"; /** * This controls the number of working thread queue size of assignment service. */ @IsInteger - public static final String NIMBUS_ASSIGNMENTS_SERVICE_THREAD_QUEUE_SIZE = "nimbus.assignments.service.thread.queue.size"; + public static final String NIMBUS_ASSIGNMENTS_SERVICE_THREAD_QUEUE_SIZE = + "nimbus.assignments.service.thread.queue.size"; /** - * class controls heartbeats recovery strategy. + * Class controls heartbeats recovery strategy. */ @IsString - public static final String NIMBUS_WORKER_HEARTBEATS_RECOVERY_STRATEGY_CLASS = "nimbus.worker.heartbeats.recovery.strategy.class"; + public static final String NIMBUS_WORKER_HEARTBEATS_RECOVERY_STRATEGY_CLASS = + "nimbus.worker.heartbeats.recovery.strategy.class"; /** - * This controls the number of milliseconds nimbus will wait before deleting a topology blobstore once detected it is able to delete. + * This controls the number of milliseconds nimbus will wait before deleting a topology + * blobstore once detected it is able to delete. */ @IsInteger - public static final String NIMBUS_TOPOLOGY_BLOBSTORE_DELETION_DELAY_MS = "nimbus.topology.blobstore.deletion.delay.ms"; + public static final String NIMBUS_TOPOLOGY_BLOBSTORE_DELETION_DELAY_MS = + "nimbus.topology.blobstore.deletion.delay.ms"; /** * Storm UI binds to this host/interface. @@ -415,7 +468,8 @@ public class DaemonConfig implements Validated { public static final String UI_ENABLE_JSONP = "ui.enable.jsonp"; /** - * This controls wheather Storm Logviewer should bind to http port even if logviewer.port is > 0. + * This controls wheather Storm Logviewer should bind to http port even if logviewer.port is > + * 0. */ @IsBoolean public static final String LOGVIEWER_DISABLE_HTTP_BINDING = "logviewer.disable.http.binding"; @@ -439,7 +493,8 @@ public class DaemonConfig implements Validated { public static final String UI_CENTRAL_LOGGING_URL = "ui.central.logging.url"; /** - * Storm UI drop-down pagination value. Set ui.pagination to be a positive integer or -1 (displays all entries). Valid values: -1, 10, + * Storm UI drop-down pagination value. Set ui.pagination to be a positive integer or -1 + * (displays all entries). Valid values: -1, 10, * 20, 25 etc. */ @IsInteger @@ -476,13 +531,15 @@ public class DaemonConfig implements Validated { * The maximum number of bytes all worker log files can take up in MB. */ @IsPositiveNumber - public static final String LOGVIEWER_MAX_SUM_WORKER_LOGS_SIZE_MB = "logviewer.max.sum.worker.logs.size.mb"; + public static final String LOGVIEWER_MAX_SUM_WORKER_LOGS_SIZE_MB = + "logviewer.max.sum.worker.logs.size.mb"; /** * The maximum number of bytes per worker's files can take up in MB. */ @IsPositiveNumber - public static final String LOGVIEWER_MAX_PER_WORKER_LOGS_SIZE_MB = "logviewer.max.per.worker.logs.size.mb"; + public static final String LOGVIEWER_MAX_PER_WORKER_LOGS_SIZE_MB = + "logviewer.max.per.worker.logs.size.mb"; /** * Storm Logviewer HTTPS port. Logviewer must use HTTPS if Storm UI is using HTTPS. @@ -502,7 +559,8 @@ public class DaemonConfig implements Validated { */ @IsString @Password - public static final String LOGVIEWER_HTTPS_KEYSTORE_PASSWORD = "logviewer.https.keystore.password"; + public static final String LOGVIEWER_HTTPS_KEYSTORE_PASSWORD = + "logviewer.https.keystore.password"; /** * Type of the keystore for HTTPS for Storm Logviewer. see http://docs.oracle.com/javase/8/docs/api/java/security/KeyStore.html for more @@ -529,7 +587,8 @@ public class DaemonConfig implements Validated { */ @IsString @Password - public static final String LOGVIEWER_HTTPS_TRUSTSTORE_PASSWORD = "logviewer.https.truststore.password"; + public static final String LOGVIEWER_HTTPS_TRUSTSTORE_PASSWORD = + "logviewer.https.truststore.password"; /** * Type of the truststore for HTTPS for Storm Logviewer. see http://docs.oracle.com/javase/8/docs/api/java/security/Truststore.html for @@ -542,16 +601,20 @@ public class DaemonConfig implements Validated { * Password to the truststore used by Storm Logviewer setting up HTTPS (SSL). */ @IsBoolean - public static final String LOGVIEWER_HTTPS_WANT_CLIENT_AUTH = "logviewer.https.want.client.auth"; + public static final String LOGVIEWER_HTTPS_WANT_CLIENT_AUTH = + "logviewer.https.want.client.auth"; @IsBoolean - public static final String LOGVIEWER_HTTPS_NEED_CLIENT_AUTH = "logviewer.https.need.client.auth"; + public static final String LOGVIEWER_HTTPS_NEED_CLIENT_AUTH = + "logviewer.https.need.client.auth"; /** - * If set to true, keystore and truststore for Logviewer will be automatically reloaded when modified. + * If set to true, keystore and truststore for Logviewer will be automatically reloaded when + * modified. */ @IsBoolean - public static final String LOGVIEWER_HTTPS_ENABLE_SSL_RELOAD = "logviewer.https.enable.ssl.reload"; + public static final String LOGVIEWER_HTTPS_ENABLE_SSL_RELOAD = + "logviewer.https.enable.ssl.reload"; /** * A list of users allowed to view logs via the Log Viewer. @@ -680,11 +743,14 @@ public class DaemonConfig implements Validated { public static final String UI_HTTPS_ENABLE_SSL_RELOAD = "ui.https.enable.ssl.reload"; /** - * The maximum number of threads that should be used by the Pacemaker. When Pacemaker gets loaded it will spawn new threads, up to this + * The maximum number of threads that should be used by the Pacemaker. When Pacemaker gets + * loaded it will spawn new threads, up to this * many total, to handle the load. * - * @deprecated Pacemaker is deprecated and only kept for backward compatibility; it will be removed in a future release. - * Use the default heartbeat path (workers heartbeat to their supervisor, which reports them to Nimbus) instead. + * @deprecated Pacemaker is deprecated and only kept for backward compatibility; it will be + * removed in a future release. + * Use the default heartbeat path (workers heartbeat to their supervisor, which reports them + * to Nimbus) instead. */ @IsNumber @IsPositiveNumber @@ -692,10 +758,13 @@ public class DaemonConfig implements Validated { public static final String PACEMAKER_MAX_THREADS = "pacemaker.max.threads"; /** - * This parameter is used by the storm-deploy project to configure the jvm options for the pacemaker daemon. + * This parameter is used by the storm-deploy project to configure the jvm options for the + * pacemaker daemon. * - * @deprecated Pacemaker is deprecated and only kept for backward compatibility; it will be removed in a future release. - * Use the default heartbeat path (workers heartbeat to their supervisor, which reports them to Nimbus) instead. + * @deprecated Pacemaker is deprecated and only kept for backward compatibility; it will be + * removed in a future release. + * Use the default heartbeat path (workers heartbeat to their supervisor, which reports them + * to Nimbus) instead. */ @IsStringOrStringList @Deprecated @@ -773,7 +842,8 @@ public class DaemonConfig implements Validated { public static final String DRPC_HTTPS_NEED_CLIENT_AUTH = "drpc.https.need.client.auth"; /** - * If set to true, keystore and truststore for DRPC Server will be automatically reloaded when modified. + * If set to true, keystore and truststore for DRPC Server will be automatically reloaded when + * modified. */ @IsBoolean public static final String DRPC_HTTPS_ENABLE_SSL_RELOAD = "drpc.https.enable.ssl.reload"; @@ -785,8 +855,10 @@ public class DaemonConfig implements Validated { public static final String DRPC_AUTHORIZER = "drpc.authorizer"; /** - * The timeout on DRPC requests within the DRPC server. Defaults to 10 minutes. Note that requests can also timeout based on the socket - * timeout on the DRPC client, and separately based on the topology message timeout for the topology implementing the DRPC function. + * The timeout on DRPC requests within the DRPC server. Defaults to 10 minutes. Note that + * requests can also timeout based on the socket + * timeout on the DRPC client, and separately based on the topology message timeout for the + * topology implementing the DRPC function. */ @IsInteger @@ -801,18 +873,20 @@ public class DaemonConfig implements Validated { public static final String DRPC_CHILDOPTS = "drpc.childopts"; /** - * the metadata configured on the supervisor. + * The metadata configured on the supervisor. */ @IsMapEntryType(keyType = String.class, valueType = String.class) public static final String SUPERVISOR_SCHEDULER_META = "supervisor.scheduler.meta"; /** - * A list of ports that can run workers on this supervisor. Each worker uses one port, and the supervisor will only run one worker per + * A list of ports that can run workers on this supervisor. Each worker uses one port, and the + * supervisor will only run one worker per * port. Use this configuration to tune how many workers run on each machine. */ @IsNoDuplicateInList @NotNull - @IsListEntryCustom(entryValidatorClasses = { ConfigValidation.IntegerValidator.class, ConfigValidation.PositiveNumberValidator.class }) + @IsListEntryCustom(entryValidatorClasses = { ConfigValidation.IntegerValidator.class, + ConfigValidation.PositiveNumberValidator.class }) public static final String SUPERVISOR_SLOTS_PORTS = "supervisor.slots.ports"; /** @@ -822,39 +896,46 @@ public class DaemonConfig implements Validated { public static final String SUPERVISOR_BLOBSTORE = "supervisor.blobstore.class"; /** - * The distributed cache target size in MB. This is a soft limit to the size of the distributed cache contents. + * The distributed cache target size in MB. This is a soft limit to the size of the distributed + * cache contents. */ @IsPositiveNumber @IsInteger - public static final String SUPERVISOR_LOCALIZER_CACHE_TARGET_SIZE_MB = "supervisor.localizer.cache.target.size.mb"; + public static final String SUPERVISOR_LOCALIZER_CACHE_TARGET_SIZE_MB = + "supervisor.localizer.cache.target.size.mb"; /** - * The distributed cache cleanup interval. Controls how often it scans to attempt to cleanup anything over the cache target size. + * The distributed cache cleanup interval. Controls how often it scans to attempt to cleanup + * anything over the cache target size. */ @IsPositiveNumber @IsInteger - public static final String SUPERVISOR_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS = "supervisor.localizer.cleanup.interval.ms"; + public static final String SUPERVISOR_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS = + "supervisor.localizer.cleanup.interval.ms"; /** * The distributed cache interval for checking for blobs to update. */ @IsPositiveNumber @IsInteger - public static final String SUPERVISOR_LOCALIZER_UPDATE_BLOB_INTERVAL_SECS = "supervisor.localizer.update.blob.interval.secs"; + public static final String SUPERVISOR_LOCALIZER_UPDATE_BLOB_INTERVAL_SECS = + "supervisor.localizer.update.blob.interval.secs"; /** * What blobstore download parallelism the supervisor should use. */ @IsPositiveNumber @IsInteger - public static final String SUPERVISOR_BLOBSTORE_DOWNLOAD_THREAD_COUNT = "supervisor.blobstore.download.thread.count"; + public static final String SUPERVISOR_BLOBSTORE_DOWNLOAD_THREAD_COUNT = + "supervisor.blobstore.download.thread.count"; /** * Maximum number of retries a supervisor is allowed to make for downloading a blob. */ @IsPositiveNumber @IsInteger - public static final String SUPERVISOR_BLOBSTORE_DOWNLOAD_MAX_RETRIES = "supervisor.blobstore.download.max_retries"; + public static final String SUPERVISOR_BLOBSTORE_DOWNLOAD_MAX_RETRIES = + "supervisor.blobstore.download.max_retries"; /** * A map with keys mapped to each NUMA Node on the supervisor that will be used @@ -880,12 +961,14 @@ public class DaemonConfig implements Validated { public static final String NIMBUS_BLOBSTORE = "nimbus.blobstore.class"; /** - * During operations with the blob store, via master, how long a connection is idle before nimbus considers it dead and drops the + * During operations with the blob store, via master, how long a connection is idle before + * nimbus considers it dead and drops the * session and any associated connections. */ @IsPositiveNumber @IsInteger - public static final String NIMBUS_BLOBSTORE_EXPIRATION_SECS = "nimbus.blobstore.expiration.secs"; + public static final String NIMBUS_BLOBSTORE_EXPIRATION_SECS = + "nimbus.blobstore.expiration.secs"; /** * A number representing the maximum number of workers any single topology can acquire. @@ -915,42 +998,50 @@ public class DaemonConfig implements Validated { public static final String NIMBUS_EXECUTORS_PER_TOPOLOGY = "nimbus.executors.perTopology"; /** - * This parameter is used by the storm-deploy project to configure the jvm options for the supervisor daemon. + * This parameter is used by the storm-deploy project to configure the jvm options for the + * supervisor daemon. */ @IsStringOrStringList public static final String SUPERVISOR_CHILDOPTS = "supervisor.childopts"; /** - * How long a worker can go without heartbeating during the initial launch before the supervisor tries to restart the worker process. - * This value override supervisor.worker.timeout.secs during launch because there is additional overhead to starting and configuring the + * How long a worker can go without heartbeating during the initial launch before the supervisor + * tries to restart the worker process. + * This value override supervisor.worker.timeout.secs during launch because there is additional + * overhead to starting and configuring the * JVM on launch. * Can be exceeded when {@link Config#TOPOLOGY_WORKER_TIMEOUT_SECS} is set. */ @IsInteger @IsPositiveNumber @NotNull - public static final String SUPERVISOR_WORKER_START_TIMEOUT_SECS = "supervisor.worker.start.timeout.secs"; + public static final String SUPERVISOR_WORKER_START_TIMEOUT_SECS = + "supervisor.worker.start.timeout.secs"; /** - * Whether or not the supervisor should launch workers assigned to it. Defaults to true -- and you should probably never change this + * Whether or not the supervisor should launch workers assigned to it. Defaults to true -- and + * you should probably never change this * value. This configuration is used in the Storm unit tests. */ @IsBoolean public static final String SUPERVISOR_ENABLE = "supervisor.enable"; /** - * how often the supervisor sends a heartbeat to the master. + * How often the supervisor sends a heartbeat to the master. */ @IsInteger - public static final String SUPERVISOR_HEARTBEAT_FREQUENCY_SECS = "supervisor.heartbeat.frequency.secs"; + public static final String SUPERVISOR_HEARTBEAT_FREQUENCY_SECS = + "supervisor.heartbeat.frequency.secs"; /** - * How often the supervisor checks the worker heartbeats to see if any of them need to be restarted. + * How often the supervisor checks the worker heartbeats to see if any of them need to be + * restarted. */ @IsInteger @IsPositiveNumber - public static final String SUPERVISOR_MONITOR_FREQUENCY_SECS = "supervisor.monitor.frequency.secs"; + public static final String SUPERVISOR_MONITOR_FREQUENCY_SECS = + "supervisor.monitor.frequency.secs"; /** @@ -960,26 +1051,32 @@ public class DaemonConfig implements Validated { public static final String WORKER_PROFILER_CHILDOPTS = "worker.profiler.childopts"; /** - * Enable profiling of worker JVMs using Oracle's Java Flight Recorder. Unlocking commercial features requires a special license from + * Enable profiling of worker JVMs using Oracle's Java Flight Recorder. Unlocking commercial + * features requires a special license from * Oracle. See http://www.oracle.com/technetwork/java/javase/terms/products/index.html */ @IsBoolean public static final String WORKER_PROFILER_ENABLED = "worker.profiler.enabled"; /** - * The command launched supervisor with worker arguments pid, action and [target_directory] Where action is - start profile, stop + * The command launched supervisor with worker arguments pid, action and [target_directory] + * Where action is - start profile, stop * profile, jstack, heapdump and kill against pid. */ @IsString public static final String WORKER_PROFILER_COMMAND = "worker.profiler.command"; /** - * A list of classes implementing IClusterMetricsConsumer (See storm.yaml.example for exact config format). Each listed class will be - * routed cluster related metrics data. Each listed class maps 1:1 to a ClusterMetricsConsumerExecutor and they're executed in Nimbus. + * A list of classes implementing IClusterMetricsConsumer (See storm.yaml.example for exact + * config format). Each listed class will be + * routed cluster related metrics data. Each listed class maps 1:1 to a + * ClusterMetricsConsumerExecutor and they're executed in Nimbus. * Only consumers which run in leader Nimbus receives metrics data. */ - @IsListEntryCustom(entryValidatorClasses = { ConfigValidation.ClusterMetricRegistryValidator.class }) - public static final String STORM_CLUSTER_METRICS_CONSUMER_REGISTER = "storm.cluster.metrics.consumer.register"; + @IsListEntryCustom(entryValidatorClasses = + { ConfigValidation.ClusterMetricRegistryValidator.class }) + public static final String STORM_CLUSTER_METRICS_CONSUMER_REGISTER = + "storm.cluster.metrics.consumer.register"; /** * How often cluster metrics data is published to metrics consumer. @@ -993,25 +1090,30 @@ public class DaemonConfig implements Validated { * Enables user-first classpath. See topology.classpath.beginning. */ @IsBoolean - public static final String STORM_TOPOLOGY_CLASSPATH_BEGINNING_ENABLED = "storm.topology.classpath.beginning.enabled"; + public static final String STORM_TOPOLOGY_CLASSPATH_BEGINNING_ENABLED = + "storm.topology.classpath.beginning.enabled"; /** - * This value is passed to spawned JVMs (e.g., Nimbus, Supervisor, and Workers) for the java.library.path value. java.library.path tells - * the JVM where to look for native libraries. It is necessary to set this config correctly since Storm uses the ZeroMQ and JZMQ native + * This value is passed to spawned JVMs (e.g., Nimbus, Supervisor, and Workers) for the + * java.library.path value. java.library.path tells + * the JVM where to look for native libraries. It is necessary to set this config correctly + * since Storm uses the ZeroMQ and JZMQ native * libs. */ @IsString public static final String JAVA_LIBRARY_PATH = "java.library.path"; /** - * The path to use as the zookeeper dir when running a zookeeper server via "storm dev-zookeeper". This zookeeper instance is only + * The path to use as the zookeeper dir when running a zookeeper server via "storm + * dev-zookeeper". This zookeeper instance is only * intended for development; it is not a production grade zookeeper setup. */ @IsString public static final String DEV_ZOOKEEPER_PATH = "dev.zookeeper.path"; /** - * A map from topology name to the number of machines that should be dedicated for that topology. Set storm.scheduler to + * A map from topology name to the number of machines that should be dedicated for that + * topology. Set storm.scheduler to * org.apache.storm.scheduler.IsolationScheduler to make use of the isolation scheduler. */ @IsMapEntryType(keyType = String.class, valueType = Number.class) @@ -1022,64 +1124,79 @@ public class DaemonConfig implements Validated { */ @IsInteger @IsPositiveNumber(includeZero = true) - public static final String SCHEDULER_CONFIG_CACHE_EXPIRATION_SECS = "scheduler.config.cache.expiration.secs"; + public static final String SCHEDULER_CONFIG_CACHE_EXPIRATION_SECS = + "scheduler.config.cache.expiration.secs"; /** - * For ArtifactoryConfigLoader, this can either be a reference to an individual file in Artifactory or to a directory. If it is a - * directory, the file with the largest lexographic name will be returned. Users need to add "artifactory+" to the beginning of the real + * For ArtifactoryConfigLoader, this can either be a reference to an individual file in + * Artifactory or to a directory. If it is a + * directory, the file with the largest lexographic name will be returned. Users need to add + * "artifactory+" to the beginning of the real * URI to use ArtifactoryConfigLoader. For FileConfigLoader, this is the URI pointing to a file. */ @IsString public static final String SCHEDULER_CONFIG_LOADER_URI = "scheduler.config.loader.uri"; /** - * It is the frequency at which the plugin will call out to artifactory instead of returning the most recently cached result. Currently + * It is the frequency at which the plugin will call out to artifactory instead of returning the + * most recently cached result. Currently * it's only used in ArtifactoryConfigLoader. */ @IsInteger @IsPositiveNumber - public static final String SCHEDULER_CONFIG_LOADER_POLLTIME_SECS = "scheduler.config.loader.polltime.secs"; + public static final String SCHEDULER_CONFIG_LOADER_POLLTIME_SECS = + "scheduler.config.loader.polltime.secs"; /** - * It is the amount of time an http connection to the artifactory server will wait before timing out. Currently it's only used in + * It is the amount of time an http connection to the artifactory server will wait before timing + * out. Currently it's only used in * ArtifactoryConfigLoader. */ @IsInteger @IsPositiveNumber - public static final String SCHEDULER_CONFIG_LOADER_TIMEOUT_SECS = "scheduler.config.loader.timeout.secs"; + public static final String SCHEDULER_CONFIG_LOADER_TIMEOUT_SECS = + "scheduler.config.loader.timeout.secs"; /** - * It is the part of the uri, configurable in Artifactory, which represents the top of the directory tree. It's only used in + * It is the part of the uri, configurable in Artifactory, which represents the top of the + * directory tree. It's only used in * ArtifactoryConfigLoader. */ @IsString - public static final String SCHEDULER_CONFIG_LOADER_ARTIFACTORY_BASE_DIRECTORY = "scheduler.config.loader.artifactory.base.directory"; + public static final String SCHEDULER_CONFIG_LOADER_ARTIFACTORY_BASE_DIRECTORY = + "scheduler.config.loader.artifactory.base.directory"; /** - * A map from the user name to the number of machines that should that user is allowed to use. Set storm.scheduler to + * A map from the user name to the number of machines that should that user is allowed to use. + * Set storm.scheduler to * org.apache.storm.scheduler.multitenant.MultitenantScheduler */ @IsMapEntryType(keyType = String.class, valueType = Number.class) - public static final String MULTITENANT_SCHEDULER_USER_POOLS = "multitenant.scheduler.user.pools"; + public static final String MULTITENANT_SCHEDULER_USER_POOLS = + "multitenant.scheduler.user.pools"; /** - * A map of users to another map of the resource guarantees of the user. Used by Resource Aware Scheduler to ensure per user resource + * A map of users to another map of the resource guarantees of the user. Used by Resource Aware + * Scheduler to ensure per user resource * guarantees. */ @IsMapEntryCustom( keyValidatorClasses = { ConfigValidation.StringValidator.class }, valueValidatorClasses = { ConfigValidation.UserResourcePoolEntryValidator.class }) - public static final String RESOURCE_AWARE_SCHEDULER_USER_POOLS = "resource.aware.scheduler.user.pools"; + public static final String RESOURCE_AWARE_SCHEDULER_USER_POOLS = + "resource.aware.scheduler.user.pools"; /** - * the class that specifies the scheduling priority strategy to use in ResourceAwareScheduler. + * The class that specifies the scheduling priority strategy to use in ResourceAwareScheduler. */ @NotNull @IsImplementationOfClass(implementsClass = ISchedulingPriorityStrategy.class) - public static final String RESOURCE_AWARE_SCHEDULER_PRIORITY_STRATEGY = "resource.aware.scheduler.priority.strategy"; + public static final String RESOURCE_AWARE_SCHEDULER_PRIORITY_STRATEGY = + "resource.aware.scheduler.priority.strategy"; /** - * The maximum number of times that the RAS will attempt to schedule a topology. The default is 5. + * The maximum number of times that the RAS will attempt to schedule a topology. The default is + * 5. */ @IsInteger @IsPositiveNumber @@ -1087,11 +1204,13 @@ public class DaemonConfig implements Validated { "resource.aware.scheduler.max.topology.scheduling.attempts"; /* - * The maximum number of states that will be searched looking for a solution in the constraint solver strategy + * The maximum number of states that will be searched looking for a solution in the constraint + * solver strategy */ @IsInteger @IsPositiveNumber - public static final String RESOURCE_AWARE_SCHEDULER_MAX_STATE_SEARCH = "resource.aware.scheduler.constraint.max.state.search"; + public static final String RESOURCE_AWARE_SCHEDULER_MAX_STATE_SEARCH = + "resource.aware.scheduler.constraint.max.state.search"; /** * How often nimbus's background thread to sync code for missing topologies should run. @@ -1110,29 +1229,33 @@ public class DaemonConfig implements Validated { */ /** - * resources to to be controlled by cgroups. + * Resources to to be controlled by cgroups. */ @IsStringList public static final String STORM_CGROUP_RESOURCES = "storm.cgroup.resources"; /** - * name for the cgroup hierarchy. + * Name for the cgroup hierarchy. */ @IsString public static final String STORM_CGROUP_HIERARCHY_NAME = "storm.cgroup.hierarchy.name"; /** - * flag to determine whether to use a resource isolation plugin Also determines whether the unit tests for cgroup runs. If - * storm.resource.isolation.plugin.enable is set to false the unit tests for cgroups will not run + * Flag to determine whether to use a resource isolation plugin Also determines whether the unit + * tests for cgroup runs. If + * storm.resource.isolation.plugin.enable is set to false the unit tests for cgroups will not + * run */ @IsBoolean - public static final String STORM_RESOURCE_ISOLATION_PLUGIN_ENABLE = "storm.resource.isolation.plugin.enable"; + public static final String STORM_RESOURCE_ISOLATION_PLUGIN_ENABLE = + "storm.resource.isolation.plugin.enable"; /** * Class implementing MetricStore. Runs on Nimbus. */ @NotNull @IsString - // Validating class implementation could fail on non-Nimbus Daemons. Nimbus will catch the class not found on startup + // Validating class implementation could fail on non-Nimbus Daemons. Nimbus will catch the class + // not found on startup // and log an error message, so just validating this as a String for now. public static final String STORM_METRIC_STORE_CLASS = "storm.metricstore.class"; /** @@ -1142,54 +1265,70 @@ public class DaemonConfig implements Validated { @IsString public static final String STORM_METRIC_PROCESSOR_CLASS = "storm.metricprocessor.class"; /** - * RocksDB file location. This setting is specific to the org.apache.storm.metricstore.rocksdb.RocksDbStore implementation for the + * RocksDB file location. This setting is specific to the + * org.apache.storm.metricstore.rocksdb.RocksDbStore implementation for the * storm.metricstore.class. */ @IsString public static final String STORM_ROCKSDB_LOCATION = "storm.metricstore.rocksdb.location"; /** - * RocksDB create if missing flag. This setting is specific to the org.apache.storm.metricstore.rocksdb.RocksDbStore implementation for + * RocksDB create if missing flag. This setting is specific to the + * org.apache.storm.metricstore.rocksdb.RocksDbStore implementation for * the storm.metricstore.class. */ @IsBoolean - public static final String STORM_ROCKSDB_CREATE_IF_MISSING = "storm.metricstore.rocksdb.create_if_missing"; + public static final String STORM_ROCKSDB_CREATE_IF_MISSING = + "storm.metricstore.rocksdb.create_if_missing"; /** - * RocksDB metadata cache capacity. This setting is specific to the org.apache.storm.metricstore.rocksdb.RocksDbStore implementation for + * RocksDB metadata cache capacity. This setting is specific to the + * org.apache.storm.metricstore.rocksdb.RocksDbStore implementation for * the storm.metricstore.class. */ @IsInteger - public static final String STORM_ROCKSDB_METADATA_STRING_CACHE_CAPACITY = "storm.metricstore.rocksdb.metadata_string_cache_capacity"; + public static final String STORM_ROCKSDB_METADATA_STRING_CACHE_CAPACITY = + "storm.metricstore.rocksdb.metadata_string_cache_capacity"; /** - * RocksDB setting for length of metric retention. This setting is specific to the org.apache.storm.metricstore.rocksdb.RocksDbStore + * RocksDB setting for length of metric retention. This setting is specific to the + * org.apache.storm.metricstore.rocksdb.RocksDbStore * implementation for the storm.metricstore.class. */ @IsInteger - public static final String STORM_ROCKSDB_METRIC_RETENTION_HOURS = "storm.metricstore.rocksdb.retention_hours"; + public static final String STORM_ROCKSDB_METRIC_RETENTION_HOURS = + "storm.metricstore.rocksdb.retention_hours"; // Configs for memory enforcement done by the supervisor (not cgroups directly) /** - * RocksDB setting for period of metric deletion thread. This setting is specific to the org.apache.storm.metricstore.rocksdb + * RocksDB setting for period of metric deletion thread. This setting is specific to the + * org.apache.storm.metricstore.rocksdb * .RocksDbStore * implementation for the storm.metricstore.class. */ @IsInteger - public static final String STORM_ROCKSDB_METRIC_DELETION_PERIOD_HOURS = "storm.metricstore.rocksdb.deletion_period_hours"; + public static final String STORM_ROCKSDB_METRIC_DELETION_PERIOD_HOURS = + "storm.metricstore.rocksdb.deletion_period_hours"; /** - * In nimbus on startup check if all of the zookeeper ACLs are correct before starting. If not don't start nimbus. + * In nimbus on startup check if all of the zookeeper ACLs are correct before starting. If not + * don't start nimbus. */ @IsBoolean - public static final String STORM_NIMBUS_ZOOKEEPER_ACLS_CHECK = "storm.nimbus.zookeeper.acls.check"; + public static final String STORM_NIMBUS_ZOOKEEPER_ACLS_CHECK = + "storm.nimbus.zookeeper.acls.check"; /** - * In nimbus on startup check if all of the zookeeper ACLs are correct before starting. If not do your best to fix them before nimbus - * starts, if it cannot fix them nimbus will not start. This overrides any value set for storm.nimbus.zookeeper.acls.check. + * In nimbus on startup check if all of the zookeeper ACLs are correct before starting. If not + * do your best to fix them before nimbus + * starts, if it cannot fix them nimbus will not start. This overrides any value set for + * storm.nimbus.zookeeper.acls.check. */ @IsBoolean - public static final String STORM_NIMBUS_ZOOKEEPER_ACLS_FIXUP = "storm.nimbus.zookeeper.acls.fixup"; + public static final String STORM_NIMBUS_ZOOKEEPER_ACLS_FIXUP = + "storm.nimbus.zookeeper.acls.fixup"; /** - * Server side validation that @{see Config#TOPOLOGY_SCHEDULER_STRATEGY} is set ot a subclass of IStrategy. + * Server side validation that @{see Config#TOPOLOGY_SCHEDULER_STRATEGY} is set ot a subclass of + * IStrategy. */ @IsImplementationOfClass(implementsClass = IStrategy.class) - public static final String VALIDATE_TOPOLOGY_SCHEDULER_STRATEGY = Config.TOPOLOGY_SCHEDULER_STRATEGY; + public static final String VALIDATE_TOPOLOGY_SCHEDULER_STRATEGY = + Config.TOPOLOGY_SCHEDULER_STRATEGY; /** * Class name of the HTTP credentials plugin for the UI. @@ -1204,85 +1343,109 @@ public class DaemonConfig implements Validated { public static final String DRPC_HTTP_CREDS_PLUGIN = "drpc.http.creds.plugin"; /** - * root directory for cgoups. + * Root directory for cgoups. */ @IsString public static String STORM_SUPERVISOR_CGROUP_ROOTDIR = "storm.supervisor.cgroup.rootdir"; /** - * the manually set memory limit (in MB) for each CGroup on supervisor node. + * The manually set memory limit (in MB) for each CGroup on supervisor node. */ @IsPositiveNumber - public static String STORM_WORKER_CGROUP_MEMORY_MB_LIMIT = "storm.worker.cgroup.memory.mb.limit"; + public static String STORM_WORKER_CGROUP_MEMORY_MB_LIMIT = + "storm.worker.cgroup.memory.mb.limit"; /** - * the manually set cpu share for each CGroup on supervisor node. + * The manually set cpu share for each CGroup on supervisor node. */ @IsPositiveNumber public static String STORM_WORKER_CGROUP_CPU_LIMIT = "storm.worker.cgroup.cpu.limit"; /** - * full path to cgexec command. + * Full path to cgexec command. */ @IsString public static String STORM_CGROUP_CGEXEC_CMD = "storm.cgroup.cgexec.cmd"; /** - * Please use STORM_SUPERVISOR_MEMORY_LIMIT_TOLERANCE_MARGIN_MB instead. The amount of memory a worker can exceed its allocation before + * Please use STORM_SUPERVISOR_MEMORY_LIMIT_TOLERANCE_MARGIN_MB instead. The amount of memory a + * worker can exceed its allocation before * cgroup will kill it. */ @IsPositiveNumber(includeZero = true) public static String STORM_CGROUP_MEMORY_LIMIT_TOLERANCE_MARGIN_MB = "storm.cgroup.memory.limit.tolerance.margin.mb"; /** - * To determine whether or not to cgroups should inherit cpuset.cpus and cpuset.mems config values form parent cgroup - * Note that cpuset.cpus and cpuset.mems configs in a cgroup must be initialized (i.e. contain a valid value) prior to - * being able to launch processes in that cgroup. The common use case for this config is when the linux distribution + * To determine whether or not to cgroups should inherit cpuset.cpus and cpuset.mems config + * values form parent cgroup + * Note that cpuset.cpus and cpuset.mems configs in a cgroup must be initialized (i.e. contain a + * valid value) prior to + * being able to launch processes in that cgroup. The common use case for this config is when + * the linux distribution * that is used does not support the cgroup.clone_children config. */ @IsBoolean - public static String STORM_CGROUP_INHERIT_CPUSET_CONFIGS = "storm.cgroup.inherit.cpuset.configs"; + public static String STORM_CGROUP_INHERIT_CPUSET_CONFIGS = + "storm.cgroup.inherit.cpuset.configs"; /** - * Java does not always play nicely with cgroups. It is coming but not fully implemented and not for the way storm uses cgroups. In the - * short term you can disable the hard memory enforcement by cgroups and let the supervisor handle shooting workers going over their + * Java does not always play nicely with cgroups. It is coming but not fully implemented and not + * for the way storm uses cgroups. In the + * short term you can disable the hard memory enforcement by cgroups and let the supervisor + * handle shooting workers going over their * limit in a kinder way. */ @IsBoolean - public static String STORM_CGROUP_MEMORY_ENFORCEMENT_ENABLE = "storm.cgroup.memory.enforcement.enable"; + public static String STORM_CGROUP_MEMORY_ENFORCEMENT_ENABLE = + "storm.cgroup.memory.enforcement.enable"; /** - * Memory given to each worker for free (because java and storm have some overhead). This is memory on the box that the workers can use. - * This should not be included in SUPERVISOR_MEMORY_CAPACITY_MB, as nimbus does not use this memory for scheduling. + * Memory given to each worker for free (because java and storm have some overhead). This is + * memory on the box that the workers can use. + * This should not be included in SUPERVISOR_MEMORY_CAPACITY_MB, as nimbus does not use this + * memory for scheduling. */ @IsPositiveNumber public static String STORM_SUPERVISOR_MEMORY_LIMIT_TOLERANCE_MARGIN_MB = "storm.supervisor.memory.limit.tolerance.margin.mb"; /** - * A multiplier for the memory limit of a worker that will have the supervisor shoot it immediately. 1.0 means shoot the worker as soon - * as it goes over. 2.0 means shoot the worker if its usage is double what was requested. This value is combined with - * STORM_SUPERVISOR_HARD_MEMORY_LIMIT_OVERAGE and which ever is greater is used for enforcement. This allows small workers to not be + * A multiplier for the memory limit of a worker that will have the supervisor shoot it + * immediately. 1.0 means shoot the worker as soon + * as it goes over. 2.0 means shoot the worker if its usage is double what was requested. This + * value is combined with + * STORM_SUPERVISOR_HARD_MEMORY_LIMIT_OVERAGE and which ever is greater is used for enforcement. + * This allows small workers to not be * shot. */ @IsPositiveNumber public static String STORM_SUPERVISOR_HARD_MEMORY_LIMIT_MULTIPLIER = "storm.supervisor.hard.memory.limit.multiplier"; /** - * If the memory usage of a worker goes over its limit by this value is it shot immediately. This value is combined with - * STORM_SUPERVISOR_HARD_LIMIT_MEMORY_MULTIPLIER and which ever is greater is used for enforcement. This allows small workers to not be + * If the memory usage of a worker goes over its limit by this value is it shot immediately. + * This value is combined with + * STORM_SUPERVISOR_HARD_LIMIT_MEMORY_MULTIPLIER and which ever is greater is used for + * enforcement. This allows small workers to not be * shot. */ @IsPositiveNumber(includeZero = true) - public static String STORM_SUPERVISOR_HARD_LIMIT_MEMORY_OVERAGE_MB = "storm.supervisor.hard.memory.limit.overage.mb"; + public static String STORM_SUPERVISOR_HARD_LIMIT_MEMORY_OVERAGE_MB = + "storm.supervisor.hard.memory.limit.overage.mb"; /** - * If the amount of memory that is free in the system (either on the box or in the supervisor's cgroup) is below this number (in MB) - * consider the system to be in low memory mode and start shooting workers if they are over their limit. + * If the amount of memory that is free in the system (either on the box or in the supervisor's + * cgroup) is below this number (in MB) + * consider the system to be in low memory mode and start shooting workers if they are over + * their limit. */ @IsPositiveNumber - public static String STORM_SUPERVISOR_LOW_MEMORY_THRESHOLD_MB = "storm.supervisor.low.memory.threshold.mb"; + public static String STORM_SUPERVISOR_LOW_MEMORY_THRESHOLD_MB = + "storm.supervisor.low.memory.threshold.mb"; /** - * If the amount of memory that is free in the system (either on the box or in the supervisor's cgroup) is below this number (in MB) - * consider the system to be a little low on memory and start shooting workers if they are over their limit for a given grace period + * If the amount of memory that is free in the system (either on the box or in the supervisor's + * cgroup) is below this number (in MB) + * consider the system to be a little low on memory and start shooting workers if they are over + * their limit for a given grace period * STORM_SUPERVISOR_MEDIUM_MEMORY_GRACE_PERIOD_MS. */ @IsPositiveNumber - public static String STORM_SUPERVISOR_MEDIUM_MEMORY_THRESHOLD_MB = "storm.supervisor.medium.memory.threshold.mb"; + public static String STORM_SUPERVISOR_MEDIUM_MEMORY_THRESHOLD_MB = + "storm.supervisor.medium.memory.threshold.mb"; /** - * The number of milliseconds that a worker is allowed to be over their limit when there is a medium amount of memory free in the + * The number of milliseconds that a worker is allowed to be over their limit when there is a + * medium amount of memory free in the * system. */ @IsPositiveNumber @@ -1290,23 +1453,31 @@ public class DaemonConfig implements Validated { "storm.supervisor.medium.memory.grace.period.ms"; /** - * The config indicates the minimum percentage of cpu for a core that a worker will use. Assuming the a core value to be - * 100, a value of 10 indicates 10% of the core. The P in PCORE represents the term "physical". A default value will be set for this + * The config indicates the minimum percentage of cpu for a core that a worker will use. + * Assuming the a core value to be + * 100, a value of 10 indicates 10% of the core. The P in PCORE represents the term "physical". + * A default value will be set for this * config if user does not override. + * *

      - * Workers in containers or cgroups may require a minimum amount of CPU in order to launch within the supervisor timeout. + * Workers in containers or cgroups may require a minimum amount of CPU in order to launch + * within the supervisor timeout. * This setting allows configuring this to occur. */ @IsPositiveNumber(includeZero = true) public static String STORM_WORKER_MIN_CPU_PCORE_PERCENT = "storm.worker.min.cpu.pcore.percent"; // VALIDATION ONLY CONFIGS - // Some configs inside Config.java may reference classes we don't want to expose in storm-client, but we still want to validate - // That they reference a valid class. To allow this to happen we do part of the validation on the client side with annotations on - // static final members of the Config class, and other validations here. We avoid naming them the same thing because clojure code + // Some configs inside Config.java may reference classes we don't want to expose in + // storm-client, but we still want to validate + // That they reference a valid class. To allow this to happen we do part of the validation on + // the client side with annotations on + // static final members of the Config class, and other validations here. We avoid naming them + // the same thing because clojure code // walks these two classes and creates clojure constants for these values. /** - * The number of hours a worker token is valid for. This also sets how frequently worker tokens will be renewed. + * The number of hours a worker token is valid for. This also sets how frequently worker tokens + * will be renewed. */ @IsPositiveNumber public static String STORM_WORKER_TOKEN_LIFE_TIME_HOURS = "storm.worker.token.life.time.hours"; @@ -1335,8 +1506,10 @@ public class DaemonConfig implements Validated { * The cgroup root for oci container. (Also a --cgroup-parent config for docker command) * Must follow the constraints of the docker command. * The path will be made as absolute path if it's a relative path - * because we saw some weird bugs (the cgroup memory directory disappears after a while) when a relative path is used. - * Note that we only support cgroupfs cgroup driver because of some issues with systemd; restricting to `cgroupfs` + * because we saw some weird bugs (the cgroup memory directory disappears after a while) when a + * relative path is used. + * Note that we only support cgroupfs cgroup driver because of some issues with systemd; + * restricting to `cgroupfs` * also makes cgroup paths simple. */ @IsString @@ -1351,8 +1524,10 @@ public class DaemonConfig implements Validated { /** * A list of oci image that are allowed. - * A special entry of asterisk(*) means any image is allowed, but the image has to pass other checks. - * Storm currently assumes OCI container is not supported on the cluster if this is not configured. + * A special entry of asterisk(*) means any image is allowed, but the image has to pass other + * checks. + * Storm currently assumes OCI container is not supported on the cluster if this is not + * configured. */ @IsStringList public static String STORM_OCI_ALLOWED_IMAGES = "storm.oci.allowed.images"; @@ -1373,13 +1548,15 @@ public class DaemonConfig implements Validated { * The plugin to be used to get the image-tag to manifest mappings. */ @IsImplementationOfClass(implementsClass = OciImageTagToManifestPluginInterface.class) - public static final String STORM_OCI_IMAGE_TAG_TO_MANIFEST_PLUGIN = "storm.oci.image.tag.to.manifest.plugin"; + public static final String STORM_OCI_IMAGE_TAG_TO_MANIFEST_PLUGIN = + "storm.oci.image.tag.to.manifest.plugin"; /** * The plugin to be used to get oci resource according to the manifest. */ @IsImplementationOfClass(implementsClass = OciManifestToResourcesPluginInterface.class) - public static final String STORM_OCI_MANIFEST_TO_RESOURCES_PLUGIN = "storm.oci.manifest.to.resources.plugin"; + public static final String STORM_OCI_MANIFEST_TO_RESOURCES_PLUGIN = + "storm.oci.manifest.to.resources.plugin"; /** * The plugin to use for oci resources localization. diff --git a/storm-server/src/main/java/org/apache/storm/ILocalClusterTrackedTopologyAware.java b/storm-server/src/main/java/org/apache/storm/ILocalClusterTrackedTopologyAware.java index 5e94a094bf0..ba36362af15 100644 --- a/storm-server/src/main/java/org/apache/storm/ILocalClusterTrackedTopologyAware.java +++ b/storm-server/src/main/java/org/apache/storm/ILocalClusterTrackedTopologyAware.java @@ -24,7 +24,8 @@ import org.apache.storm.thrift.TException; /** - * This is here mostly for backwards compatibility. Please see {@link org.apache.storm.LocalCluster} for more details on testing a Storm + * This is here mostly for backwards compatibility. Please see {@link org.apache.storm.LocalCluster} + * for more details on testing a Storm * Topology. */ public interface ILocalClusterTrackedTopologyAware extends ILocalCluster { @@ -39,7 +40,8 @@ public interface ILocalClusterTrackedTopologyAware extends ILocalCluster { * * @throws TException on any error from nimbus */ - ILocalTopology submitTopology(String topologyName, Map conf, TrackedTopology topology) throws TException; + ILocalTopology submitTopology(String topologyName, Map conf, + TrackedTopology topology) throws TException; /** * Submit a tracked topology to be run in local mode. @@ -52,7 +54,8 @@ public interface ILocalClusterTrackedTopologyAware extends ILocalCluster { * * @throws TException on any error from nimbus */ - ILocalTopology submitTopologyWithOpts(String topologyName, Map conf, TrackedTopology topology, + ILocalTopology submitTopologyWithOpts(String topologyName, Map conf, + TrackedTopology topology, SubmitOptions submitOpts) throws TException; } diff --git a/storm-server/src/main/java/org/apache/storm/LocalCluster.java b/storm-server/src/main/java/org/apache/storm/LocalCluster.java index 71ee890c6cc..530ea9b7fcf 100644 --- a/storm-server/src/main/java/org/apache/storm/LocalCluster.java +++ b/storm-server/src/main/java/org/apache/storm/LocalCluster.java @@ -45,8 +45,8 @@ import org.apache.storm.daemon.DaemonCommon; import org.apache.storm.daemon.Shutdownable; import org.apache.storm.daemon.StormCommon; -import org.apache.storm.daemon.nimbus.Nimbus; import org.apache.storm.daemon.nimbus.Nimbus.StandaloneINimbus; +import org.apache.storm.daemon.nimbus.Nimbus; import org.apache.storm.daemon.nimbus.TopoCache; import org.apache.storm.daemon.supervisor.ReadClusterState; import org.apache.storm.daemon.supervisor.StandaloneSupervisor; @@ -108,8 +108,8 @@ import org.apache.storm.utils.ObjectReader; import org.apache.storm.utils.RegisteredGlobalState; import org.apache.storm.utils.StormCommonInstaller; -import org.apache.storm.utils.Time; import org.apache.storm.utils.Time.SimulatedTime; +import org.apache.storm.utils.Time; import org.apache.storm.utils.Utils; import org.apache.storm.utils.WrappedAuthorizationException; import org.apache.storm.utils.WrappedKeyNotFoundException; @@ -117,12 +117,15 @@ import org.slf4j.LoggerFactory; /** - * A stand alone storm cluster that runs inside a single process. It is intended to be used for testing. Both internal testing for Apache + * A stand alone storm cluster that runs inside a single process. It is intended to be used for + * testing. Both internal testing for Apache * Storm itself and for people building storm topologies. - *

      - * LocalCluster is an AutoCloseable so if you are using it in tests you can use a try block to be sure it is shut down. + * + *

      LocalCluster is an AutoCloseable so if you are using it in tests you can use a try block to be + * sure it is shut down. *

      - * try (LocalCluster cluster = new LocalCluster()) { // Do some tests } // The cluster has been shut down. + * try (LocalCluster cluster = new LocalCluster()) { // Do some tests } // The cluster has been shut + * down. */ public class LocalCluster implements ILocalClusterTrackedTopologyAware, Iface { public static final KillOptions KILL_NOW = new KillOptions(); @@ -134,7 +137,7 @@ public class LocalCluster implements ILocalClusterTrackedTopologyAware, Iface { } private final Nimbus nimbus; - //This is very private and does not need to be exposed + // This is very private and does not need to be exposed private int portCounter; private final Map daemonConf; private final List supervisors; @@ -188,7 +191,8 @@ private LocalCluster(Builder builder) throws Exception { metrics.put("spout-emitted", new AtomicInteger(0)); metrics.put("transferred", new AtomicInteger(0)); metrics.put("processed", new AtomicInteger(0)); - this.commonInstaller = new StormCommonInstaller(new TrackedStormCommon(this.trackId)); + this.commonInstaller = + new StormCommonInstaller(new TrackedStormCommon(this.trackId)); LOG.warn("Adding tracked metrics for ID {}", this.trackId); RegisteredGlobalState.setState(this.trackId, metrics); LocalExecutor.setTrackId(this.trackId); @@ -203,7 +207,8 @@ private LocalCluster(Builder builder) throws Exception { stormHomeBackup = System.getProperty(ConfigUtils.STORM_HOME); TmpPath stormHome = new TmpPath(); if (!stormHome.getFile().mkdirs()) { - throw new IllegalStateException("Failed to create storm.home directory " + stormHome.getPath()); + throw new IllegalStateException("Failed to create storm.home directory " + stormHome + .getPath()); } this.tmpDirs.add(stormHome); System.setProperty(ConfigUtils.STORM_HOME, stormHome.getPath()); @@ -235,12 +240,13 @@ private LocalCluster(Builder builder) throws Exception { this.clusterState = builder.clusterState; } if (!Time.isSimulating()) { - //Ensure Nimbus assigns topologies as quickly as possible + // Ensure Nimbus assigns topologies as quickly as possible conf.put(DaemonConfig.NIMBUS_MONITOR_FREQ_SECS, 1); } - //Set it for nimbus only + // Set it for nimbus only conf.put(Config.STORM_LOCAL_DIR, nimbusTmp.getPath()); - Nimbus nimbus = new Nimbus(conf, builder.inimbus == null ? new StandaloneINimbus() : builder.inimbus, + Nimbus nimbus = new Nimbus(conf, builder.inimbus == null + ? new StandaloneINimbus() : builder.inimbus, this.getClusterState(), null, builder.store, builder.topoCache, builder.leaderElector, builder.groupMapper, metricRegistry); if (builder.nimbusWrapper != null) { @@ -249,7 +255,7 @@ private LocalCluster(Builder builder) throws Exception { this.nimbus = nimbus; this.nimbus.launchServer(); if (!this.nimbus.awaitLeadership(Testing.TEST_TIMEOUT_MS, TimeUnit.MILLISECONDS)) { - //Ensure Nimbus has leadership, otherwise topology submission will fail. + // Ensure Nimbus has leadership, otherwise topology submission will fail. throw new RuntimeException("LocalCluster Nimbus failed to gain leadership."); } IContext context = null; @@ -258,26 +264,29 @@ private LocalCluster(Builder builder) throws Exception { context.prepare(this.daemonConf); } this.sharedContext = context; - this.thriftServer = builder.nimbusDaemon ? startNimbusDaemon(this.daemonConf, this.nimbus) : null; + this.thriftServer = builder.nimbusDaemon ? startNimbusDaemon(this.daemonConf, + this.nimbus) : null; for (int i = 0; i < builder.supervisors; i++) { addSupervisor(builder.portsPerSupervisor, null, null); } - //Wait for a leader to be elected (or topology submission can be rejected) + // Wait for a leader to be elected (or topology submission can be rejected) try { long timeoutAfter = System.currentTimeMillis() + 10_000; while (!hasLeader()) { if (timeoutAfter > System.currentTimeMillis()) { - throw new IllegalStateException("Timed out waiting for nimbus to become the leader"); + throw new IllegalStateException("Timed out waiting for nimbus to become " + + "the leader"); } Thread.sleep(1); } } catch (Exception e) { - //Ignore any exceptions we might be doing a test for authentication + // Ignore any exceptions we might be doing a test for authentication } if (thriftServer == null) { - //We don't want to override the client if there is a thrift server up and running, or we would not test any + // We don't want to override the client if there is a thrift server up and running, + // or we would not test any // Of the actual thrift code this.nimbusOverride = new NimbusClient.LocalOverride(this); } else { @@ -294,7 +303,8 @@ private LocalCluster(Builder builder) throws Exception { } private static ThriftServer startNimbusDaemon(Map conf, Nimbus nimbus) { - ThriftServer ret = new ThriftServer(conf, new Processor<>(nimbus), ThriftConnectionType.NIMBUS); + ThriftServer ret = new ThriftServer(conf, new Processor<>(nimbus), + ThriftConnectionType.NIMBUS); LOG.info("Starting Nimbus server..."); new Thread(() -> ret.serve()).start(); return ret; @@ -311,7 +321,8 @@ private static boolean areAllWorkersWaiting() { } /** - * Run c with a local mode cluster overriding the NimbusClient and DRPCClient calls. NOTE local mode override happens by default now + * Run c with a local mode cluster overriding the NimbusClient and DRPCClient calls. NOTE local + * mode override happens by default now * unless netty is turned on for the local cluster. * * @param c the callable to run in this mode @@ -325,7 +336,8 @@ public static T withLocalModeOverride(Callable c, long ttlSec) throws Exc } /** - * Run c with a local mode cluster overriding the NimbusClient and DRPCClient calls. NOTE local mode override happens by default now + * Run c with a local mode cluster overriding the NimbusClient and DRPCClient calls. NOTE local + * mode override happens by default now * unless netty is turned on for the local cluster. * * @param c the callable to run in this mode @@ -336,7 +348,8 @@ public static T withLocalModeOverride(Callable c, long ttlSec) throws Exc * @throws Exception on any Exception. */ @SuppressWarnings("checkstyle:VariableDeclarationUsageDistance") - public static T withLocalModeOverride(Callable c, long ttlSec, Map daemonConf) throws Exception { + public static T withLocalModeOverride(Callable c, long ttlSec, Map daemonConf) throws Exception { LOG.info("\n\n\t\tSTARTING LOCAL MODE CLUSTER\n\n"); Builder builder = new Builder(); if (daemonConf != null) { @@ -357,6 +370,7 @@ public static T withLocalModeOverride(Callable c, long ttlSec, Map getDaemonConf() { @@ -448,14 +465,17 @@ public Map getDaemonConf() { } @Override - public LocalTopology submitTopology(String topologyName, Map conf, StormTopology topology) + public LocalTopology submitTopology(String topologyName, Map conf, + StormTopology topology) throws TException { if (!Utils.isValidConf(conf)) { throw new IllegalArgumentException("Topology conf is not json-serializable"); } - getNimbus().submitTopology(topologyName, null, JSONValue.toJSONString(conf), Utils.addVersions(topology)); + getNimbus().submitTopology(topologyName, null, JSONValue.toJSONString(conf), Utils + .addVersions(topology)); - ISubmitterHook hook = (ISubmitterHook) Utils.getConfiguredClass(conf, Config.STORM_TOPOLOGY_SUBMISSION_NOTIFIER_PLUGIN); + ISubmitterHook hook = (ISubmitterHook) Utils.getConfiguredClass(conf, + Config.STORM_TOPOLOGY_SUBMISSION_NOTIFIER_PLUGIN); if (hook != null) { TopologyInfo topologyInfo = Utils.getTopologyInfo(topologyName, null, conf); try { @@ -468,13 +488,15 @@ public LocalTopology submitTopology(String topologyName, Map con } @Override - public LocalTopology submitTopology(String topologyName, Map conf, TrackedTopology topology) + public LocalTopology submitTopology(String topologyName, Map conf, + TrackedTopology topology) throws TException { return submitTopology(topologyName, conf, topology.getTopology()); } @Override - public void submitTopology(String name, String uploadedJarLocation, String jsonConf, StormTopology topology) + public void submitTopology(String name, String uploadedJarLocation, String jsonConf, + StormTopology topology) throws AlreadyAliveException, InvalidTopologyException, AuthorizationException, TException { try { @SuppressWarnings("unchecked") @@ -486,25 +508,29 @@ public void submitTopology(String name, String uploadedJarLocation, String jsonC } @Override - public LocalTopology submitTopologyWithOpts(String topologyName, Map conf, StormTopology topology, + public LocalTopology submitTopologyWithOpts(String topologyName, Map conf, + StormTopology topology, SubmitOptions submitOpts) throws TException { if (!Utils.isValidConf(conf)) { throw new IllegalArgumentException("Topology conf is not json-serializable"); } - getNimbus().submitTopologyWithOpts(topologyName, null, JSONValue.toJSONString(conf), Utils.addVersions(topology), submitOpts); + getNimbus().submitTopologyWithOpts(topologyName, null, JSONValue.toJSONString(conf), Utils + .addVersions(topology), submitOpts); return new LocalTopology(topologyName, topology); } @Override - public LocalTopology submitTopologyWithOpts(String topologyName, Map conf, TrackedTopology topology, + public LocalTopology submitTopologyWithOpts(String topologyName, Map conf, + TrackedTopology topology, SubmitOptions submitOpts) throws TException { return submitTopologyWithOpts(topologyName, conf, topology.getTopology(), submitOpts); } @Override - public void submitTopologyWithOpts(String name, String uploadedJarLocation, String jsonConf, StormTopology topology, + public void submitTopologyWithOpts(String name, String uploadedJarLocation, String jsonConf, + StormTopology topology, SubmitOptions options) throws AlreadyAliveException, InvalidTopologyException, AuthorizationException, TException { try { @@ -531,19 +557,16 @@ public void killTopologyWithOpts(String name, KillOptions options) throws TExcep getNimbus().killTopologyWithOpts(name, options); } - @Override public void activate(String topologyName) throws TException { getNimbus().activate(topologyName); } - @Override public void deactivate(String topologyName) throws TException { getNimbus().deactivate(topologyName); } - @Override public void rebalance(String name, RebalanceOptions options) throws TException { getNimbus().rebalance(name, options); @@ -599,12 +622,14 @@ public TopologyInfo getTopologyInfoByName(String name) throws TException { } @Override - public TopologyInfo getTopologyInfoWithOpts(String id, GetInfoOptions options) throws TException { + public TopologyInfo getTopologyInfoWithOpts(String id, + GetInfoOptions options) throws TException { return nimbus.getTopologyInfoWithOpts(id, options); } @Override - public TopologyInfo getTopologyInfoByNameWithOpts(String name, GetInfoOptions options) throws TException { + public TopologyInfo getTopologyInfoByNameWithOpts(String name, + GetInfoOptions options) throws TException { return nimbus.getTopologyInfoByNameWithOpts(name, options); } @@ -703,7 +728,7 @@ public synchronized void killSupervisor(String id) { if (id.equals(s.getId())) { it.remove(); s.close(); - //tmpDir will be handled separately + // tmpDir will be handled separately return; } } @@ -739,10 +764,12 @@ public Supervisor addSupervisor(Number ports, String id) throws Exception { * Add another supervisor to the topology. This is intended mostly for internal testing. * * @param ports the number of ports/slots the supervisor should have - * @param conf any config values that should be added/over written in the daemon conf of the cluster. + * @param conf any config values that should be added/over written in the daemon conf of the + * cluster. * @param id the id of the new supervisor, so you can find it later. */ - public synchronized Supervisor addSupervisor(Number ports, Map conf, String id) throws Exception { + public synchronized Supervisor addSupervisor(Number ports, Map conf, + String id) throws Exception { if (ports == null) { ports = 2; } @@ -761,7 +788,8 @@ public synchronized Supervisor addSupervisor(Number ports, Map c superConf.put(Config.STORM_LOCAL_DIR, tmpDir.getPath()); superConf.put(DaemonConfig.SUPERVISOR_SLOTS_PORTS, portNumbers); if (!Time.isSimulating()) { - //Monitor for assignment changes as often as possible, so e.g. shutdown happens as fast as possible. + // Monitor for assignment changes as often as possible, so e.g. shutdown happens as fast + // as possible. superConf.put(DaemonConfig.SUPERVISOR_MONITOR_FREQUENCY_SECS, 1); } @@ -793,7 +821,8 @@ private boolean areAllSupervisorsWaiting() { } /** - * Wait for the cluster to be idle. This is intended to be used with Simulated time and is for internal testing. + * Wait for the cluster to be idle. This is intended to be used with Simulated time and is for + * internal testing. * Note that this does not wait for spout or bolt executors to be idle. * * @throws InterruptedException if interrupted while waiting. @@ -804,7 +833,8 @@ public void waitForIdle() throws InterruptedException { } /** - * Wait for the cluster to be idle. This is intended to be used with Simulated time and is for internal testing. + * Wait for the cluster to be idle. This is intended to be used with Simulated time and is for + * internal testing. * Note that this does not wait for spout or bolt executors to be idle. * * @param timeoutMs the number of ms to wait before throwing an error. @@ -813,7 +843,7 @@ public void waitForIdle() throws InterruptedException { */ public void waitForIdle(long timeoutMs) throws InterruptedException { Random rand = ThreadLocalRandom.current(); - //wait until all workers, supervisors, and nimbus is waiting + // wait until all workers, supervisors, and nimbus is waiting final long endTime = System.currentTimeMillis() + timeoutMs; while (!(nimbus.isWaiting() && areAllSupervisorsWaiting() @@ -852,7 +882,7 @@ public String getTrackedId() { return trackId; } - //Nimbus Compatibility + // Nimbus Compatibility @Override public void setLogConfig(String name, LogConfig config) throws TException { @@ -880,7 +910,8 @@ public void setWorkerProfiler(String id, ProfileRequest profileRequest) throws T } @Override - public List getComponentPendingProfileActions(String id, String componentId, ProfileAction action) + public List getComponentPendingProfileActions(String id, String componentId, + ProfileAction action) throws TException { // TODO: Auto-generated method stub throw new RuntimeException("NOT IMPLEMENTED YET"); @@ -898,7 +929,8 @@ public String beginUpdateBlob(String key) throws AuthorizationException, KeyNotF } @Override - public void uploadBlobChunk(String session, ByteBuffer chunk) throws AuthorizationException, TException { + public void uploadBlobChunk(String session, + ByteBuffer chunk) throws AuthorizationException, TException { throw new RuntimeException("BLOBS NOT SUPPORTED IN LOCAL MODE"); } @@ -941,7 +973,7 @@ public void deleteBlob(String key) throws AuthorizationException, KeyNotFoundExc @Override public ListBlobsResult listBlobs(String session) throws TException { - //Blobs are not supported in local mode. Return nothing + // Blobs are not supported in local mode. Return nothing ListBlobsResult ret = new ListBlobsResult(); ret.set_keys(new ArrayList<>()); return ret; @@ -966,18 +998,19 @@ public void createStateInZookeeper(String key) throws TException { @Override public String beginFileUpload() throws AuthorizationException, TException { - //Just ignore these for now. We are going to throw it away anyways + // Just ignore these for now. We are going to throw it away anyways return Utils.uuid(); } @Override - public void uploadChunk(String location, ByteBuffer chunk) throws AuthorizationException, TException { - //Just throw it away in local mode + public void uploadChunk(String location, + ByteBuffer chunk) throws AuthorizationException, TException { + // Just throw it away in local mode } @Override public void finishFileUpload(String location) throws AuthorizationException, TException { - //Just throw it away in local mode + // Just throw it away in local mode } @Override @@ -1016,7 +1049,8 @@ public SupervisorPageInfo getSupervisorPageInfo(String id, String host, boolean } @Override - public ComponentPageInfo getComponentPageInfo(String topologyId, String componentId, String window, + public ComponentPageInfo getComponentPageInfo(String topologyId, String componentId, + String window, boolean isIncludeSys) throws NotAliveException, AuthorizationException, TException { // TODO: Auto-generated method stub throw new RuntimeException("NOT IMPLEMENTED YET"); @@ -1153,14 +1187,16 @@ public Builder withSupervisorSlotPortMin(Number minPort) { } /** - * Have the local nimbus actually launch a thrift server. This is intended to be used mostly for internal storm testing. + * Have the local nimbus actually launch a thrift server. This is intended to be used mostly + * for internal storm testing. */ public Builder withNimbusDaemon() { return withNimbusDaemon(true); } /** - * If nimbusDaemon is true the local nimbus will launch a thrift server. This is intended to be used mostly for internal storm + * If nimbusDaemon is true the local nimbus will launch a thrift server. This is intended to + * be used mostly for internal storm * testing. */ public Builder withNimbusDaemon(Boolean nimbusDaemon) { @@ -1173,8 +1209,10 @@ public Builder withNimbusDaemon(Boolean nimbusDaemon) { } /** - * Turn on simulated time in the cluster. This allows someone to simulate long periods of time for timeouts etc when testing - * nimbus/supervisors themselves. NOTE: that this only works for code that uses the {@link org.apache.storm.utils.Time} class for + * Turn on simulated time in the cluster. This allows someone to simulate long periods of + * time for timeouts etc when testing + * nimbus/supervisors themselves. NOTE: that this only works for code that uses the {@link + * org.apache.storm.utils.Time} class for * time management so it will not work in all cases. */ public Builder withSimulatedTime() { @@ -1182,8 +1220,10 @@ public Builder withSimulatedTime() { } /** - * Turn on simulated time in the cluster. This allows someone to simulate long periods of time for timeouts etc when testing - * nimbus/supervisors themselves. NOTE: that this only works for code that uses the {@link org.apache.storm.utils.Time} class for + * Turn on simulated time in the cluster. This allows someone to simulate long periods of + * time for timeouts etc when testing + * nimbus/supervisors themselves. NOTE: that this only works for code that uses the {@link + * org.apache.storm.utils.Time} class for * time management so it will not work in all cases. */ public Builder withSimulatedTime(boolean simulateTime) { @@ -1192,7 +1232,8 @@ public Builder withSimulatedTime(boolean simulateTime) { } /** - * Before nimbus is created/used call nimbusWrapper on it first and use the result instead. This is intended for internal testing + * Before nimbus is created/used call nimbusWrapper on it first and use the result instead. + * This is intended for internal testing * only, and it here to allow a mocking framework to spy on the nimbus class. */ public Builder withNimbusWrapper(UnaryOperator nimbusWrapper) { @@ -1201,7 +1242,8 @@ public Builder withNimbusWrapper(UnaryOperator nimbusWrapper) { } /** - * Use the following blobstore instead of the one in the config. This is intended mostly for internal testing with Mocks. + * Use the following blobstore instead of the one in the config. This is intended mostly for + * internal testing with Mocks. */ public Builder withBlobStore(BlobStore store) { this.store = store; @@ -1209,7 +1251,8 @@ public Builder withBlobStore(BlobStore store) { } /** - * Use the following topo cache instead of creating out own. This is intended mostly for internal testing with Mocks. + * Use the following topo cache instead of creating out own. This is intended mostly for + * internal testing with Mocks. */ public Builder withTopoCache(TopoCache topoCache) { this.topoCache = topoCache; @@ -1217,7 +1260,8 @@ public Builder withTopoCache(TopoCache topoCache) { } /** - * Use the following clusterState instead of the one in the config. This is intended mostly for internal testing with Mocks. + * Use the following clusterState instead of the one in the config. This is intended mostly + * for internal testing with Mocks. */ public Builder withClusterState(IStormClusterState clusterState) { this.clusterState = clusterState; @@ -1225,7 +1269,8 @@ public Builder withClusterState(IStormClusterState clusterState) { } /** - * Use the following leaderElector instead of the one in the config. This is intended mostly for internal testing with Mocks. + * Use the following leaderElector instead of the one in the config. This is intended mostly + * for internal testing with Mocks. */ public Builder withLeaderElector(ILeaderElector leaderElector) { this.leaderElector = leaderElector; @@ -1233,7 +1278,8 @@ public Builder withLeaderElector(ILeaderElector leaderElector) { } /** - * A tracked cluster can run tracked topologies. See {@link org.apache.storm.testing.TrackedTopology} for more information on + * A tracked cluster can run tracked topologies. See {@link + * org.apache.storm.testing.TrackedTopology} for more information on * tracked topologies. * * @param trackId an arbitrary unique id that is used to keep track of tracked topologies @@ -1244,7 +1290,8 @@ public Builder withTracked(String trackId) { } /** - * A tracked cluster can run tracked topologies. See {@link org.apache.storm.testing.TrackedTopology} for more information on + * A tracked cluster can run tracked topologies. See {@link + * org.apache.storm.testing.TrackedTopology} for more information on * tracked topologies. */ public Builder withTracked() { @@ -1257,7 +1304,8 @@ public Builder withTracked() { * * @return the LocalCluster * - * @throws Exception on any one of many different errors. This is intended for testing so yes it is ugly and throws Exception... + * @throws Exception on any one of many different errors. This is intended for testing so + * yes it is ugly and throws Exception... */ public LocalCluster build() throws Exception { return new LocalCluster(this); @@ -1279,8 +1327,10 @@ public IBolt makeAckerBoltImpl() { } /** - * When running a topology locally, for tests etc. It is helpful to be sure that the topology is dead before the test exits. This is - * an AutoCloseable topology that not only gives you access to the compiled StormTopology but also will kill the topology when it + * When running a topology locally, for tests etc. It is helpful to be sure that the topology is + * dead before the test exits. This is + * an AutoCloseable topology that not only gives you access to the compiled StormTopology but + * also will kill the topology when it * closes. * * diff --git a/storm-server/src/main/java/org/apache/storm/LocalDRPC.java b/storm-server/src/main/java/org/apache/storm/LocalDRPC.java index b11b2db2a2b..40342a10c22 100644 --- a/storm-server/src/main/java/org/apache/storm/LocalDRPC.java +++ b/storm-server/src/main/java/org/apache/storm/LocalDRPC.java @@ -49,6 +49,7 @@ public LocalDRPC() { /** * Creates a LocalDRPC with the specified metrics registry. + * * @param metricsRegistry The registry */ public LocalDRPC(StormMetricsRegistry metricsRegistry) { @@ -68,7 +69,8 @@ public void result(String id, String result) throws AuthorizationException, TExc } @Override - public String execute(String functionName, String funcArgs) throws DRPCExecutionException, AuthorizationException, TException { + public String execute(String functionName, + String funcArgs) throws DRPCExecutionException, AuthorizationException, TException { return drpc.executeBlocking(functionName, funcArgs); } @@ -77,9 +79,9 @@ public void failRequest(String id) throws AuthorizationException, TException { drpc.failRequest(id, null); } - @Override - public void failRequestV2(String id, DRPCExecutionException e) throws AuthorizationException, TException { + public void failRequestV2(String id, + DRPCExecutionException e) throws AuthorizationException, TException { drpc.failRequest(id, e); } diff --git a/storm-server/src/main/java/org/apache/storm/ProcessSimulator.java b/storm-server/src/main/java/org/apache/storm/ProcessSimulator.java index a647428e873..ac9c5a3dfd8 100644 --- a/storm-server/src/main/java/org/apache/storm/ProcessSimulator.java +++ b/storm-server/src/main/java/org/apache/storm/ProcessSimulator.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -26,7 +32,8 @@ * in place of actual processes (in cluster mode). */ public class ProcessSimulator { - protected static ConcurrentHashMap processMap = new ConcurrentHashMap(); + protected static ConcurrentHashMap processMap = + new ConcurrentHashMap(); private static Logger LOG = LoggerFactory.getLogger(ProcessSimulator.class); private static Object lock = new Object(); @@ -75,7 +82,7 @@ public static void killAllProcesses() { } else if (e instanceof RuntimeException) { throw e; } else { - //TODO: once everything is in java this should not be possible any more + // TODO: once everything is in java this should not be possible any more throw new RuntimeException(e); } } diff --git a/storm-server/src/main/java/org/apache/storm/ServerConstants.java b/storm-server/src/main/java/org/apache/storm/ServerConstants.java index 0a7f6a488e1..3947aef3492 100644 --- a/storm-server/src/main/java/org/apache/storm/ServerConstants.java +++ b/storm-server/src/main/java/org/apache/storm/ServerConstants.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-server/src/main/java/org/apache/storm/Testing.java b/storm-server/src/main/java/org/apache/storm/Testing.java index 45e585c440f..a1914efe6fe 100644 --- a/storm-server/src/main/java/org/apache/storm/Testing.java +++ b/storm-server/src/main/java/org/apache/storm/Testing.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -16,8 +22,8 @@ import java.util.Collection; import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Map.Entry; +import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -49,8 +55,8 @@ import org.apache.storm.tuple.TupleImpl; import org.apache.storm.utils.ConfigUtils; import org.apache.storm.utils.RegisteredGlobalState; -import org.apache.storm.utils.Time; import org.apache.storm.utils.Time.SimulatedTime; +import org.apache.storm.utils.Time; import org.apache.storm.utils.Utils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -72,7 +78,7 @@ public class Testing { try { timeout = Integer.parseInt(System.getenv("STORM_TEST_TIMEOUT_MS")); } catch (Exception e) { - //Ignored, will go with default timeout + // Ignored, will go with default timeout } TEST_TIMEOUT_MS = timeout; } @@ -80,6 +86,7 @@ public class Testing { /** * Continue to execute body repeatedly until condition is true or TEST_TIMEOUT_MS has * passed. + * * @param condition what we are waiting for * @param body what to run in the loop * @throws AssertionError if the loop timed out. @@ -91,6 +98,7 @@ public static void whileTimeout(Condition condition, Runnable body) { /** * Continue to execute body repeatedly until condition is true or TEST_TIMEOUT_MS has * passed. + * * @param timeoutMs the number of ms to wait before timing out. * @param condition what we are waiting for * @param body what to run in the loop @@ -103,7 +111,8 @@ public static void whileTimeout(long timeoutMs, Condition condition, Runnable bo while (condition.exec()) { count++; if (System.currentTimeMillis() > endTime) { - LOG.info("Condition {} not met in {} ms after calling {} times", condition, timeoutMs, count); + LOG.info("Condition {} not met in {} ms after calling {} times", condition, + timeoutMs, count); LOG.info(Utils.threadDump()); throw new AssertionError("Test timed out (" + timeoutMs + "ms) " + condition); } @@ -122,12 +131,12 @@ public static boolean isEvery(Collection data, Predicate pred) { /** * Run with simulated time. * + * @param code what to run * @deprecated use ``` * try (Time.SimulatedTime time = new Time.SimulatedTime()) { * ... * } * ``` - * @param code what to run */ @Deprecated public static void withSimulatedTime(Runnable code) { @@ -144,7 +153,8 @@ private static LocalCluster cluster(MkClusterParam param) throws Exception { return cluster(param, null, false); } - private static LocalCluster cluster(MkClusterParam param, String id, boolean simulated) throws Exception { + private static LocalCluster cluster(MkClusterParam param, String id, + boolean simulated) throws Exception { Integer supervisors = param.getSupervisors(); if (supervisors == null) { supervisors = 2; @@ -170,12 +180,12 @@ private static LocalCluster cluster(MkClusterParam param, String id, boolean sim /** * Run with a local cluster. * + * @param code what to run * @deprecated use ``` * try (LocalCluster cluster = new LocalCluster()) { * ... * } * ``` - * @param code what to run */ @Deprecated public static void withLocalCluster(TestJob code) { @@ -185,13 +195,13 @@ public static void withLocalCluster(TestJob code) { /** * Run with a local cluster. * + * @param param configs to set in the cluster + * @param code what to run * @deprecated use ``` * try (LocalCluster cluster = new LocalCluster.Builder()....build()) { * ... * } * ``` - * @param param configs to set in the cluster - * @param code what to run */ @Deprecated public static void withLocalCluster(MkClusterParam param, TestJob code) { @@ -205,12 +215,12 @@ public static void withLocalCluster(MkClusterParam param, TestJob code) { /** * Run with a local cluster. * + * @param clusterConf some configs to set in the cluster * @deprecated use ``` * try (LocalCluster cluster = new LocalCluster.Builder()....build()) { * ... * } * ``` - * @param clusterConf some configs to set in the cluster */ @Deprecated public static ILocalCluster getLocalCluster(Map clusterConf) { @@ -241,12 +251,12 @@ public static ILocalCluster getLocalCluster(Map clusterConf) { /** * Run with a local cluster. * + * @param code what to run * @deprecated use ``` * try (LocalCluster cluster = new LocalCluster.Builder().withSimulatedTime().build()) { * ... * } * ``` - * @param code what to run */ @Deprecated public static void withSimulatedTimeLocalCluster(TestJob code) { @@ -256,13 +266,13 @@ public static void withSimulatedTimeLocalCluster(TestJob code) { /** * Run with a local cluster. * + * @param param configs to set in the cluster + * @param code what to run * @deprecated use ``` * try (LocalCluster cluster = new LocalCluster.Builder().withSimulatedTime()....build()) { * ... * } * ``` - * @param param configs to set in the cluster - * @param code what to run */ @Deprecated public static void withSimulatedTimeLocalCluster(MkClusterParam param, TestJob code) { @@ -276,12 +286,12 @@ public static void withSimulatedTimeLocalCluster(MkClusterParam param, TestJob c /** * Run with a local cluster. * + * @param code what to run * @deprecated use ``` * try (LocalCluster cluster = new LocalCluster.Builder().withTracked().build()) { * ... * } * ``` - * @param code what to run */ @Deprecated public static void withTrackedCluster(TestJob code) { @@ -291,13 +301,13 @@ public static void withTrackedCluster(TestJob code) { /** * Run with a local tracked cluster. * + * @param param configs to set in the cluster + * @param code what to run * @deprecated use ``` * try (LocalCluster cluster = new LocalCluster.Builder().withTracked()....build()) { * ... * } * ``` - * @param param configs to set in the cluster - * @param code what to run */ @Deprecated public static void withTrackedCluster(MkClusterParam param, TestJob code) { @@ -320,51 +330,61 @@ public static void withTrackedCluster(MkClusterParam param, TestJob code) { @Deprecated public static int globalAmt(String id, String key) { LOG.warn("Reading tracked metrics for ID {}", id); - return ((ConcurrentHashMap) RegisteredGlobalState.getState(id)).get(key).get(); + return ((ConcurrentHashMap) RegisteredGlobalState.getState(id)) + .get(key).get(); } /** * Track and capture a topology. * This is intended mostly for internal testing. */ - public static CapturedTopology trackAndCaptureTopology(ILocalCluster cluster, StormTopology topology) { + public static CapturedTopology trackAndCaptureTopology(ILocalCluster cluster, + StormTopology topology) { CapturedTopology captured = captureTopology(topology); - return new CapturedTopology<>(new TrackedTopology(captured.topology, cluster), captured.capturer); + return new CapturedTopology<>(new TrackedTopology(captured.topology, cluster), + captured.capturer); } /** * Rewrites a topology so that all the tuples flowing through it are captured. + * * @param topology the topology to rewrite * @return the modified topology and a new Bolt that can retrieve the * captured tuples. */ public static CapturedTopology captureTopology(StormTopology topology) { - topology = topology.deepCopy(); //Don't modify the original + topology = topology.deepCopy(); // Don't modify the original TupleCaptureBolt capturer = new TupleCaptureBolt(); Map captureBoltInputs = new HashMap<>(); for (Map.Entry spoutEntry : topology.get_spouts().entrySet()) { String id = spoutEntry.getKey(); - for (Entry streamEntry : spoutEntry.getValue().get_common().get_streams().entrySet()) { + for (Entry streamEntry : spoutEntry.getValue().get_common() + .get_streams().entrySet()) { String stream = streamEntry.getKey(); StreamInfo info = streamEntry.getValue(); if (info.is_direct()) { - captureBoltInputs.put(new GlobalStreamId(id, stream), Thrift.prepareDirectGrouping()); + captureBoltInputs.put(new GlobalStreamId(id, stream), Thrift + .prepareDirectGrouping()); } else { - captureBoltInputs.put(new GlobalStreamId(id, stream), Thrift.prepareGlobalGrouping()); + captureBoltInputs.put(new GlobalStreamId(id, stream), Thrift + .prepareGlobalGrouping()); } } } for (Entry boltEntry : topology.get_bolts().entrySet()) { String id = boltEntry.getKey(); - for (Entry streamEntry : boltEntry.getValue().get_common().get_streams().entrySet()) { + for (Entry streamEntry : boltEntry.getValue().get_common() + .get_streams().entrySet()) { String stream = streamEntry.getKey(); StreamInfo info = streamEntry.getValue(); if (info.is_direct()) { - captureBoltInputs.put(new GlobalStreamId(id, stream), Thrift.prepareDirectGrouping()); + captureBoltInputs.put(new GlobalStreamId(id, stream), Thrift + .prepareDirectGrouping()); } else { - captureBoltInputs.put(new GlobalStreamId(id, stream), Thrift.prepareGlobalGrouping()); + captureBoltInputs.put(new GlobalStreamId(id, stream), Thrift + .prepareGlobalGrouping()); } } } @@ -374,28 +394,35 @@ public static CapturedTopology captureTopology(StormTopology topo } /** - * Run a topology to completion capturing all of the messages that are emitted. This only works when all of the spouts are + * Run a topology to completion capturing all of the messages that are emitted. This only works + * when all of the spouts are * instances of {@link org.apache.storm.testing.CompletableSpout}. + * * @param cluster the cluster to submit the topology to * @param topology the topology itself * @return a map of the component to the list of tuples it emitted * @throws TException on any error from nimbus */ - public static Map> completeTopology(ILocalCluster cluster, StormTopology topology) throws InterruptedException, + public static Map> completeTopology(ILocalCluster cluster, + StormTopology topology) throws InterruptedException, TException { return completeTopology(cluster, topology, new CompleteTopologyParam()); } /** - * Run a topology to completion capturing all of the messages that are emitted. This only works when all of the spouts are - * instances of {@link org.apache.storm.testing.CompletableSpout} or are overwritten by MockedSources in param + * Run a topology to completion capturing all of the messages that are emitted. This only works + * when all of the spouts are + * instances of {@link org.apache.storm.testing.CompletableSpout} or are overwritten by + * MockedSources in param + * * @param cluster the cluster to submit the topology to * @param topology the topology itself * @param param parameters to describe how to complete a topology * @return a map of the component to the list of tuples it emitted * @throws TException on any error from nimbus. */ - public static Map> completeTopology(ILocalCluster cluster, StormTopology topology, + public static Map> completeTopology(ILocalCluster cluster, + StormTopology topology, CompleteTopologyParam param) throws TException, InterruptedException { Map> ret = null; CapturedTopology capTopo = captureTopology(topology); @@ -410,7 +437,8 @@ public static Map> completeTopology(ILocalCluster clust if (ms != null) { for (Entry> mocked : ms.getData().entrySet()) { FixedTupleSpout newSpout = new FixedTupleSpout(mocked.getValue()); - spouts.get(mocked.getKey()).set_spout_object(Thrift.serializeComponentObject(newSpout)); + spouts.get(mocked.getKey()).set_spout_object(Thrift + .serializeComponentObject(newSpout)); } } List spoutObjects = spouts.values() @@ -421,7 +449,8 @@ public static Map> completeTopology(ILocalCluster clust for (Object o : spoutObjects) { if (!(o instanceof CompletableSpout)) { throw new RuntimeException( - "Cannot complete topology unless every spout is a CompletableSpout (or mocked to be); failed by " + o); + "Cannot complete topology unless every spout is a CompletableSpout (or mocked " + + "to be); failed by " + o); } } @@ -437,7 +466,7 @@ public static Map> completeTopology(ILocalCluster clust IStormClusterState state = cluster.getClusterState(); String topoId = state.getTopoId(topoName).get(); - //Give the topology time to come up without using it to wait for the spouts to complete + // Give the topology time to come up without using it to wait for the spouts to complete simulateWait(cluster); Integer timeoutMs = param.getTimeoutMs(); if (timeoutMs == null) { @@ -480,7 +509,8 @@ public static Map> completeTopology(ILocalCluster clust } /** - * If using simulated time simulate waiting for 10 seconds. This is intended for internal testing only. + * If using simulated time simulate waiting for 10 seconds. This is intended for internal + * testing only. */ public static void simulateWait(ILocalCluster cluster) throws InterruptedException { if (Time.isSimulating()) { @@ -491,22 +521,26 @@ public static void simulateWait(ILocalCluster cluster) throws InterruptedExcepti /** * Get all of the tuples from a given component on the default stream. + * * @param results the results of running a completed topology * @param componentId the id of the component to look at * @return a list of the tuple values. */ - public static List> readTuples(Map> results, String componentId) { + public static List> readTuples(Map> results, + String componentId) { return readTuples(results, componentId, Utils.DEFAULT_STREAM_ID); } /** * Get all of the tuples from a given component on a given stream. + * * @param results the results of running a completed topology * @param componentId the id of the component to look at * @param streamId the id of the stream to look for. * @return a list of the tuple values. */ - public static List> readTuples(Map> results, String componentId, String streamId) { + public static List> readTuples(Map> results, + String componentId, String streamId) { List> ret = new ArrayList<>(); List streamResult = results.get(componentId); if (streamResult != null) { @@ -521,6 +555,7 @@ public static List> readTuples(Map> result /** * Create a tracked topology. + * * @deprecated use {@link org.apache.storm.testing.TrackedTopology} directly. */ @Deprecated @@ -545,7 +580,8 @@ public static void trackedWait(CapturedTopology topo, Integer a /** * Simulated time wait for a tracked topology. This is intended for internal testing. */ - public static void trackedWait(CapturedTopology topo, Integer amt, Integer timeoutMs) { + public static void trackedWait(CapturedTopology topo, Integer amt, + Integer timeoutMs) { topo.topology.trackedWait(amt, timeoutMs); } @@ -573,19 +609,22 @@ public static void trackedWait(TrackedTopology topo, Integer amt, Integer timeou /** * Simulated time wait for a cluster. This is intended for internal testing. */ - public static void advanceClusterTime(ILocalCluster cluster, Integer secs) throws InterruptedException { + public static void advanceClusterTime(ILocalCluster cluster, + Integer secs) throws InterruptedException { advanceClusterTime(cluster, secs, 1); } /** * Simulated time wait for a cluster. This is intended for internal testing. */ - public static void advanceClusterTime(ILocalCluster cluster, Integer secs, Integer step) throws InterruptedException { + public static void advanceClusterTime(ILocalCluster cluster, Integer secs, + Integer step) throws InterruptedException { cluster.advanceClusterTime(secs, step); } /** * Count how many times each element appears in the Collection. + * * @param c a collection of values * @return a map of the unique values in c to the count of those values. */ @@ -637,6 +676,7 @@ public static boolean multiseteq(Collection a, Collection b) { /** * Create a {@link org.apache.storm.tuple.Tuple} for use with testing. + * * @param values the values to appear in the tuple */ public static Tuple testTuple(List values) { @@ -645,6 +685,7 @@ public static Tuple testTuple(List values) { /** * Create a {@link org.apache.storm.tuple.Tuple} for use with testing. + * * @param values the values to appear in the tuple * @param param parametrs describing more details about the tuple */ @@ -707,12 +748,13 @@ public interface Condition { /** * A topology that has all messages captured and can be read later on. * This is intended mostly for internal testing. + * * @param the topology (tracked or regular) */ public static final class CapturedTopology { public final T topology; /** - * a Bolt that will hold all of the captured data. + * A Bolt that will hold all of the captured data. */ public final TupleCaptureBolt capturer; diff --git a/storm-server/src/main/java/org/apache/storm/blobstore/BlobKeySequenceInfo.java b/storm-server/src/main/java/org/apache/storm/blobstore/BlobKeySequenceInfo.java index e50575a4074..292a56bf92f 100644 --- a/storm-server/src/main/java/org/apache/storm/blobstore/BlobKeySequenceInfo.java +++ b/storm-server/src/main/java/org/apache/storm/blobstore/BlobKeySequenceInfo.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-server/src/main/java/org/apache/storm/blobstore/BlobStoreUtils.java b/storm-server/src/main/java/org/apache/storm/blobstore/BlobStoreUtils.java index f42b0c7e03e..c38ce8d119e 100644 --- a/storm-server/src/main/java/org/apache/storm/blobstore/BlobStoreUtils.java +++ b/storm-server/src/main/java/org/apache/storm/blobstore/BlobStoreUtils.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -55,7 +61,8 @@ public static CuratorFramework createZKClient(Map conf, DaemonTy Object port = conf.get(Config.STORM_ZOOKEEPER_PORT); ZookeeperAuthInfo zkAuthInfo = new ZookeeperAuthInfo(conf); CuratorFramework zkClient = CuratorUtils.newCurator(conf, zkServers, port, - (String) conf.get(Config.STORM_ZOOKEEPER_ROOT), zkAuthInfo, + (String) conf + .get(Config.STORM_ZOOKEEPER_ROOT), zkAuthInfo, type.getDefaultZkAcls(conf)); zkClient.start(); return zkClient; @@ -76,7 +83,8 @@ public static BlobKeySequenceInfo normalizeNimbusHostPortSequenceNumberInfo(Stri return keySequenceInfo; } - // Check for latest sequence number of a key inside zookeeper and return nimbodes containing the latest sequence number + // Check for latest sequence number of a key inside zookeeper and return nimbodes containing the + // latest sequence number public static Set getNimbodesWithLatestSequenceNumberOfBlob(CuratorFramework zkClient, String key) throws Exception { List stateInfoList; try { @@ -89,7 +97,8 @@ public static Set getNimbodesWithLatestSequenceNumberOfBlob(CuratorF Set nimbusInfoSet = new HashSet(); int latestSeqNumber = getLatestSequenceNumber(stateInfoList); - LOG.debug("getNimbodesWithLatestSequenceNumberOfBlob stateInfo {} version {}", stateInfoList, latestSeqNumber); + LOG.debug("getNimbodesWithLatestSequenceNumberOfBlob stateInfo {} version {}", + stateInfoList, latestSeqNumber); // Get the nimbodes with the latest version for (String state : stateInfoList) { BlobKeySequenceInfo sequenceInfo = normalizeNimbusHostPortSequenceNumberInfo(state); @@ -104,7 +113,8 @@ public static Set getNimbodesWithLatestSequenceNumberOfBlob(CuratorF // Get sequence number details from latest sequence number of the blob public static int getLatestSequenceNumber(List stateInfoList) { int seqNumber = 0; - // Get latest sequence number of the blob present in the zookeeper --> possible to refactor this piece of code + // Get latest sequence number of the blob present in the zookeeper --> possible to refactor + // this piece of code for (String state : stateInfoList) { BlobKeySequenceInfo sequenceInfo = normalizeNimbusHostPortSequenceNumberInfo(state); int currentSeqNumber = Integer.parseInt(sequenceInfo.getSequenceNumber()); @@ -118,7 +128,8 @@ public static int getLatestSequenceNumber(List stateInfoList) { } // Download missing blobs from potential nimbodes - public static boolean downloadMissingBlob(Map conf, BlobStore blobStore, String key, Set nimbusInfos) + public static boolean downloadMissingBlob(Map conf, BlobStore blobStore, + String key, Set nimbusInfos) throws TTransportException { ReadableBlobMeta rbm; ClientBlobStore remoteBlobStore; @@ -169,7 +180,8 @@ public static boolean downloadMissingBlob(Map conf, BlobStore bl } // Download updated blobs from potential nimbodes - public static boolean downloadUpdatedBlob(Map conf, BlobStore blobStore, String key, Set nimbusInfos) + public static boolean downloadUpdatedBlob(Map conf, BlobStore blobStore, + String key, Set nimbusInfos) throws TTransportException { ClientBlobStore remoteBlobStore; AtomicOutputStream out = null; @@ -179,7 +191,8 @@ public static boolean downloadUpdatedBlob(Map conf, BlobStore bl if (isSuccess) { break; } - try (NimbusClient client = NimbusClient.Builder.withConf(conf).forDaemon().buildWithNimbusHostPort(nimbusInfo.getHost(), + try (NimbusClient client = NimbusClient.Builder.withConf(conf).forDaemon() + .buildWithNimbusHostPort(nimbusInfo.getHost(), nimbusInfo.getPort())) { remoteBlobStore = new NimbusBlobStore(); remoteBlobStore.setClient(conf, client); @@ -195,7 +208,8 @@ public static boolean downloadUpdatedBlob(Map conf, BlobStore bl } isSuccess = true; } catch (FileNotFoundException fnf) { - LOG.warn("Blobstore file for key '{}' does not exist or got deleted before it could be downloaded.", key, fnf); + LOG.warn("Blobstore file for key '{}' does not exist or got deleted before it " + + "could be downloaded.", key, fnf); } catch (IOException | AuthorizationException exception) { throw new RuntimeException(exception); } catch (KeyNotFoundException knf) { @@ -236,21 +250,28 @@ public static List getKeyListFromBlobStore(BlobStore blobStore) throws E return keyList; } - public static void createStateInZookeeper(Map conf, String key, NimbusInfo nimbusInfo) throws TTransportException { + public static void createStateInZookeeper(Map conf, String key, + NimbusInfo nimbusInfo) throws TTransportException { ClientBlobStore cb = new NimbusBlobStore(); cb.setClient(conf, NimbusClient.Builder.withConf(conf).forDaemon().buildWithNimbusHostPort( nimbusInfo.getHost(), nimbusInfo.getPort())); cb.createStateInZookeeper(key); } - public static void updateKeyForBlobStore(Map conf, BlobStore blobStore, CuratorFramework zkClient, String key, + public static void updateKeyForBlobStore(Map conf, BlobStore blobStore, + CuratorFramework zkClient, String key, NimbusInfo nimbusDetails) { try { - // Most of clojure tests currently try to access the blobs using getBlob. Since, updateKeyForBlobStore - // checks for updating the correct version of the blob as a part of nimbus ha before performing any - // operation on it, there is a necessity to stub several test cases to ignore this method. It is a valid - // trade off to return if nimbusDetails which include the details of the current nimbus host port data are - // not initialized as a part of the test. Moreover, this applies to only local blobstore when used along with + // Most of clojure tests currently try to access the blobs using getBlob. Since, + // updateKeyForBlobStore + // checks for updating the correct version of the blob as a part of nimbus ha before + // performing any + // operation on it, there is a necessity to stub several test cases to ignore this + // method. It is a valid + // trade off to return if nimbusDetails which include the details of the current nimbus + // host port data are + // not initialized as a part of the test. Moreover, this applies to only local blobstore + // when used along with // nimbus ha. if (nimbusDetails == null) { return; @@ -266,7 +287,8 @@ public static void updateKeyForBlobStore(Map conf, BlobStore blo } LOG.debug("StateInfo for update {}", stateInfo); - Set nimbusInfoList = getNimbodesWithLatestSequenceNumberOfBlob(zkClient, key); + Set nimbusInfoList = getNimbodesWithLatestSequenceNumberOfBlob(zkClient, + key); for (NimbusInfo nimbusInfo : nimbusInfoList) { if (nimbusInfo.getHost().equals(nimbusDetails.getHost())) { @@ -275,12 +297,13 @@ public static void updateKeyForBlobStore(Map conf, BlobStore blo } } - if (!isListContainsCurrentNimbusInfo && downloadUpdatedBlob(conf, blobStore, key, nimbusInfoList)) { + if (!isListContainsCurrentNimbusInfo && downloadUpdatedBlob(conf, blobStore, key, + nimbusInfoList)) { LOG.debug("Updating state inside zookeeper for an update"); createStateInZookeeper(conf, key, nimbusDetails); } } catch (KeeperException.NoNodeException | KeyNotFoundException e) { - //race condition with a delete + // race condition with a delete return; } catch (Exception exp) { throw new RuntimeException(exp); diff --git a/storm-server/src/main/java/org/apache/storm/blobstore/FileBlobStoreImpl.java b/storm-server/src/main/java/org/apache/storm/blobstore/FileBlobStoreImpl.java index 3c87a617688..c46903675f6 100644 --- a/storm-server/src/main/java/org/apache/storm/blobstore/FileBlobStoreImpl.java +++ b/storm-server/src/main/java/org/apache/storm/blobstore/FileBlobStoreImpl.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -68,6 +74,7 @@ public void run() { /** * List keys. + * * @return all keys that are available for reading * @throws IOException on any error */ @@ -85,6 +92,7 @@ protected Iterator listKeys(File path) throws IOException { /** * Get an input stream for reading a part. + * * @param key the key of the part to read. * @return the where to read the data from. * @throws IOException on any error @@ -95,6 +103,7 @@ public LocalFsBlobStoreFile read(String key) throws IOException { /** * Get an object tied to writing the data. + * * @param key the key of the part to write to. * @return an object that can be used to both write to, but also commit/cancel the operation. * @throws IOException on any error @@ -105,6 +114,7 @@ public LocalFsBlobStoreFile write(String key, boolean create) throws IOException /** * Check if the key exists in the blob store. + * * @param key the key to check for * @return true if it exists else false. */ @@ -114,12 +124,14 @@ public boolean exists(String key) { /** * Delete a key from the blob store. + * * @param key the key to delete * @throws IOException on any error */ public void deleteKey(String key) throws IOException { File keyDir = getKeyDir(key); - LocalFsBlobStoreFile pf = new LocalFsBlobStoreFile(keyDir, BlobStoreFile.BLOBSTORE_DATA_FILE); + LocalFsBlobStoreFile pf = new LocalFsBlobStoreFile(keyDir, + BlobStoreFile.BLOBSTORE_DATA_FILE); pf.delete(); delete(keyDir); } @@ -140,7 +152,7 @@ public void fullCleanup(long age) throws IOException { File keyDir = getKeyDir(key); Iterator i = listBlobStoreFiles(keyDir); if (!i.hasNext()) { - //The dir is empty, so try to delete it, may fail, but that is OK + // The dir is empty, so try to delete it, may fail, but that is OK try { keyDir.delete(); } catch (Exception e) { @@ -166,7 +178,7 @@ protected Iterator listBlobStoreFiles(File path) throws IO try { ret.add(new LocalFsBlobStoreFile(sub.getParentFile(), sub.getName())); } catch (IllegalArgumentException e) { - //Ignored the file did not match + // Ignored the file did not match LOG.warn("Found an unexpected file in {} {}", path, sub.getName()); } } @@ -180,13 +192,15 @@ protected void delete(File path) throws IOException { Files.walkFileTree(path.toPath(), new SimpleFileVisitor() { @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + public FileVisitResult visitFile(Path file, + BasicFileAttributes attrs) throws IOException { Files.delete(file); return FileVisitResult.CONTINUE; } @Override - public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { + public FileVisitResult postVisitDirectory(Path dir, + IOException exc) throws IOException { Files.delete(dir); return FileVisitResult.CONTINUE; } diff --git a/storm-server/src/main/java/org/apache/storm/blobstore/KeySequenceNumber.java b/storm-server/src/main/java/org/apache/storm/blobstore/KeySequenceNumber.java index ae0f998603b..59864ebc625 100644 --- a/storm-server/src/main/java/org/apache/storm/blobstore/KeySequenceNumber.java +++ b/storm-server/src/main/java/org/apache/storm/blobstore/KeySequenceNumber.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -27,7 +33,8 @@ /** * Class hands over the key sequence number which implies the number of updates made to a blob. - * The information regarding the keys and the sequence number which represents the number of updates are + * The information regarding the keys and the sequence number which represents the number of updates + * are * stored within the zookeeper in the following format. * /storm/blobstore/key_name/nimbushostport-sequencenumber * Example: @@ -55,7 +62,8 @@ * The znodes it creates on these nodes are /storm/blobstore/key1/leader:8080-1, * /storm/blobstore/key1/non-leader:8080-1 and /storm/blobstoremaxkeysequencenumber/key1/1. * The latter holds the global sequence number across all nimbodes more like a static variable - * indicating the true value of number of updates for a blob. This node helps to maintain sanity in case + * indicating the true value of number of updates for a blob. This node helps to maintain sanity in + * case * leadership changes due to crashing. * *

      2. Delete does not require to hand over the sequence number. @@ -105,14 +113,18 @@ * N1-Restored alive alive - Leader 3 2 3 * Read/Update alive alive - Leader 3 4 (Downloads from N1) 4 * Sync alive alive - Leader 4 (callback) 4 4 - * Here the download is triggered whenever an operation corresponding to the blob is triggered on the - * nimbus like a read or update operation. Here, in the read/update call it is hard to know which call - * is read or update. Hence, by incrementing the sequence number to max-seq-num + 1 we ensure that the + * Here the download is triggered whenever an operation corresponding to the blob is triggered on + * the + * nimbus like a read or update operation. Here, in the read/update call it is hard to know which + * call + * is read or update. Hence, by incrementing the sequence number to max-seq-num + 1 we ensure that + * the * synchronization happens appropriately and all nimbodes have the same blob. */ public class KeySequenceNumber { private static final Logger LOG = LoggerFactory.getLogger(KeySequenceNumber.class); - private static final String BLOBSTORE_MAX_KEY_SEQUENCE_SUBTREE = "/blobstoremaxkeysequencenumber"; + private static final String BLOBSTORE_MAX_KEY_SEQUENCE_SUBTREE = + "/blobstoremaxkeysequencenumber"; private final String key; private final NimbusInfo nimbusInfo; private static final int INT_CAPACITY = 4; @@ -131,32 +143,44 @@ public synchronized int getKeySequenceNumber(CuratorFramework zkClient) throws K * Hand over the sequence number for the copy of the key held by this nimbus. * * @param zkClient the zookeeper client - * @param mayCreateKey whether the key may be created when zookeeper does not know it, which is only right for the - * leader storing a blob that a client uploads. Any other nimbus mirrors a key the leader created, + * @param mayCreateKey whether the key may be created when zookeeper does not know it, which is + * only right for the + * leader storing a blob that a client uploads. Any other nimbus mirrors a key the leader + * created, * so a key zookeeper does not know was deleted and must not be registered again. * @return the sequence number - * @throws KeyNotFoundException if the key is not in zookeeper and may not be created, or it is deleted meanwhile + * @throws KeyNotFoundException if the key is not in zookeeper and may not be created, or it is + * deleted meanwhile */ - public synchronized int getKeySequenceNumber(CuratorFramework zkClient, boolean mayCreateKey) throws KeyNotFoundException { + public synchronized int getKeySequenceNumber(CuratorFramework zkClient, + boolean mayCreateKey) throws KeyNotFoundException { TreeSet sequenceNumbers = new TreeSet(); try { // Key has not been created yet and it is the first time it is being created - if (zkClient.checkExists().forPath(BlobStoreUtils.getBlobStoreSubtree() + "/" + key) == null) { + if (zkClient.checkExists().forPath(BlobStoreUtils.getBlobStoreSubtree() + "/" + + key) == null) { if (!mayCreateKey) { - throw new KeeperException.NoNodeException(BlobStoreUtils.getBlobStoreSubtree() + "/" + key); + throw new KeeperException.NoNodeException(BlobStoreUtils.getBlobStoreSubtree() + + "/" + key); } zkClient.create().creatingParentsIfNeeded().withMode(CreateMode.PERSISTENT) - .withACL(ZooDefs.Ids.OPEN_ACL_UNSAFE).forPath(BLOBSTORE_MAX_KEY_SEQUENCE_SUBTREE + "/" + key); + .withACL(ZooDefs.Ids.OPEN_ACL_UNSAFE) + .forPath(BLOBSTORE_MAX_KEY_SEQUENCE_SUBTREE + "/" + key); zkClient.setData().forPath(BLOBSTORE_MAX_KEY_SEQUENCE_SUBTREE + "/" + key, - ByteBuffer.allocate(INT_CAPACITY).putInt(INITIAL_SEQUENCE_NUMBER).array()); + ByteBuffer.allocate(INT_CAPACITY) + .putInt(INITIAL_SEQUENCE_NUMBER).array()); return INITIAL_SEQUENCE_NUMBER; } // When all nimbodes go down and one or few of them come up - // Unfortunately there might not be an exact way to know which one contains the most updated blob, - // if all go down which is unlikely. Hence there might be a need to update the blob if all go down. - List stateInfoList = zkClient.getChildren().forPath(BlobStoreUtils.getBlobStoreSubtree() + "/" + key); - LOG.debug("stateInfoList-size {} stateInfoList-data {}", stateInfoList.size(), stateInfoList); + // Unfortunately there might not be an exact way to know which one contains the most + // updated blob, + // if all go down which is unlikely. Hence there might be a need to update the blob if + // all go down. + List stateInfoList = zkClient.getChildren().forPath(BlobStoreUtils + .getBlobStoreSubtree() + "/" + key); + LOG.debug("stateInfoList-size {} stateInfoList-data {}", stateInfoList.size(), + stateInfoList); if (stateInfoList.isEmpty()) { return getMaxSequenceNumber(zkClient); } @@ -166,7 +190,8 @@ public synchronized int getKeySequenceNumber(CuratorFramework zkClient, boolean // and assign the appropriate number. Check if all are have same sequence number, // if not assign the highest sequence number. for (String stateInfo : stateInfoList) { - sequenceNumbers.add(Integer.parseInt(BlobStoreUtils.normalizeNimbusHostPortSequenceNumberInfo(stateInfo) + sequenceNumbers.add(Integer.parseInt(BlobStoreUtils + .normalizeNimbusHostPortSequenceNumberInfo(stateInfo) .getSequenceNumber())); } @@ -174,7 +199,8 @@ public synchronized int getKeySequenceNumber(CuratorFramework zkClient, boolean // especially when nimbus crashes and comes up after and before update // respectively. int currentSeqNumber = getMaxSequenceNumber(zkClient); - if (!checkIfStateContainsCurrentNimbusHost(stateInfoList, nimbusInfo) && !nimbusInfo.isLeader()) { + if (!checkIfStateContainsCurrentNimbusHost(stateInfoList, nimbusInfo) && !nimbusInfo + .isLeader()) { if (sequenceNumbers.last() < currentSeqNumber) { return currentSeqNumber; } else { @@ -183,9 +209,11 @@ public synchronized int getKeySequenceNumber(CuratorFramework zkClient, boolean } // It covers scenarios expalined in scenario 3 when nimbus-1 holding the latest - // update goes down before it is downloaded by nimbus-2. Nimbus-2 gets elected as a leader + // update goes down before it is downloaded by nimbus-2. Nimbus-2 gets elected as a + // leader // after which nimbus-1 comes back up and a read or update is performed. - if (!checkIfStateContainsCurrentNimbusHost(stateInfoList, nimbusInfo) && nimbusInfo.isLeader()) { + if (!checkIfStateContainsCurrentNimbusHost(stateInfoList, nimbusInfo) && nimbusInfo + .isLeader()) { incrementMaxSequenceNumber(zkClient, currentSeqNumber); return currentSeqNumber + 1; } @@ -218,7 +246,8 @@ public synchronized int getKeySequenceNumber(CuratorFramework zkClient, boolean } } - private boolean checkIfStateContainsCurrentNimbusHost(List stateInfoList, NimbusInfo nimbusInfo) { + private boolean checkIfStateContainsCurrentNimbusHost(List stateInfoList, + NimbusInfo nimbusInfo) { boolean containsNimbusHost = false; for (String stateInfo : stateInfoList) { if (stateInfo.contains(nimbusInfo.getHost())) { @@ -236,6 +265,7 @@ private void incrementMaxSequenceNumber(CuratorFramework zkClient, int count) th private int getMaxSequenceNumber(CuratorFramework zkClient) throws Exception { return ByteBuffer.wrap(zkClient.getData() - .forPath(BLOBSTORE_MAX_KEY_SEQUENCE_SUBTREE + "/" + key)).getInt(); + .forPath(BLOBSTORE_MAX_KEY_SEQUENCE_SUBTREE + "/" + key)) + .getInt(); } } diff --git a/storm-server/src/main/java/org/apache/storm/blobstore/LocalFsBlobStore.java b/storm-server/src/main/java/org/apache/storm/blobstore/LocalFsBlobStore.java index a5d5c5bcf8e..077a9c66d22 100644 --- a/storm-server/src/main/java/org/apache/storm/blobstore/LocalFsBlobStore.java +++ b/storm-server/src/main/java/org/apache/storm/blobstore/LocalFsBlobStore.java @@ -1,15 +1,19 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. - * See the NOTICE file distributed with this work for additional information regarding copyright ownership. + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. + * See the NOTICE file distributed with this work for additional information regarding copyright + * ownership. * The ASF licenses this file to you under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. You may obtain a copy of the License at + * you may not use this file except in compliance with the License. You may obtain a copy of the + * License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, + *

      Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and limitations under the License. + * See the License for the specific language governing permissions and limitations under the + * License. */ package org.apache.storm.blobstore; @@ -55,27 +59,32 @@ import org.apache.storm.utils.Utils; import org.apache.storm.utils.WrappedKeyAlreadyExistsException; import org.apache.storm.utils.WrappedKeyNotFoundException; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Provides a local file system backed blob store implementation for Nimbus. * - *

      For a local blob store the user and the supervisor use NimbusBlobStore Client API in order to talk to nimbus through thrift. + *

      For a local blob store the user and the supervisor use NimbusBlobStore Client API in order to + * talk to nimbus through thrift. * The authentication and authorization here is based on the subject. - * We currently have NIMBUS_ADMINS and SUPERVISOR_ADMINS configuration. NIMBUS_ADMINS are given READ, WRITE and ADMIN - * access whereas the SUPERVISOR_ADMINS are given READ access in order to read and download the blobs form the nimbus. + * We currently have NIMBUS_ADMINS and SUPERVISOR_ADMINS configuration. NIMBUS_ADMINS are given + * READ, WRITE and ADMIN + * access whereas the SUPERVISOR_ADMINS are given READ access in order to read and download the + * blobs form the nimbus. * - *

      The ACLs for the blob store are validated against whether the subject is a NIMBUS_ADMIN, SUPERVISOR_ADMIN or USER + *

      The ACLs for the blob store are validated against whether the subject is a NIMBUS_ADMIN, + * SUPERVISOR_ADMIN or USER * who has read, write or admin privileges in order to perform respective operations on the blob. * *

      For local blob store * 1. The USER interacts with nimbus to upload and access blobs through NimbusBlobStore Client API. * 2. The USER sets the ACLs, and the blob access is validated against these ACLs. - * 3. The SUPERVISOR interacts with nimbus through the NimbusBlobStore Client API to download the blobs. + * 3. The SUPERVISOR interacts with nimbus through the NimbusBlobStore Client API to download the + * blobs. * The supervisors principal should match the set of users configured into SUPERVISOR_ADMINS. - * Here, the PrincipalToLocalPlugin takes care of mapping the principal to user name before the ACL validation. + * Here, the PrincipalToLocalPlugin takes care of mapping the principal to user name before the ACL + * validation. */ public class LocalFsBlobStore extends BlobStore { public static final Logger LOG = LoggerFactory.getLogger(LocalFsBlobStore.class); @@ -93,7 +102,8 @@ public class LocalFsBlobStore extends BlobStore { private ILeaderElector leaderElector; @Override - public void prepare(Map conf, String overrideBase, NimbusInfo nimbusInfo, ILeaderElector leaderElector) { + public void prepare(Map conf, String overrideBase, NimbusInfo nimbusInfo, + ILeaderElector leaderElector) { this.conf = conf; this.nimbusInfo = nimbusInfo; zkClient = BlobStoreUtils.createZKClient(conf, DaemonType.NIMBUS); @@ -108,9 +118,11 @@ public void prepare(Map conf, String overrideBase, NimbusInfo ni } aclHandler = new BlobStoreAclHandler(conf); try { - this.stormClusterState = ClusterUtils.mkStormClusterState(conf, new ClusterStateContext(DaemonType.NIMBUS, conf)); + this.stormClusterState = ClusterUtils.mkStormClusterState(conf, + new ClusterStateContext(DaemonType.NIMBUS, conf)); } catch (Exception e) { - throw new RuntimeException("Failed to initialize cluster state for LocalFsBlobStore", e); + throw new RuntimeException("Failed to initialize cluster state for LocalFsBlobStore", + e); } timer = new Timer("BLOB-STORE-TIMER", true); this.leaderElector = leaderElector; @@ -136,10 +148,12 @@ private void setupBlobstore() throws AuthorizationException, KeyNotFoundExceptio for (String toDelete : keysToDelete) { store.deleteBlob(toDelete, NIMBUS_SUBJECT); } - LOG.debug("Creating list of key entries for blobstore inside zookeeper {} local {}", activeKeys, activeLocalKeys); + LOG.debug("Creating list of key entries for blobstore inside zookeeper {} local {}", + activeKeys, activeLocalKeys); for (String key : activeLocalKeys) { try { - state.setupBlob(key, nimbusInfo, getVersionForKey(key, nimbusInfo, zkClient, false)); + state.setupBlob(key, nimbusInfo, getVersionForKey(key, nimbusInfo, zkClient, + false)); } catch (KeyNotFoundException e) { // invalid key, remove it from blobstore store.deleteBlob(key, NIMBUS_SUBJECT); @@ -148,7 +162,8 @@ private void setupBlobstore() throws AuthorizationException, KeyNotFoundExceptio } /** - * Tell whether this nimbus is the leader, which is the only one that may register a key zookeeper does not know. + * Tell whether this nimbus is the leader, which is the only one that may register a key + * zookeeper does not know. * Without a leader elector there is a single nimbus, which is then the leader. */ private boolean isLeader() { @@ -159,7 +174,6 @@ private boolean isLeader() { } } - private void blobSync() throws Exception { if ("distributed".equals(conf.get(Config.STORM_CLUSTER_MODE))) { if (!this.leaderElector.isLeader()) { @@ -184,14 +198,13 @@ private void blobSync() throws Exception { sync.setZookeeperKeySet(zkKeys); sync.setZkClient(zkClient); sync.syncBlobs(); - } //else leader (NOOP) - } //else local (NOOP) + } // else leader (NOOP) + } // else local (NOOP) } - @Override public void startSyncBlobs() throws KeyNotFoundException, AuthorizationException { - //register call back for blob-store + // register call back for blob-store this.stormClusterState.blobstore(() -> { try { blobSync(); @@ -201,7 +214,7 @@ public void startSyncBlobs() throws KeyNotFoundException, AuthorizationExceptio }); setupBlobstore(); - //Schedule nimbus code sync thread to sync code from other nimbuses. + // Schedule nimbus code sync thread to sync code from other nimbuses. this.timer.scheduleAtFixedRate(new TimerTask() { @Override public void run() { @@ -216,7 +229,8 @@ public void run() { } @Override - public AtomicOutputStream createBlob(String key, SettableBlobMeta meta, Subject who) throws AuthorizationException, + public AtomicOutputStream createBlob(String key, SettableBlobMeta meta, + Subject who) throws AuthorizationException, KeyAlreadyExistsException { LOG.debug("Creating Blob for key {}", key); validateKey(key); @@ -228,8 +242,9 @@ public AtomicOutputStream createBlob(String key, SettableBlobMeta meta, Subject } BlobStoreFileOutputStream outputStream = null; try { - //Taken before anything is written, so that a non-leader downloading a key that was deleted meanwhile is - //refused without leaving a copy behind. + // Taken before anything is written, so that a non-leader downloading a key that was + // deleted meanwhile is + // refused without leaving a copy behind. int version = getVersionForKey(key, this.nimbusInfo, zkClient, isLeader()); outputStream = new BlobStoreFileOutputStream(fbs.write(META_PREFIX + key, true)); outputStream.write(Utils.thriftSerialize(meta)); @@ -246,14 +261,15 @@ public AtomicOutputStream createBlob(String key, SettableBlobMeta meta, Subject try { outputStream.cancel(); } catch (IOException e) { - //Ignored + // Ignored } } } } @Override - public AtomicOutputStream updateBlob(String key, Subject who) throws AuthorizationException, KeyNotFoundException { + public AtomicOutputStream updateBlob(String key, + Subject who) throws AuthorizationException, KeyNotFoundException { validateKey(key); checkPermission(key, who, WRITE); try { @@ -288,14 +304,15 @@ private SettableBlobMeta getStoredBlobMeta(String key) throws KeyNotFoundExcepti try { in.close(); } catch (IOException e) { - //Ignored + // Ignored } } } } @Override - public ReadableBlobMeta getBlobMeta(String key, Subject who) throws AuthorizationException, KeyNotFoundException { + public ReadableBlobMeta getBlobMeta(String key, + Subject who) throws AuthorizationException, KeyNotFoundException { validateKey(key); if (!checkForBlobOrDownload(key)) { checkForBlobUpdate(key); @@ -322,7 +339,8 @@ public void setLeaderElector(ILeaderElector leaderElector) { } @Override - public void setBlobMeta(String key, SettableBlobMeta meta, Subject who) throws AuthorizationException, KeyNotFoundException { + public void setBlobMeta(String key, SettableBlobMeta meta, + Subject who) throws AuthorizationException, KeyNotFoundException { validateKey(key); checkForBlobOrDownload(key); aclHandler.normalizeSettableBlobMeta(key, meta, who, ADMIN); @@ -342,14 +360,15 @@ public void setBlobMeta(String key, SettableBlobMeta meta, Subject who) throws A try { outputStream.cancel(); } catch (IOException e) { - //Ignored + // Ignored } } } } @Override - public void deleteBlob(String key, Subject who) throws AuthorizationException, KeyNotFoundException { + public void deleteBlob(String key, + Subject who) throws AuthorizationException, KeyNotFoundException { validateKey(key); if (!aclHandler.checkForValidUsers(who, WRITE)) { @@ -359,13 +378,15 @@ public void deleteBlob(String key, Subject who) throws AuthorizationException, K try { checkPermission(key, who, WRITE); } catch (KeyNotFoundException e) { - LOG.error("Error while retrieving meta from ZK or local... key: {} subject: {}", key, who); + LOG.error("Error while retrieving meta from ZK or local... key: {} subject: {}", + key, who); throw e; } } else { // able to delete the blob without checking meta's ACL // skip checking everything and continue deleting local files - LOG.debug("Given subject is eligible to delete key without checking ACL, skipping... key: {} subject: {}", + LOG.debug("Given subject is eligible to delete key without checking ACL, skipping... " + + "key: {} subject: {}", key, who); } @@ -379,7 +400,8 @@ public void deleteBlob(String key, Subject who) throws AuthorizationException, K this.stormClusterState.removeKeyVersion(key); } - private void checkPermission(String key, Subject who, int mask) throws KeyNotFoundException, AuthorizationException { + private void checkPermission(String key, Subject who, + int mask) throws KeyNotFoundException, AuthorizationException { checkForBlobOrDownload(key); SettableBlobMeta meta = getStoredBlobMeta(key); aclHandler.hasPermissions(meta.get_acl(), mask, who, key); @@ -390,7 +412,8 @@ private void deleteKeyIgnoringFileNotFound(String key) throws IOException { fbs.deleteKey(key); } catch (IOException e) { if (e instanceof FileNotFoundException || e instanceof NoSuchFileException) { - LOG.debug("Ignoring FileNotFoundException since we're about to delete such key... key: {}", key); + LOG.debug("Ignoring FileNotFoundException since we're about to delete such key... " + + "key: {}", key); } else { throw e; } @@ -398,7 +421,8 @@ private void deleteKeyIgnoringFileNotFound(String key) throws IOException { } @Override - public InputStreamWithMeta getBlob(String key, Subject who) throws AuthorizationException, KeyNotFoundException { + public InputStreamWithMeta getBlob(String key, + Subject who) throws AuthorizationException, KeyNotFoundException { validateKey(key); if (!checkForBlobOrDownload(key)) { checkForBlobUpdate(key); @@ -444,26 +468,31 @@ public int getBlobReplication(String key, Subject who) throws Exception { try { replicationCount = zkClient.getChildren().forPath(BLOBSTORE_SUBTREE + key).size(); } catch (KeeperException.NoNodeException e) { - //Race with delete - //If it is not here the replication is 0 + // Race with delete + // If it is not here the replication is 0 } return replicationCount; } @Override - public int updateBlobReplication(String key, int replication, Subject who) throws AuthorizationException, KeyNotFoundException { - throw new UnsupportedOperationException("For local file system blob store the update blobs function does not work. " - + "Please use HDFS blob store to make this feature available."); + public int updateBlobReplication(String key, int replication, + Subject who) throws AuthorizationException, KeyNotFoundException { + throw new UnsupportedOperationException("For local file system blob store the update " + + "blobs function does not work. " + + "Please use HDFS blob store to make this " + + "feature available."); } - //This additional check and download is for nimbus high availability in case you have more than one nimbus + // This additional check and download is for nimbus high availability in case you have more than + // one nimbus public synchronized boolean checkForBlobOrDownload(String key) throws KeyNotFoundException { boolean checkBlobDownload = false; try { List keyList = BlobStoreUtils.getKeyListFromBlobStore(this); if (!keyList.contains(key)) { if (zkClient.checkExists().forPath(BLOBSTORE_SUBTREE + key) != null) { - Set nimbusSet = BlobStoreUtils.getNimbodesWithLatestSequenceNumberOfBlob(zkClient, key); + Set nimbusSet = BlobStoreUtils + .getNimbodesWithLatestSequenceNumberOfBlob(zkClient, key); nimbusSet.remove(this.nimbusInfo); if (BlobStoreUtils.downloadMissingBlob(conf, this, key, nimbusSet)) { LOG.debug("Updating blobs state"); diff --git a/storm-server/src/main/java/org/apache/storm/blobstore/LocalFsBlobStoreFile.java b/storm-server/src/main/java/org/apache/storm/blobstore/LocalFsBlobStoreFile.java index 128377f9811..6b1b17204e2 100644 --- a/storm-server/src/main/java/org/apache/storm/blobstore/LocalFsBlobStoreFile.java +++ b/storm-server/src/main/java/org/apache/storm/blobstore/LocalFsBlobStoreFile.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

      - * http://www.apache.org/licenses/LICENSE-2.0 - *

      - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

      http://www.apache.org/licenses/LICENSE-2.0 + * + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -23,11 +29,9 @@ import java.util.regex.Matcher; import java.util.zip.CRC32C; import java.util.zip.Checksum; - import org.apache.commons.io.FileUtils; import org.apache.storm.generated.SettableBlobMeta; - public class LocalFsBlobStoreFile extends BlobStoreFile { private final String key; @@ -43,7 +47,8 @@ public LocalFsBlobStoreFile(File base, String name) { } else { Matcher m = TMP_NAME_PATTERN.matcher(name); if (!m.matches()) { - throw new IllegalArgumentException("File name does not match '" + name + "' !~ " + TMP_NAME_PATTERN); + throw new IllegalArgumentException("File name does not match '" + name + "' !~ " + + TMP_NAME_PATTERN); } isTmp = true; } @@ -107,7 +112,7 @@ public OutputStream getOutputStream() throws IOException { try { success = path.createNewFile(); } catch (IOException e) { - //Try to create the parent directory, may not work + // Try to create the parent directory, may not work path.getParentFile().mkdirs(); success = path.createNewFile(); } @@ -127,7 +132,8 @@ public void commit() throws IOException { if (mustBeNew) { Files.move(path.toPath(), dest.toPath(), StandardCopyOption.ATOMIC_MOVE); } else { - Files.move(path.toPath(), dest.toPath(), StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + Files.move(path.toPath(), dest.toPath(), StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING); } } diff --git a/storm-server/src/main/java/org/apache/storm/blobstore/LocalFsBlobStoreSynchronizer.java b/storm-server/src/main/java/org/apache/storm/blobstore/LocalFsBlobStoreSynchronizer.java index 79bbdeb3270..e45d7b8f14c 100644 --- a/storm-server/src/main/java/org/apache/storm/blobstore/LocalFsBlobStoreSynchronizer.java +++ b/storm-server/src/main/java/org/apache/storm/blobstore/LocalFsBlobStoreSynchronizer.java @@ -9,7 +9,7 @@ * You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, + *

      Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, * either express or implied. @@ -79,17 +79,22 @@ public void setZookeeperKeySet(Set zookeeperKeySet) { public synchronized void syncBlobs() { try { - LOG.debug("Sync blobs - blobstore keys {}, zookeeper keys {}", getBlobStoreKeySet(), getZookeeperKeySet()); + LOG.debug("Sync blobs - blobstore keys {}, zookeeper keys {}", getBlobStoreKeySet(), + getZookeeperKeySet()); deleteKeySetFromBlobStoreNotOnZookeeper(getBlobStoreKeySet(), getZookeeperKeySet()); updateKeySetForBlobStore(getBlobStoreKeySet()); - Set keySetToDownload = getKeySetToDownload(getBlobStoreKeySet(), getZookeeperKeySet()); - LOG.debug("Key set Blobstore-> Zookeeper-> DownloadSet {}-> {}-> {}", getBlobStoreKeySet(), getZookeeperKeySet(), + Set keySetToDownload = getKeySetToDownload(getBlobStoreKeySet(), + getZookeeperKeySet()); + LOG.debug("Key set Blobstore-> Zookeeper-> DownloadSet {}-> {}-> {}", + getBlobStoreKeySet(), getZookeeperKeySet(), keySetToDownload); for (String key : keySetToDownload) { try { - Set nimbusInfoSet = BlobStoreUtils.getNimbodesWithLatestSequenceNumberOfBlob(zkClient, key); - // Removing self so as not to create a deadlock where a nimbus is trying to download a missing blob + Set nimbusInfoSet = BlobStoreUtils + .getNimbodesWithLatestSequenceNumberOfBlob(zkClient, key); + // Removing self so as not to create a deadlock where a nimbus is trying to + // download a missing blob // from itself nimbusInfoSet.remove(this.nimbusInfo); LOG.debug("syncBlobs, key: {}, nimbusInfoSet: {}", key, nimbusInfoSet); @@ -97,7 +102,8 @@ public synchronized void syncBlobs() { BlobStoreUtils.createStateInZookeeper(conf, key, nimbusInfo); } } catch (KeyNotFoundException e) { - LOG.debug("Detected deletion for the key {} while downloading - skipping download", key); + LOG.debug("Detected deletion for the key {} while downloading - skipping " + + "download", key); } } } catch (InterruptedException | ClosedByInterruptException exp) { @@ -107,7 +113,8 @@ public synchronized void syncBlobs() { } } - public void deleteKeySetFromBlobStoreNotOnZookeeper(Set keySetBlobStore, Set keySetZookeeper) throws Exception { + public void deleteKeySetFromBlobStoreNotOnZookeeper(Set keySetBlobStore, + Set keySetZookeeper) throws Exception { if (keySetBlobStore.removeAll(keySetZookeeper) || (keySetZookeeper.isEmpty() && !keySetBlobStore.isEmpty())) { LOG.debug("Key set to delete in blobstore {}", keySetBlobStore); @@ -130,7 +137,8 @@ public void updateKeySetForBlobStore(Set keySetBlobStore) { } // Make a key list to download - public Set getKeySetToDownload(Set blobStoreKeySet, Set zookeeperKeySet) { + public Set getKeySetToDownload(Set blobStoreKeySet, + Set zookeeperKeySet) { zookeeperKeySet.removeAll(blobStoreKeySet); LOG.debug("Key list to download {}", zookeeperKeySet); return zookeeperKeySet; diff --git a/storm-server/src/main/java/org/apache/storm/container/DefaultResourceIsolationManager.java b/storm-server/src/main/java/org/apache/storm/container/DefaultResourceIsolationManager.java index c5a0466b4b6..3f1175574db 100644 --- a/storm-server/src/main/java/org/apache/storm/container/DefaultResourceIsolationManager.java +++ b/storm-server/src/main/java/org/apache/storm/container/DefaultResourceIsolationManager.java @@ -1,12 +1,17 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -31,10 +36,12 @@ import org.slf4j.LoggerFactory; /** - * This is the default class to manage worker processes, including launching, killing, profiling and etc. + * This is the default class to manage worker processes, including launching, killing, profiling and + * etc. */ public class DefaultResourceIsolationManager implements ResourceIsolationInterface { - private static final Logger LOG = LoggerFactory.getLogger(DefaultResourceIsolationManager.class); + private static final Logger LOG = LoggerFactory + .getLogger(DefaultResourceIsolationManager.class); protected Map conf; protected boolean runAsUser; @@ -45,13 +52,14 @@ public void prepare(Map conf) throws IOException { } @Override - public void reserveResourcesForWorker(String workerId, Integer workerMemory, Integer workerCpu, String numaId) { - //NO OP + public void reserveResourcesForWorker(String workerId, Integer workerMemory, Integer workerCpu, + String numaId) { + // NO OP } @Override public void cleanup(String user, String workerId, int port) throws IOException { - //NO OP + // NO OP } @Override @@ -60,13 +68,15 @@ public void launchWorkerProcess(String user, String topologyId, Map args = Arrays.asList("worker", workerDir, ServerUtils.writeScript(workerDir, command, env)); + List args = Arrays.asList("worker", workerDir, ServerUtils + .writeScript(workerDir, command, env)); ClientSupervisorUtils.processLauncher( conf, user, null, args, null, logPrefix, processExitCallback, targetDir ); } else { - ClientSupervisorUtils.launchProcess(command, env, logPrefix, processExitCallback, targetDir); + ClientSupervisorUtils.launchProcess(command, env, logPrefix, processExitCallback, + targetDir); } } @@ -90,6 +100,7 @@ public void kill(String user, String workerId) throws IOException { /** * Kill a given process. + * * @param pid the id of the process to kill * @throws IOException on I/O exception */ @@ -111,6 +122,7 @@ public void forceKill(String user, String workerId) throws IOException { /** * Kill a given process forcefully. + * * @param pid the id of the process to kill * @throws IOException on I/O exception */ @@ -124,18 +136,21 @@ private void forceKill(long pid, String user) throws IOException { /** * Get all the pids that are a part of the container. + * * @return all of the pids that are a part of this container */ protected Set getAllPids(String workerId) throws IOException { Set ret = new HashSet<>(); - for (String listing : ConfigUtils.readDirContents(ConfigUtils.workerPidsRoot(conf, workerId))) { + for (String listing : ConfigUtils.readDirContents(ConfigUtils.workerPidsRoot(conf, + workerId))) { ret.add(Long.valueOf(listing)); } return ret; } private void signal(long pid, int signal, String user) throws IOException { - List commands = Arrays.asList("signal", String.valueOf(pid), String.valueOf(signal)); + List commands = Arrays.asList("signal", String.valueOf(pid), String + .valueOf(signal)); String logPrefix = "kill -" + signal + " " + pid; ClientSupervisorUtils.processLauncherAndWait(conf, user, commands, null, logPrefix); } @@ -147,7 +162,8 @@ public boolean areAllProcessesDead(String user, String workerId) throws IOExcept } @Override - public boolean runProfilingCommand(String user, String workerId, List command, Map env, + public boolean runProfilingCommand(String user, String workerId, List command, + Map env, String logPrefix, File targetDir) throws IOException, InterruptedException { if (runAsUser) { String td = targetDir.getAbsolutePath(); @@ -162,10 +178,12 @@ public boolean runProfilingCommand(String user, String workerId, List co } String script = ServerUtils.writeScript(td, command, env); List args = Arrays.asList("profiler", td, script); - int ret = ClientSupervisorUtils.processLauncherAndWait(conf, user, args, env, logPrefix); + int ret = ClientSupervisorUtils.processLauncherAndWait(conf, user, args, env, + logPrefix); return ret == 0; } else { - Process p = ClientSupervisorUtils.launchProcess(command, env, logPrefix, null, targetDir); + Process p = ClientSupervisorUtils.launchProcess(command, env, logPrefix, null, + targetDir); int ret = p.waitFor(); return ret == 0; } @@ -173,6 +191,7 @@ public boolean runProfilingCommand(String user, String workerId, List co /** * This class doesn't really manage resources. + * * @return false */ @Override diff --git a/storm-server/src/main/java/org/apache/storm/container/ResourceIsolationInterface.java b/storm-server/src/main/java/org/apache/storm/container/ResourceIsolationInterface.java index 1ea3f651fb3..4fa2e721706 100644 --- a/storm-server/src/main/java/org/apache/storm/container/ResourceIsolationInterface.java +++ b/storm-server/src/main/java/org/apache/storm/container/ResourceIsolationInterface.java @@ -1,12 +1,17 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

      * http://www.apache.org/licenses/LICENSE-2.0 *

      - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -33,17 +38,20 @@ public interface ResourceIsolationInterface { void prepare(Map conf) throws IOException; /** - * This function should be used prior to starting the worker to reserve resources for the worker. + * This function should be used prior to starting the worker to reserve resources for the + * worker. * * @param workerId worker id of the worker to start * @param workerMemory the amount of memory for the worker or null if not enforced * @param workerCpu the amount of cpu for the worker or null if not enforced * @param numaId NUMA zone if applicable the worker should be bound to */ - void reserveResourcesForWorker(String workerId, Integer workerMemory, Integer workerCpu, String numaId); + void reserveResourcesForWorker(String workerId, Integer workerMemory, Integer workerCpu, + String numaId); /** - * This function will be called when the worker needs to shutdown. This function should include logic to clean up + * This function will be called when the worker needs to shutdown. This function should include + * logic to clean up * after a worker is shutdown. * * @param user the user of the worker @@ -55,6 +63,7 @@ public interface ResourceIsolationInterface { /** * After reserving resources for the worker (i.e. calling reserveResourcesForWorker), * this function can be used to launch worker process. + * * @param user the user who runs the command as * @param topologyId the Id of the topology * @param topoConf the topology configuration @@ -74,6 +83,7 @@ void launchWorkerProcess(String user, String topologyId, Map top /** * Get the current memory usage of the a given worker. + * * @param user the user that the worker is running as * @param workerId the id of the worker * @param port the port of the worker @@ -85,6 +95,7 @@ void launchWorkerProcess(String user, String topologyId, Map top /** * Get the amount of free memory in MB. * This might not be the entire box, it might be within a parent resource group. + * * @return The amount of memory in MB that are free on the system. * @throws IOException on I/O exception */ @@ -92,6 +103,7 @@ void launchWorkerProcess(String user, String topologyId, Map top /** * Kill the given worker. + * * @param user the user that the worker is running as * @param workerId the id of the worker to kill * @throws IOException on I/O exception @@ -100,6 +112,7 @@ void launchWorkerProcess(String user, String topologyId, Map top /** * Kill the given worker forcefully. + * * @param user the user that the worker is running as * @param workerId the id of the worker to kill * @throws IOException on I/O exception @@ -108,6 +121,7 @@ void launchWorkerProcess(String user, String topologyId, Map top /** * Check if all the processes are dead. + * * @param user the user that the processes are running as * @param workerId the id of the worker to kill * @return true if all the processed are dead; false otherwise @@ -117,6 +131,7 @@ void launchWorkerProcess(String user, String topologyId, Map top /** * Run profiling command. + * * @param user the user that the worker is running as * @param workerId the id of the worker * @param command the command to run @@ -127,12 +142,15 @@ void launchWorkerProcess(String user, String topologyId, Map top * @throws IOException on I/O exception * @throws InterruptedException if interrupted */ - boolean runProfilingCommand(String user, String workerId, List command, Map env, + boolean runProfilingCommand(String user, String workerId, List command, Map env, String logPrefix, File targetDir) throws IOException, InterruptedException; /** * Return true if resources are being managed. - * The {@link DefaultResourceIsolationManager} will have it return false since it doesn't really manage resources. + * The {@link DefaultResourceIsolationManager} will have it return false since it doesn't really + * manage resources. + * * @return true if resources are being managed. */ boolean isResourceManaged(); diff --git a/storm-server/src/main/java/org/apache/storm/container/cgroup/CgroupManager.java b/storm-server/src/main/java/org/apache/storm/container/cgroup/CgroupManager.java index cf8b4edc8a8..00d34235400 100644 --- a/storm-server/src/main/java/org/apache/storm/container/cgroup/CgroupManager.java +++ b/storm-server/src/main/java/org/apache/storm/container/cgroup/CgroupManager.java @@ -1,12 +1,17 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

      * http://www.apache.org/licenses/LICENSE-2.0 *

      - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -26,7 +31,6 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; - import org.apache.commons.lang3.SystemUtils; import org.apache.storm.Config; import org.apache.storm.DaemonConfig; @@ -54,7 +58,7 @@ public class CgroupManager extends DefaultResourceIsolationManager { private Map workerToNumaId; /** - * initialize data structures. + * Initialize data structures. * * @param conf storm confs */ @@ -63,14 +67,16 @@ public void prepare(Map conf) throws IOException { super.prepare(conf); this.rootDir = DaemonConfig.getCgroupRootDir(this.conf); if (this.rootDir == null) { - throw new RuntimeException("Check configuration file. The storm.supervisor.cgroup.rootdir is missing."); + throw new RuntimeException("Check configuration file. The " + + "storm.supervisor.cgroup.rootdir is missing."); } File file = new File(DaemonConfig.getCgroupStormHierarchyDir(conf), rootDir); if (!file.exists()) { LOG.error("{} does not exist", file.getPath()); throw new RuntimeException( - "Check if cgconfig service starts or /etc/cgconfig.conf is consistent with configuration file."); + "Check if cgconfig service starts or /etc/cgconfig.conf is consistent with " + + "configuration file."); } this.center = CgroupCenter.getInstance(); if (this.center == null) { @@ -101,10 +107,12 @@ private void prepareSubSystem(Map conf) throws IOException { new CgroupCommon(this.rootDir, this.hierarchy, this.hierarchy.getRootCgroups()); // set upper limit to how much cpu can be used by all workers running on supervisor node. - // This is done so that some cpu cycles will remain free to run the daemons and other miscellaneous OS + // This is done so that some cpu cycles will remain free to run the daemons and other + // miscellaneous OS // operations. CpuCore supervisorRootCpu = (CpuCore) this.rootCgroup.getCores().get(SubSystemType.cpu); - setCpuUsageUpperLimit(supervisorRootCpu, ((Number) this.conf.get(Config.SUPERVISOR_CPU_CAPACITY)).intValue()); + setCpuUsageUpperLimit(supervisorRootCpu, ((Number) this.conf + .get(Config.SUPERVISOR_CPU_CAPACITY)).intValue()); } /** @@ -123,9 +131,12 @@ private void setCpuUsageUpperLimit(CpuCore cpuCore, int cpuCoreUpperLimit) throw } @Override - public void reserveResourcesForWorker(String workerId, Integer totalMem, Integer cpuNum, String numaId) throws SecurityException { - LOG.info("Creating cgroup for worker {} with resources {} MB {} % CPU", workerId, totalMem, cpuNum); - // The manually set STORM_WORKER_CGROUP_CPU_LIMIT config on supervisor will overwrite resources assigned by + public void reserveResourcesForWorker(String workerId, Integer totalMem, Integer cpuNum, + String numaId) throws SecurityException { + LOG.info("Creating cgroup for worker {} with resources {} MB {} % CPU", workerId, totalMem, + cpuNum); + // The manually set STORM_WORKER_CGROUP_CPU_LIMIT config on supervisor will overwrite + // resources assigned by // RAS (Resource Aware Scheduler) if (conf.get(DaemonConfig.STORM_WORKER_CGROUP_CPU_LIMIT) != null) { cpuNum = ((Number) conf.get(DaemonConfig.STORM_WORKER_CGROUP_CPU_LIMIT)).intValue(); @@ -135,7 +146,8 @@ public void reserveResourcesForWorker(String workerId, Integer totalMem, Integer // resources assigned by RAS (Resource Aware Scheduler) if (this.conf.get(DaemonConfig.STORM_WORKER_CGROUP_MEMORY_MB_LIMIT) != null) { totalMem = - ((Number) this.conf.get(DaemonConfig.STORM_WORKER_CGROUP_MEMORY_MB_LIMIT)).intValue(); + ((Number) this.conf.get(DaemonConfig.STORM_WORKER_CGROUP_MEMORY_MB_LIMIT)) + .intValue(); } CgroupCommon workerGroup = new CgroupCommon(workerId, this.hierarchy, this.rootCgroup); @@ -160,7 +172,8 @@ public void reserveResourcesForWorker(String workerId, Integer totalMem, Integer (int) (Math.ceil( ObjectReader.getDouble( - this.conf.get(DaemonConfig.STORM_CGROUP_MEMORY_LIMIT_TOLERANCE_MARGIN_MB), + this.conf + .get(DaemonConfig.STORM_CGROUP_MEMORY_LIMIT_TOLERANCE_MARGIN_MB), 0.0))); long memLimit = Long.valueOf((totalMem.longValue() + cgroupMem) * 1024 * 1024); MemoryCore memCore = (MemoryCore) workerGroup.getCores().get(SubSystemType.memory); @@ -169,20 +182,24 @@ public void reserveResourcesForWorker(String workerId, Integer totalMem, Integer } catch (IOException e) { throw new RuntimeException("Cannot set memory.limit_in_bytes! Exception: ", e); } - // need to set memory.memsw.limit_in_bytes after setting memory.limit_in_bytes or error + // need to set memory.memsw.limit_in_bytes after setting memory.limit_in_bytes or + // error // might occur try { memCore.setWithSwapUsageLimit(memLimit); } catch (IOException e) { - throw new RuntimeException("Cannot set memory.memsw.limit_in_bytes! Exception: ", e); + throw new RuntimeException("Cannot set memory.memsw.limit_in_bytes! " + + "Exception: ", e); } } } if ((boolean) this.conf.get(DaemonConfig.STORM_CGROUP_INHERIT_CPUSET_CONFIGS)) { if (workerGroup.getParent().getCores().containsKey(SubSystemType.cpuset)) { - CpusetCore parentCpusetCore = (CpusetCore) workerGroup.getParent().getCores().get(SubSystemType.cpuset); - CpusetCore cpusetCore = (CpusetCore) workerGroup.getCores().get(SubSystemType.cpuset); + CpusetCore parentCpusetCore = (CpusetCore) workerGroup.getParent().getCores() + .get(SubSystemType.cpuset); + CpusetCore cpusetCore = (CpusetCore) workerGroup.getCores() + .get(SubSystemType.cpuset); try { cpusetCore.setCpus(parentCpusetCore.getCpus()); } catch (IOException e) { @@ -208,7 +225,8 @@ public void cleanup(String user, String workerId, int port) throws IOException { try { Set tasks = workerGroup.getTasks(); if (!tasks.isEmpty()) { - throw new Exception("Cannot correctly shutdown worker CGroup " + workerId + "tasks " + tasks + throw new Exception("Cannot correctly shutdown worker CGroup " + workerId + + "tasks " + tasks + " still running!"); } this.center.deleteCgroup(workerGroup); @@ -219,6 +237,7 @@ public void cleanup(String user, String workerId, int port) throws IOException { /** * Extracting out to mock it for tests. + * * @return true if on Linux. */ protected static boolean isOnLinux() { @@ -248,18 +267,21 @@ public void launchWorkerProcess(String user, String topologyId, Map args = Arrays.asList("worker", workerDir, ServerUtils.writeScript(workerDir, command, env)); + List args = Arrays.asList("worker", workerDir, ServerUtils + .writeScript(workerDir, command, env)); List commandPrefix = getLaunchCommandPrefix(workerId); ClientSupervisorUtils.processLauncher(conf, user, commandPrefix, args, null, logPrefix, processExitCallback, targetDir); } else { command = getLaunchCommand(workerId, command); - ClientSupervisorUtils.launchProcess(command, env, logPrefix, processExitCallback, targetDir); + ClientSupervisorUtils.launchProcess(command, env, logPrefix, processExitCallback, + targetDir); } } /** * To compose launch command based on workerId and existing command. + * * @param workerId the worker id * @param existingCommand the current command to run that may need to be modified * @return new commandline with necessary additions to launch worker @@ -281,7 +303,8 @@ private List getLaunchCommandPrefix(String workerId) { if (!this.rootCgroup.getChildren().contains(workerGroup)) { throw new RuntimeException( - "cgroup " + workerGroup + " doesn't exist! Need to reserve resources for worker first!"); + "cgroup " + workerGroup + + " doesn't exist! Need to reserve resources for worker first!"); } StringBuilder sb = new StringBuilder(); @@ -314,6 +337,7 @@ private Set getRunningPids(String workerId) throws IOException { /** * Get all of the pids that are a part of this container. + * * @param workerId the worker id * @return all of the pids that are a part of this container */ @@ -339,13 +363,13 @@ public long getSystemFreeMemoryMb() throws IOException { try { MemoryCore memRoot = (MemoryCore) rootCgroup.getCores().get(SubSystemType.memory); if (memRoot != null) { - //For cgroups no limit is max long. + // For cgroups no limit is max long. long limit = memRoot.getPhysicalUsageLimit(); long used = memRoot.getMaxPhysicalUsage(); rootCgroupLimitFree = (limit - used) / 1024 / 1024; } } catch (FileNotFoundException e) { - //Ignored if cgroups is not setup don't do anything with it + // Ignored if cgroups is not setup don't do anything with it } return Long.min(rootCgroupLimitFree, ServerUtils.getMemInfoFreeMb()); diff --git a/storm-server/src/main/java/org/apache/storm/container/docker/DockerCommand.java b/storm-server/src/main/java/org/apache/storm/container/docker/DockerCommand.java index fe77070f641..df45eb42275 100644 --- a/storm-server/src/main/java/org/apache/storm/container/docker/DockerCommand.java +++ b/storm-server/src/main/java/org/apache/storm/container/docker/DockerCommand.java @@ -1,12 +1,17 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -27,14 +32,18 @@ protected DockerCommand(String command) { commandWithArguments.add(command); } - /** Returns the docker sub-command string being used, e.g 'run'. + /** + * Returns the docker sub-command string being used, e.g 'run'. + * * @return the sub-command */ public final String getCommandOption() { return this.command; } - /** Add command commandWithArguments. This method is only meant for use by sub-classes. + /** + * Add command commandWithArguments. This method is only meant for use by sub-classes. + * * @param arguments to be added */ protected final void addCommandArguments(String... arguments) { @@ -43,6 +52,7 @@ protected final void addCommandArguments(String... arguments) { /** * Get the full command. + * * @return the full command */ public String getCommandWithArguments() { diff --git a/storm-server/src/main/java/org/apache/storm/container/docker/DockerExecCommand.java b/storm-server/src/main/java/org/apache/storm/container/docker/DockerExecCommand.java index 9eaa2a48e93..d3f30ec455a 100644 --- a/storm-server/src/main/java/org/apache/storm/container/docker/DockerExecCommand.java +++ b/storm-server/src/main/java/org/apache/storm/container/docker/DockerExecCommand.java @@ -1,12 +1,17 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -31,6 +36,7 @@ public DockerExecCommand(String containerName) { /** * Add the command to run from inside container. + * * @param commandInContainer the command to run from inside container * @return the self */ @@ -41,6 +47,7 @@ public DockerExecCommand addExecCommand(List commandInContainer) { /** * Get the full command. + * * @return the full command */ @Override diff --git a/storm-server/src/main/java/org/apache/storm/container/docker/DockerInspectCommand.java b/storm-server/src/main/java/org/apache/storm/container/docker/DockerInspectCommand.java index 7dcd2c016de..b42122037f2 100644 --- a/storm-server/src/main/java/org/apache/storm/container/docker/DockerInspectCommand.java +++ b/storm-server/src/main/java/org/apache/storm/container/docker/DockerInspectCommand.java @@ -1,12 +1,17 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -35,6 +40,7 @@ public DockerInspectCommand withGettingContainerStatus() { /** * Get the full command. + * * @return the full command. */ @Override diff --git a/storm-server/src/main/java/org/apache/storm/container/docker/DockerManager.java b/storm-server/src/main/java/org/apache/storm/container/docker/DockerManager.java index 988e9512dbf..9ddbfa67100 100644 --- a/storm-server/src/main/java/org/apache/storm/container/docker/DockerManager.java +++ b/storm-server/src/main/java/org/apache/storm/container/docker/DockerManager.java @@ -1,12 +1,17 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -38,7 +43,8 @@ /** * For security, we can launch worker processes inside the docker container. - * This class manages the interaction with docker containers including launching, stopping, profiling and etc. + * This class manages the interaction with docker containers including launching, stopping, + * profiling and etc. */ public class DockerManager extends OciContainerManager { private static final Logger LOG = LoggerFactory.getLogger(DockerManager.class); @@ -80,7 +86,8 @@ public void launchWorkerProcess(String user, String topologyId, Map directory so that jps can work // 2. jstack needs to create a socket under /tmp directory. - //Otherwise profiling will not work properly. + // Otherwise profiling will not work properly. .addReadWriteMountLocation(workerTmpRoot, TMP_DIR, false) - //a list of read-only bind mount locations + // a list of read-only bind mount locations .addAllReadOnlyMountLocations(readonlyBindmounts, false) .addAllReadWriteMountLocations(readwriteBindmounts, false); @@ -152,13 +160,16 @@ public void launchWorkerProcess(String user, String topologyId, Map() { @Override @@ -166,7 +177,8 @@ public Long call() throws IOException { DockerWaitCommand dockerWaitCommand = new DockerWaitCommand(workerId); try { runDockerCommandWaitFor(conf, user, CmdType.RUN_DOCKER_CMD, - dockerWaitCommand.getCommandWithArguments(), null, logPrefix, processExitCallback, targetDir, "docker-wait"); + dockerWaitCommand + .getCommandWithArguments(), null, logPrefix, processExitCallback, targetDir, "docker-wait"); } catch (IOException e) { LOG.error("IOException on running docker wait command:", e); throw e; @@ -177,7 +189,7 @@ public Long call() throws IOException { } - //Get the container ID of the worker + // Get the container ID of the worker private String getContainerId(String workerId) throws IOException { String cid = workerToCid.get(workerId); if (cid == null) { @@ -214,11 +226,13 @@ public long getMemoryUsage(String user, String workerId, int port) throws IOExce public void kill(String user, String workerId) throws IOException { String workerDir = ConfigUtils.workerRoot(conf, workerId); DockerStopCommand dockerStopCommand = new DockerStopCommand(workerId); - runDockerCommandWaitFor(conf, user, CmdType.RUN_DOCKER_CMD, dockerStopCommand.getCommandWithArguments(), + runDockerCommandWaitFor(conf, user, CmdType.RUN_DOCKER_CMD, dockerStopCommand + .getCommandWithArguments(), null, null, null, new File(workerDir), "docker-stop"); DockerRmCommand dockerRmCommand = new DockerRmCommand(workerId); - runDockerCommandWaitFor(conf, user, CmdType.RUN_DOCKER_CMD, dockerRmCommand.getCommandWithArguments(), + runDockerCommandWaitFor(conf, user, CmdType.RUN_DOCKER_CMD, dockerRmCommand + .getCommandWithArguments(), null, null, null, new File(workerDir), "docker-rm"); } @@ -227,14 +241,16 @@ public void forceKill(String user, String workerId) throws IOException { String workerDir = ConfigUtils.workerRoot(conf, workerId); DockerRmCommand dockerRmCommand = new DockerRmCommand(workerId); dockerRmCommand.withForce(); - runDockerCommandWaitFor(conf, user, CmdType.RUN_DOCKER_CMD, dockerRmCommand.getCommandWithArguments(), + runDockerCommandWaitFor(conf, user, CmdType.RUN_DOCKER_CMD, dockerRmCommand + .getCommandWithArguments(), null, null, null, new File(workerDir), "docker-force-rm"); } /** * Currently it only checks if the container is alive. * If the worker process inside the container dies, the container will exit. - * So we only need to check if the container is running to know if the worker process is still alive. + * So we only need to check if the container is running to know if the worker process is still + * alive. * * @param user the user of the processes * @param workerId the id of the worker to kill @@ -260,19 +276,21 @@ public boolean areAllProcessesDead(String user, String workerId) throws IOExcept } if (p.exitValue() != 0) { - String errorMessage = "The exitValue of the docker command [" + command + "] is non-zero: " + p.exitValue(); + String errorMessage = "The exitValue of the docker command [" + command + + "] is non-zero: " + p.exitValue(); LOG.error(errorMessage); throw new IOException(errorMessage); } String output = IOUtils.toString(p.getInputStream(), Charset.forName("UTF-8")); - LOG.debug("The output of the docker command [{}] is: [{}]; the exitValue is {}", command, output, p.exitValue()); - //The output might include some things else - //The real output of the docker-ps command is either empty or the container's short ID + LOG.debug("The output of the docker command [{}] is: [{}]; the exitValue is {}", command, + output, p.exitValue()); + // The output might include some things else + // The real output of the docker-ps command is either empty or the container's short ID output = output.trim(); String[] lines = output.split("\n"); if (lines.length == 0) { - //output is empty, the container is not running + // output is empty, the container is not running return true; } String lastLine = lines[lines.length - 1].trim(); @@ -292,6 +310,7 @@ public boolean areAllProcessesDead(String user, String workerId) throws IOExcept /** * Run profiling command in the container. + * * @param user the user that the worker is running as * @param workerId the id of the worker * @param command the command to run. @@ -304,16 +323,18 @@ public boolean areAllProcessesDead(String user, String workerId) throws IOExcept * @throws InterruptedException if interrupted */ @Override - public boolean runProfilingCommand(String user, String workerId, List command, Map env, + public boolean runProfilingCommand(String user, String workerId, List command, + Map env, String logPrefix, File targetDir) throws IOException, InterruptedException { String workerDir = targetDir.getAbsolutePath(); String profilingArgs = StringUtils.join(command, " "); - //run nsenter + // run nsenter String nsenterScriptPath = writeToCommandFile(workerDir, profilingArgs, "profile"); - List args = Arrays.asList(CmdType.PROFILE_DOCKER_CONTAINER.toString(), workerId, nsenterScriptPath); + List args = Arrays.asList(CmdType.PROFILE_DOCKER_CONTAINER.toString(), workerId, + nsenterScriptPath); Process process = ClientSupervisorUtils.processLauncher( conf, user, null, args, env, logPrefix, null, targetDir @@ -322,7 +343,8 @@ public boolean runProfilingCommand(String user, String workerId, List co process.waitFor(); int exitCode = process.exitValue(); - LOG.debug("WorkerId {} : exitCode from {}: {}", workerId, CmdType.PROFILE_DOCKER_CONTAINER.toString(), exitCode); + LOG.debug("WorkerId {} : exitCode from {}: {}", workerId, CmdType.PROFILE_DOCKER_CONTAINER + .toString(), exitCode); return exitCode == 0; } diff --git a/storm-server/src/main/java/org/apache/storm/container/docker/DockerRmCommand.java b/storm-server/src/main/java/org/apache/storm/container/docker/DockerRmCommand.java index 8e5fe416d60..e062cc44788 100644 --- a/storm-server/src/main/java/org/apache/storm/container/docker/DockerRmCommand.java +++ b/storm-server/src/main/java/org/apache/storm/container/docker/DockerRmCommand.java @@ -1,12 +1,17 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -35,6 +40,7 @@ public DockerRmCommand withForce() { /** * Get the full command. + * * @return the full command */ @Override diff --git a/storm-server/src/main/java/org/apache/storm/container/docker/DockerRunCommand.java b/storm-server/src/main/java/org/apache/storm/container/docker/DockerRunCommand.java index 0e69dc9b6c5..fe1e2a62303 100644 --- a/storm-server/src/main/java/org/apache/storm/container/docker/DockerRunCommand.java +++ b/storm-server/src/main/java/org/apache/storm/container/docker/DockerRunCommand.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -32,6 +38,7 @@ public class DockerRunCommand extends DockerCommand { /** * The Construction function. + * * @param containerName the container name * @param userInfo the info of the user, e.g. "uid:gid" * @param image the container image @@ -44,6 +51,7 @@ public DockerRunCommand(String containerName, String userInfo, String image) { /** * Add --rm option. + * * @return the self */ public DockerRunCommand removeContainerOnExit() { @@ -53,6 +61,7 @@ public DockerRunCommand removeContainerOnExit() { /** * Add -d option. + * * @return the self */ public DockerRunCommand detachOnRun() { @@ -62,6 +71,7 @@ public DockerRunCommand detachOnRun() { /** * Set --workdir option. + * * @param workdir the working directory * @return the self */ @@ -72,6 +82,7 @@ public DockerRunCommand setContainerWorkDir(String workdir) { /** * Set --net option. + * * @param type the network type * @return the self */ @@ -82,6 +93,7 @@ public DockerRunCommand setNetworkType(String type) { /** * Add bind mount locations. + * * @param sourcePath the source path * @param destinationPath the destination path * @param createSource if createSource is false and the source path doesn't exist, do nothing @@ -106,6 +118,7 @@ public DockerRunCommand addReadWriteMountLocation(String sourcePath, String /** * Add all the rw bind mount locations. + * * @param paths the locations * @return the self */ @@ -115,6 +128,7 @@ public DockerRunCommand addAllReadWriteMountLocations(List paths) throws /** * Add all the rw bind mount locations. + * * @param paths the locations * @param createSource if createSource is false and the source path doesn't exist, do nothing * @return the self @@ -129,6 +143,7 @@ public DockerRunCommand addAllReadWriteMountLocations(List paths, /** * Add readonly bind mount location. + * * @param sourcePath the source path * @param destinationPath the destination path * @param createSource if createSource is false and the source path doesn't exist, do nothing @@ -148,6 +163,7 @@ public DockerRunCommand addReadOnlyMountLocation(String sourcePath, String desti /** * Add readonly bind mout location. + * * @param sourcePath the source path * @param destinationPath the destination path * @return the self @@ -159,6 +175,7 @@ public DockerRunCommand addReadOnlyMountLocation(String sourcePath, /** * Add all readonly locations. + * * @param paths the locations * @return the self */ @@ -168,6 +185,7 @@ public DockerRunCommand addAllReadOnlyMountLocations(List paths) throws /** * Add all readonly locations. + * * @param paths the locations * @param createSource if createSource is false and the source path doesn't exist, do nothing * @return the self @@ -193,6 +211,7 @@ public DockerRunCommand addCpuSetBindings(List cores, String memoryNode) /** * Set --cgroup-parent option. + * * @param parentPath the cgroup parent path * @return the self */ @@ -203,6 +222,7 @@ public DockerRunCommand setCGroupParent(String parentPath) { /** * Set --privileged option to run a privileged container. Use with extreme care. + * * @return the self. */ public DockerRunCommand setPrivileged() { @@ -212,14 +232,15 @@ public DockerRunCommand setPrivileged() { /** * Set capabilities of the container. + * * @param capabilities the capabilities to be added * @return the self */ public DockerRunCommand setCapabilities(Set capabilities) { - //first, drop all capabilities + // first, drop all capabilities super.addCommandArguments("--cap-drop=ALL"); - //now, add the capabilities supplied + // now, add the capabilities supplied for (String capability : capabilities) { super.addCommandArguments("--cap-add=" + capability); } @@ -229,6 +250,7 @@ public DockerRunCommand setCapabilities(Set capabilities) { /** * Set --device option. + * * @param sourceDevice the source device * @param destinationDevice the destination device * @return the self @@ -240,6 +262,7 @@ public DockerRunCommand addDevice(String sourceDevice, String destinationDevice) /** * Enable detach. + * * @return the self */ public DockerRunCommand enableDetach() { @@ -249,6 +272,7 @@ public DockerRunCommand enableDetach() { /** * Disable detach. + * * @return the self */ public DockerRunCommand disableDetach() { @@ -258,6 +282,7 @@ public DockerRunCommand disableDetach() { /** * Set --group-add option. + * * @param groups the groups to be added * @return the self */ @@ -270,6 +295,7 @@ public DockerRunCommand groupAdd(String[] groups) { /** * Set extra commands and args. It can override the existing commands. + * * @param overrideCommandWithArgs the extra commands and args * @return the self */ @@ -281,6 +307,7 @@ public DockerRunCommand setOverrideCommandWithArgs( /** * Add --read-only option. + * * @return the self */ public DockerRunCommand setReadonly() { @@ -290,6 +317,7 @@ public DockerRunCommand setReadonly() { /** * Set --security-opt option. + * * @param jsonPath the path to the json file * @return the self */ @@ -300,6 +328,7 @@ public DockerRunCommand setSeccompProfile(String jsonPath) { /** * Set no-new-privileges option. + * * @return the self */ public DockerRunCommand setNoNewPrivileges() { @@ -309,6 +338,7 @@ public DockerRunCommand setNoNewPrivileges() { /** * Set cpuShares. + * * @param cpuShares the cpu shares * @return the self */ @@ -323,6 +353,7 @@ public DockerRunCommand setCpuShares(int cpuShares) { /** * Set the number of cpus to use. + * * @param cpus the number of cpus * @return the self */ @@ -333,6 +364,7 @@ public DockerRunCommand setCpus(double cpus) { /** * Set the number of memory in MB to use. + * * @param memoryMb the number of memory in MB * @return the self */ @@ -343,6 +375,7 @@ public DockerRunCommand setMemoryMb(int memoryMb) { /** * Set the output container id file location. + * * @param cidFile the container id file * @return the self */ @@ -353,6 +386,7 @@ public DockerRunCommand setCidFile(String cidFile) { /** * Get the full command. + * * @return the full command */ @Override diff --git a/storm-server/src/main/java/org/apache/storm/container/docker/DockerStopCommand.java b/storm-server/src/main/java/org/apache/storm/container/docker/DockerStopCommand.java index 09a7f95dfae..9c5af6fb8f8 100644 --- a/storm-server/src/main/java/org/apache/storm/container/docker/DockerStopCommand.java +++ b/storm-server/src/main/java/org/apache/storm/container/docker/DockerStopCommand.java @@ -1,12 +1,17 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -35,6 +40,7 @@ public DockerStopCommand setGracePeriod(int value) { /** * Get the full command. + * * @return the full command */ @Override diff --git a/storm-server/src/main/java/org/apache/storm/container/docker/DockerWaitCommand.java b/storm-server/src/main/java/org/apache/storm/container/docker/DockerWaitCommand.java index 62b0a9b998c..9919a857f8c 100644 --- a/storm-server/src/main/java/org/apache/storm/container/docker/DockerWaitCommand.java +++ b/storm-server/src/main/java/org/apache/storm/container/docker/DockerWaitCommand.java @@ -1,12 +1,17 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -30,6 +35,7 @@ public DockerWaitCommand(String containerName) { /** * Get the full command. + * * @return the full command */ @Override diff --git a/storm-server/src/main/java/org/apache/storm/container/oci/OciContainerExecutorConfig.java b/storm-server/src/main/java/org/apache/storm/container/oci/OciContainerExecutorConfig.java index fedb8365b72..91db01b39d3 100644 --- a/storm-server/src/main/java/org/apache/storm/container/oci/OciContainerExecutorConfig.java +++ b/storm-server/src/main/java/org/apache/storm/container/oci/OciContainerExecutorConfig.java @@ -21,7 +21,6 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonRawValue; - import java.util.List; import java.util.Map; @@ -248,7 +247,6 @@ public String toString() { } } - @JsonInclude(JsonInclude.Include.NON_DEFAULT) static class OciProcessConfig { private final boolean terminal; @@ -453,7 +451,8 @@ static class OciHooksConfig { private final List poststart; private final List poststop; - OciHooksConfig(List prestart, List poststart, List poststop) { + OciHooksConfig(List prestart, List poststart, + List poststop) { this.prestart = prestart; this.poststart = poststart; this.poststop = poststop; diff --git a/storm-server/src/main/java/org/apache/storm/container/oci/OciContainerManager.java b/storm-server/src/main/java/org/apache/storm/container/oci/OciContainerManager.java index 4d5f4c5c1ea..fcc2dd9a2a5 100644 --- a/storm-server/src/main/java/org/apache/storm/container/oci/OciContainerManager.java +++ b/storm-server/src/main/java/org/apache/storm/container/oci/OciContainerManager.java @@ -30,7 +30,6 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; - import org.apache.storm.Config; import org.apache.storm.DaemonConfig; import org.apache.storm.container.ResourceIsolationInterface; @@ -68,9 +67,11 @@ public abstract class OciContainerManager implements ResourceIsolationInterface public void prepare(Map conf) throws IOException { this.conf = conf; - readonlyBindmounts = ObjectReader.getStrings(conf.get(DaemonConfig.STORM_OCI_READONLY_BINDMOUNTS)); + readonlyBindmounts = ObjectReader.getStrings(conf + .get(DaemonConfig.STORM_OCI_READONLY_BINDMOUNTS)); - readwriteBindmounts = ObjectReader.getStrings(conf.get(DaemonConfig.STORM_OCI_READWRITE_BINDMOUNTS)); + readwriteBindmounts = ObjectReader.getStrings(conf + .get(DaemonConfig.STORM_OCI_READWRITE_BINDMOUNTS)); seccompJsonFile = (String) conf.get(DaemonConfig.STORM_OCI_SECCOMP_PROFILE); @@ -84,17 +85,21 @@ public void prepare(Map conf) throws IOException { if (!cgroupParent.startsWith(File.separator)) { cgroupParent = File.separator + cgroupParent; - LOG.warn("{} is not an absolute path. Changing it to be absolute: {}", DaemonConfig.STORM_OCI_CGROUP_PARENT, cgroupParent); + LOG.warn("{} is not an absolute path. Changing it to be absolute: {}", + DaemonConfig.STORM_OCI_CGROUP_PARENT, cgroupParent); } - memoryCgroupRootPath = cgroupRootPath + File.separator + "memory" + File.separator + cgroupParent; + memoryCgroupRootPath = cgroupRootPath + File.separator + "memory" + File.separator + + cgroupParent; memoryCoreAtRoot = new MemoryCore(memoryCgroupRootPath); validatedNumaMap = SupervisorUtils.getNumaMap(conf); } @Override - public void reserveResourcesForWorker(String workerId, Integer workerMemoryMb, Integer workerCpu, String numaId) { - // The manually set STORM_WORKER_CGROUP_CPU_LIMIT config on supervisor will overwrite resources assigned by + public void reserveResourcesForWorker(String workerId, Integer workerMemoryMb, + Integer workerCpu, String numaId) { + // The manually set STORM_WORKER_CGROUP_CPU_LIMIT config on supervisor will overwrite + // resources assigned by // RAS (Resource Aware Scheduler) if (conf.get(DaemonConfig.STORM_WORKER_CGROUP_CPU_LIMIT) != null) { workerCpu = ((Number) conf.get(DaemonConfig.STORM_WORKER_CGROUP_CPU_LIMIT)).intValue(); @@ -127,12 +132,12 @@ public long getSystemFreeMemoryMb() throws IOException { long rootCgroupLimitFree = Long.MAX_VALUE; try { - //For cgroups no limit is max long. + // For cgroups no limit is max long. long limit = memoryCoreAtRoot.getPhysicalUsageLimit(); long used = memoryCoreAtRoot.getMaxPhysicalUsage(); rootCgroupLimitFree = (limit - used) / 1024 / 1024; } catch (FileNotFoundException e) { - //Ignored if cgroups is not setup don't do anything with it + // Ignored if cgroups is not setup don't do anything with it } return Long.min(rootCgroupLimitFree, ServerUtils.getMemInfoFreeMb()); @@ -140,6 +145,7 @@ public long getSystemFreeMemoryMb() throws IOException { /** * Get image name from topology Conf. + * * @param topoConf topology configuration * @return the image name */ @@ -151,7 +157,8 @@ protected String commandFilePath(String dir, String commandTag) { return dir + File.separator + commandTag + ".sh"; } - protected String writeToCommandFile(String workerDir, String command, String commandTag) throws IOException { + protected String writeToCommandFile(String workerDir, String command, + String commandTag) throws IOException { String scriptPath = commandFilePath(workerDir, commandTag); try (BufferedWriter out = new BufferedWriter(new FileWriter(scriptPath))) { out.write(command); diff --git a/storm-server/src/main/java/org/apache/storm/container/oci/OciManifestToResourcesPluginInterface.java b/storm-server/src/main/java/org/apache/storm/container/oci/OciManifestToResourcesPluginInterface.java index b73f1193566..cf83f992fc0 100644 --- a/storm-server/src/main/java/org/apache/storm/container/oci/OciManifestToResourcesPluginInterface.java +++ b/storm-server/src/main/java/org/apache/storm/container/oci/OciManifestToResourcesPluginInterface.java @@ -27,6 +27,7 @@ public interface OciManifestToResourcesPluginInterface { /** * Initialization. + * * @param conf the storm conf * @throws IOException on I/O exception */ @@ -35,6 +36,7 @@ public interface OciManifestToResourcesPluginInterface { /** * Get the layers information from the manifest. * The layers should be returned in the order in which they appear in the manifest + * * @param manifest the manifest of a image * @return a list of layers information * @throws IOException on I/O exception @@ -43,6 +45,7 @@ public interface OciManifestToResourcesPluginInterface { /** * Get the image config information from the manifest. + * * @param manifest the manifest of a image * @return the config of this image * @throws IOException on I/O exception diff --git a/storm-server/src/main/java/org/apache/storm/container/oci/OciResource.java b/storm-server/src/main/java/org/apache/storm/container/oci/OciResource.java index 5aa692c89ab..85357c3749c 100644 --- a/storm-server/src/main/java/org/apache/storm/container/oci/OciResource.java +++ b/storm-server/src/main/java/org/apache/storm/container/oci/OciResource.java @@ -28,13 +28,15 @@ public class OciResource { /** * Constructor. + * * @param path the path to the resource * @param fileName the filename of the resource * @param size the size of the resource * @param timestamp the modification time of the resource * @param type the type of the resource */ - public OciResource(String path, String fileName, long size, long timestamp, OciResourceType type) { + public OciResource(String path, String fileName, long size, long timestamp, + OciResourceType type) { this.path = path; this.fileName = fileName; this.size = size; diff --git a/storm-server/src/main/java/org/apache/storm/container/oci/OciResourcesLocalizerInterface.java b/storm-server/src/main/java/org/apache/storm/container/oci/OciResourcesLocalizerInterface.java index e2527afb758..e70838385ce 100644 --- a/storm-server/src/main/java/org/apache/storm/container/oci/OciResourcesLocalizerInterface.java +++ b/storm-server/src/main/java/org/apache/storm/container/oci/OciResourcesLocalizerInterface.java @@ -30,6 +30,7 @@ public interface OciResourcesLocalizerInterface { /** * Localize the oci resource. + * * @param ociResource the oci resource to be localized * @return the destination of the localized resource. * @throws IOException on I/O exception @@ -38,6 +39,7 @@ public interface OciResourcesLocalizerInterface { /** * Localize a list of oci resources. + * * @param resourceList a list of oci resources. * @return a list of destinations. * @throws IOException on I/O exception diff --git a/storm-server/src/main/java/org/apache/storm/container/oci/OciUtils.java b/storm-server/src/main/java/org/apache/storm/container/oci/OciUtils.java index 9f0b5cf9ee3..184d1c9bff2 100644 --- a/storm-server/src/main/java/org/apache/storm/container/oci/OciUtils.java +++ b/storm-server/src/main/java/org/apache/storm/container/oci/OciUtils.java @@ -36,15 +36,17 @@ public class OciUtils { * Adjust the image config for the topology. * If OCI container is not supported, remove the oci image setting from the topoConf; * otherwise, set it to the default image if it's null. + * * @param conf the daemon conf * @param topoConf the topology conf * @param topoId the topology Id * @throws InvalidTopologyException if image config is invalid */ - public static void adjustImageConfigForTopo(Map conf, Map topoConf, String topoId) + public static void adjustImageConfigForTopo(Map conf, Map topoConf, String topoId) throws InvalidTopologyException { - //don't need sanity check here as we assume it's already done during daemon startup + // don't need sanity check here as we assume it's already done during daemon startup List allowedImages = getAllowedImages(conf, false); String topoImage = (String) topoConf.get(Config.TOPOLOGY_OCI_IMAGE); @@ -57,11 +59,12 @@ public static void adjustImageConfigForTopo(Map conf, Map conf, Map conf) { List allowedImages = getAllowedImages(conf, true); if (allowedImages.isEmpty()) { - LOG.debug("{} is not configured; skip image validation", DaemonConfig.STORM_OCI_ALLOWED_IMAGES); + LOG.debug("{} is not configured; skip image validation", + DaemonConfig.STORM_OCI_ALLOWED_IMAGES); } else { String defaultImage = (String) conf.get(DaemonConfig.STORM_OCI_IMAGE); validateImage(allowedImages, defaultImage, DaemonConfig.STORM_OCI_IMAGE); } } - private static final String OCI_IMAGE_PATTERN = "^(([a-zA-Z0-9.-]+)(:\\d+)?/)?([a-z0-9_./-]+)(:[\\w.-]+)?$"; + private static final String OCI_IMAGE_PATTERN = + "^(([a-zA-Z0-9.-]+)(:\\d+)?/)?([a-z0-9_./-]+)(:[\\w.-]+)?$"; private static final Pattern ociImagePattern = Pattern.compile(OCI_IMAGE_PATTERN); /** - * special case for allowing all images; should only be used in {@link DaemonConfig#STORM_OCI_ALLOWED_IMAGES}. + * Special case for allowing all images; should only be used in {@link + * DaemonConfig#STORM_OCI_ALLOWED_IMAGES}. */ private static final String ASTERISK = "*"; /** * This is a helper function to validate the image. + * * @param allowedImages the allowed image list * @param imageToValidate the image to be validated * @param imageConfigKey the config where this image comes from; this is for logging purpose. */ - private static void validateImage(List allowedImages, String imageToValidate, String imageConfigKey) { + private static void validateImage(List allowedImages, String imageToValidate, + String imageConfigKey) { if (imageToValidate == null) { throw new IllegalArgumentException(imageConfigKey + " is null"); } @@ -118,11 +127,13 @@ private static void validateImage(List allowedImages, String imageToVali } } - private static List getAllowedImages(Map conf, boolean validationEnforced) { - List allowedImages = ObjectReader.getStrings(conf.get(DaemonConfig.STORM_OCI_ALLOWED_IMAGES)); + private static List getAllowedImages(Map conf, + boolean validationEnforced) { + List allowedImages = ObjectReader.getStrings(conf + .get(DaemonConfig.STORM_OCI_ALLOWED_IMAGES)); if (validationEnforced) { - //check if image name matches the required pattern + // check if image name matches the required pattern for (String image : allowedImages) { if (!image.equals(ASTERISK) && !ociImagePattern.matcher(image).matches()) { throw new IllegalArgumentException(image + " in the list of " diff --git a/storm-server/src/main/java/org/apache/storm/container/oci/RuncLibContainerManager.java b/storm-server/src/main/java/org/apache/storm/container/oci/RuncLibContainerManager.java index 033f7542376..62d1eb23822 100644 --- a/storm-server/src/main/java/org/apache/storm/container/oci/RuncLibContainerManager.java +++ b/storm-server/src/main/java/org/apache/storm/container/oci/RuncLibContainerManager.java @@ -39,21 +39,19 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; - import net.minidev.json.JSONObject; import net.minidev.json.parser.JSONParser; import net.minidev.json.parser.ParseException; - import org.apache.commons.lang3.StringUtils; import org.apache.storm.DaemonConfig; import org.apache.storm.StormTimer; import org.apache.storm.container.cgroup.CgroupUtils; import org.apache.storm.container.cgroup.core.MemoryCore; import org.apache.storm.container.oci.OciContainerExecutorConfig.OciLayer; -import org.apache.storm.container.oci.OciContainerExecutorConfig.OciRuntimeConfig; import org.apache.storm.container.oci.OciContainerExecutorConfig.OciRuntimeConfig.OciLinuxConfig; import org.apache.storm.container.oci.OciContainerExecutorConfig.OciRuntimeConfig.OciMount; import org.apache.storm.container.oci.OciContainerExecutorConfig.OciRuntimeConfig.OciProcessConfig; +import org.apache.storm.container.oci.OciContainerExecutorConfig.OciRuntimeConfig; import org.apache.storm.daemon.supervisor.ClientSupervisorUtils; import org.apache.storm.daemon.supervisor.ExitCodeCallback; import org.apache.storm.utils.ConfigUtils; @@ -61,7 +59,6 @@ import org.apache.storm.utils.ReflectionUtils; import org.apache.storm.utils.ServerUtils; import org.apache.storm.utils.Utils; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.yaml.snakeyaml.DumperOptions; @@ -84,7 +81,7 @@ public class RuncLibContainerManager extends OciContainerManager { private static final String SQUASHFS_MEDIA_TYPE = "application/vnd.squashfs"; - //CPU CFS (Completely Fair Scheduler) period + // CPU CFS (Completely Fair Scheduler) period private static final long CPU_CFS_PERIOD_US = 100000; private final Map workerToContainerPid = new ConcurrentHashMap<>(); @@ -118,13 +115,15 @@ public void prepare(Map conf) throws IOException { if (checkContainerAliveTimer == null) { checkContainerAliveTimer = - new StormTimer("CheckRuncContainerAlive", Utils.createDefaultUncaughtExceptionHandler()); + new StormTimer("CheckRuncContainerAlive", Utils + .createDefaultUncaughtExceptionHandler()); checkContainerAliveTimer - .scheduleRecurring(0, (Integer) conf.get(DaemonConfig.SUPERVISOR_MONITOR_FREQUENCY_SECS), () -> { + .scheduleRecurring(0, (Integer) conf + .get(DaemonConfig.SUPERVISOR_MONITOR_FREQUENCY_SECS), () -> { try { checkContainersAlive(); } catch (Exception e) { - //Ignore + // Ignore LOG.warn("The CheckRuncContainerAlive thread has exception. Ignored", e); } }); @@ -156,9 +155,10 @@ private OciResourcesLocalizerInterface chooseOciResourcesLocalizer() return ReflectionUtils.newInstance(pluginName); } - //the container process ID in the process namespace of the host. + // the container process ID in the process namespace of the host. private String containerPidFile(String workerId) { - return ConfigUtils.workerArtifactsSymlink(conf, workerId) + FILE_SEPARATOR + "container-" + workerId + ".pid"; + return ConfigUtils.workerArtifactsSymlink(conf, workerId) + FILE_SEPARATOR + "container-" + + workerId + ".pid"; } @Override @@ -169,18 +169,19 @@ public void launchWorkerProcess(String user, String topologyId, Map layersResource = manifestToResourcesPlugin.getLayerResources(manifest); LOG.info("workerId {}: Got layers metadata: {}", workerId, layersResource.toString()); - //localize resource + // localize resource String configLocalPath = ociResourcesLocalizer.localize(configResource); List ociEnv = new ArrayList<>(); @@ -198,7 +199,7 @@ public void launchWorkerProcess(String user, String topologyId, Map layers = new ArrayList<>(); File file = new File(configLocalPath); - //extract env + // extract env List imageEnv = extractImageEnv(file); if (imageEnv != null && !imageEnv.isEmpty()) { ociEnv.addAll(imageEnv); @@ -208,16 +209,16 @@ public void launchWorkerProcess(String user, String topologyId, Map entrypoint = extractImageEntrypoint(file); if (entrypoint != null && !entrypoint.isEmpty()) { args.addAll(entrypoint); } LOG.debug("workerId {}: args: {}", workerId, args); - //localize layers + // localize layers List layersLocalPath = ociResourcesLocalizer.localize((layersResource)); - //compose layers + // compose layers for (String layerLocalPath : layersLocalPath) { OciLayer layer = new OciLayer(SQUASHFS_MEDIA_TYPE, layerLocalPath); layers.add(layer); @@ -227,7 +228,7 @@ public void launchWorkerProcess(String user, String topologyId, Map/ + // / String workerDir = targetDir.getAbsolutePath(); String workerScriptPath = ServerUtils.writeScript(workerDir, command, env, "0027"); args.add("bash"); args.add(workerScriptPath); - //The container PID (on the host) will be written to this file. + // The container PID (on the host) will be written to this file. String containerPidFilePath = containerPidFile(workerId); OciProcessConfig processConfig = createOciProcessConfig(workerDir, ociEnv, args); OciLinuxConfig linuxConfig = - createOciLinuxConfig(cpusQuotas, memoryInBytes, cgroupParent + "/" + containerId, seccomp, workerId); + createOciLinuxConfig(cpusQuotas, memoryInBytes, cgroupParent + "/" + containerId, + seccomp, workerId); OciRuntimeConfig ociRuntimeConfig = new OciRuntimeConfig(null, mounts, processConfig, null, null, null, linuxConfig); @@ -261,21 +264,27 @@ public void launchWorkerProcess(String user, String topologyId, Map cmdArgs = Arrays.asList(CmdType.RUN_OCI_CONTAINER.toString(), workerDir, executorConfigToJsonFile, + List cmdArgs = Arrays.asList(CmdType.RUN_OCI_CONTAINER.toString(), workerDir, + executorConfigToJsonFile, ConfigUtils.workerArtifactsSymlink(conf, workerId)); - // launch the oci container. waiting prevents possible race condition that could prevent cleanup of container - int exitCode = ClientSupervisorUtils.processLauncherAndWait(conf, user, cmdArgs, env, logPrefix, targetDir); + // launch the oci container. waiting prevents possible race condition that could prevent + // cleanup of container + int exitCode = ClientSupervisorUtils.processLauncherAndWait(conf, user, cmdArgs, env, + logPrefix, targetDir); if (exitCode != 0) { - LOG.error("launchWorkerProcess RuncCommand {} exited with code: {}", "LaunchWorker-" + containerId, exitCode); - throw new RuntimeException("launchWorkerProcess Failed to create Runc Container. ContainerId: " + containerId); + LOG.error("launchWorkerProcess RuncCommand {} exited with code: {}", "LaunchWorker-" + + containerId, exitCode); + throw new RuntimeException("launchWorkerProcess Failed to create Runc Container. " + + "ContainerId: " + containerId); } - //Add to the watched list + // Add to the watched list LOG.debug("Adding {} to the watched workers list", workerId); workerToExitCallback.put(workerId, processExitCallback); workerToUser.put(workerId, user); @@ -283,7 +292,7 @@ public void launchWorkerProcess(String user, String topologyId, Map { if (isContainerDead(workerId, user)) { invokeProcessExitCallback(workerId); @@ -295,12 +304,13 @@ private boolean isContainerDead(String workerId, String user) { boolean isDead = true; Long pid = getContainerPid(workerId); LOG.debug("Checking container {}, pid {}, user {}", workerId, pid, user); - //do nothing if pid is null. + // do nothing if pid is null. if (pid != null && user != null) { try { - isDead = ServerUtils.areAllProcessesDead(conf, user, workerId, Collections.singleton(pid)); + isDead = ServerUtils.areAllProcessesDead(conf, user, workerId, Collections + .singleton(pid)); } catch (IOException e) { - //ignore + // ignore LOG.debug("Error while checking if container is dead.", e); } } @@ -336,7 +346,8 @@ private String getContainerIdFromOciJson(String workerId) throws IOException { } // save runc.yaml in artifacts dir so we can track which image the worker was launched with - private void saveRuncYaml(String topologyId, int port, String containerId, String imageName, OciResource configResource) { + private void saveRuncYaml(String topologyId, int port, String containerId, String imageName, + OciResource configResource) { String fname = String.format("runc-%s.yaml", containerId); File file = new File(ConfigUtils.workerArtifactsRoot(conf, topologyId, port), fname); DumperOptions options = new DumperOptions(); @@ -355,7 +366,8 @@ private void saveRuncYaml(String topologyId, int port, String containerId, Strin } } - private String writeOciExecutorConfigToJsonFile(ObjectMapper mapper, OciContainerExecutorConfig ociContainerExecutorConfig, + private String writeOciExecutorConfigToJsonFile(ObjectMapper mapper, + OciContainerExecutorConfig ociContainerExecutorConfig, String workerDir) throws IOException { File cmdDir = new File(workerDir); if (!cmdDir.exists()) { @@ -367,8 +379,10 @@ private String writeOciExecutorConfigToJsonFile(ObjectMapper mapper, OciContaine return commandFile.getAbsolutePath(); } - private void setContainerMounts(ArrayList mounts, String topologyId, String workerId, Integer port) throws IOException { - //read-only bindmounts need to be added before read-write bindmounts otherwise read-write bindmounts may be overridden. + private void setContainerMounts(ArrayList mounts, String topologyId, String workerId, + Integer port) throws IOException { + // read-only bindmounts need to be added before read-write bindmounts otherwise read-write + // bindmounts may be overridden. for (String readonlyMount : readonlyBindmounts) { addOciMountLocation(mounts, readonlyMount, readonlyMount, false, false); } @@ -384,7 +398,7 @@ private void setContainerMounts(ArrayList mounts, String topologyId, S addOciMountLocation(mounts, stormHome, stormHome, false, false); addOciMountLocation(mounts, cgroupRootPath, cgroupRootPath, false, false); - //set of locations to be bind mounted + // set of locations to be bind mounted String supervisorLocalDir = ConfigUtils.supervisorLocalDir(conf); addOciMountLocation(mounts, supervisorLocalDir, supervisorLocalDir, false, false); @@ -430,7 +444,8 @@ private OciContainerExecutorConfig createOciContainerExecutorConfig( pidFile, containerScriptPath, layers, layersToKeep, ociRuntimeConfig); } - private OciProcessConfig createOciProcessConfig(String cwd, List env, List args) { + private OciProcessConfig createOciProcessConfig(String cwd, List env, + List args) { return new OciProcessConfig(false, null, cwd, env, args, null, null, null, true, 0, null, null); } @@ -489,7 +504,8 @@ public long getMemoryUsage(String user, String workerId, int port) throws IOExce String containerId = getContainerId(workerId, port); String memoryCgroupPath = memoryCgroupRootPath + File.separator + containerId; MemoryCore memoryCore = new MemoryCore(memoryCgroupPath); - LOG.debug("ContainerId {} : Got memory getPhysicalUsage {} from {}", containerId, memoryCore.getPhysicalUsage(), memoryCgroupPath); + LOG.debug("ContainerId {} : Got memory getPhysicalUsage {} from {}", containerId, memoryCore + .getPhysicalUsage(), memoryCgroupPath); return memoryCore.getPhysicalUsage(); } @@ -505,7 +521,8 @@ public void kill(String user, String workerId) throws IOException { } private void signal(long pid, int signal, String user) throws IOException { - List commands = Arrays.asList("signal", String.valueOf(pid), String.valueOf(signal)); + List commands = Arrays.asList("signal", String.valueOf(pid), String + .valueOf(signal)); String logPrefix = "kill -" + signal + " " + pid; ClientSupervisorUtils.processLauncherAndWait(conf, user, commands, null, logPrefix); } @@ -517,7 +534,8 @@ public void forceKill(String user, String workerId) throws IOException { if (pid != null) { signal(pid, 9, user); } else { - LOG.warn("Trying to forceKill container for workerId {} but pidfile is not found", workerId); + LOG.warn("Trying to forceKill container for workerId {} but pidfile is not found", + workerId); } } @@ -543,6 +561,7 @@ private Long getContainerPid(String workerId) { /** * The container terminates if any process inside the container dies. * So we only need to check if the initial process is alive or not. + * * @param user the user that the processes are running as * @param workerId the id of the worker to kill * @return true if all processes are dead; false otherwise @@ -562,18 +581,21 @@ public void cleanup(String user, String workerId, int port) throws IOException { LOG.debug("clean up worker {}", workerId); try { String containerId = getContainerId(workerId, port); - List commands = Arrays.asList(CmdType.REAP_OCI_CONTAINER.toString(), containerId, String.valueOf(layersToKeep)); + List commands = Arrays.asList(CmdType.REAP_OCI_CONTAINER.toString(), + containerId, String.valueOf(layersToKeep)); String logPrefix = "Worker Process " + workerId; - int result = ClientSupervisorUtils.processLauncherAndWait(conf, user, commands, null, logPrefix); + int result = ClientSupervisorUtils.processLauncherAndWait(conf, user, commands, null, + logPrefix); if (result != 0) { LOG.warn("Failed cleaning up RuncWorker {}", workerId); } } catch (FileNotFoundException e) { // This could happen if we had an IOException and failed launching the worker. // We need to continue on in order for the worker directory to get cleaned up. - LOG.error("Failed to find container id for {} ({}), unable to reap container", workerId, e.getMessage()); + LOG.error("Failed to find container id for {} ({}), unable to reap container", workerId, + e.getMessage()); } - //remove from the watched list + // remove from the watched list LOG.debug("Removing {} from the watched workers list", workerId); workerToUser.remove(workerId); workerToExitCallback.remove(workerId); @@ -582,6 +604,7 @@ public void cleanup(String user, String workerId, int port) throws IOException { /** * Run profiling command in the container. + * * @param user the user that the worker is running as * @param workerId the id of the worker * @param command the command to run. @@ -594,13 +617,14 @@ public void cleanup(String user, String workerId, int port) throws IOException { * @throws InterruptedException if interrupted */ @Override - public boolean runProfilingCommand(String user, String workerId, List command, Map env, + public boolean runProfilingCommand(String user, String workerId, List command, + Map env, String logPrefix, File targetDir) throws IOException, InterruptedException { String workerDir = targetDir.getAbsolutePath(); String profilingArgs = StringUtils.join(command, " "); - //run nsenter + // run nsenter String nsenterScriptPath = writeToCommandFile(workerDir, profilingArgs, "profile"); Long containerPid = getContainerPid(workerId); @@ -609,10 +633,13 @@ public boolean runProfilingCommand(String user, String workerId, List co return false; } - List args = Arrays.asList(CmdType.PROFILE_OCI_CONTAINER.toString(), containerPid.toString(), nsenterScriptPath); + List args = Arrays.asList(CmdType.PROFILE_OCI_CONTAINER.toString(), containerPid + .toString(), nsenterScriptPath); - int exitCode = ClientSupervisorUtils.processLauncherAndWait(conf, user, args, env, logPrefix, targetDir); - LOG.debug("WorkerId {} : exitCode from {}: {}", workerId, CmdType.PROFILE_OCI_CONTAINER.toString(), exitCode); + int exitCode = ClientSupervisorUtils.processLauncherAndWait(conf, user, args, env, + logPrefix, targetDir); + LOG.debug("WorkerId {} : exitCode from {}: {}", workerId, CmdType.PROFILE_OCI_CONTAINER + .toString(), exitCode); return exitCode == 0; } diff --git a/storm-server/src/main/java/org/apache/storm/daemon/drpc/BlockingOutstandingRequest.java b/storm-server/src/main/java/org/apache/storm/daemon/drpc/BlockingOutstandingRequest.java index c2395b13443..c79f9f64986 100644 --- a/storm-server/src/main/java/org/apache/storm/daemon/drpc/BlockingOutstandingRequest.java +++ b/storm-server/src/main/java/org/apache/storm/daemon/drpc/BlockingOutstandingRequest.java @@ -25,7 +25,8 @@ import org.apache.storm.utils.WrappedDRPCExecutionException; public class BlockingOutstandingRequest extends OutstandingRequest { - public static final RequestFactory FACTORY = BlockingOutstandingRequest::new; + public static final RequestFactory FACTORY = + BlockingOutstandingRequest::new; private Semaphore sem; private volatile String result = null; private volatile DRPCExecutionException drpcExecutionException = null; @@ -39,7 +40,7 @@ public String getResult() throws DRPCExecutionException { try { sem.acquire(); } catch (InterruptedException e) { - //Ignored + // Ignored } if (result != null) { @@ -47,7 +48,8 @@ public String getResult() throws DRPCExecutionException { } if (drpcExecutionException == null) { - drpcExecutionException = new WrappedDRPCExecutionException("Internal Error: No Result and No Exception"); + drpcExecutionException = + new WrappedDRPCExecutionException("Internal Error: No Result and No Exception"); drpcExecutionException.set_type(DRPCExceptionType.INTERNAL_ERROR); } throw drpcExecutionException; diff --git a/storm-server/src/main/java/org/apache/storm/daemon/drpc/DRPC.java b/storm-server/src/main/java/org/apache/storm/daemon/drpc/DRPC.java index df9ed364f46..1ade59148db 100644 --- a/storm-server/src/main/java/org/apache/storm/daemon/drpc/DRPC.java +++ b/storm-server/src/main/java/org/apache/storm/daemon/drpc/DRPC.java @@ -21,8 +21,8 @@ import com.codahale.metrics.Meter; import java.security.Principal; import java.util.HashMap; -import java.util.Map; import java.util.Map.Entry; +import java.util.Map; import java.util.Timer; import java.util.TimerTask; import java.util.concurrent.ConcurrentHashMap; @@ -51,9 +51,12 @@ public class DRPC implements AutoCloseable { private static final Logger LOG = LoggerFactory.getLogger(DRPC.class); private static final DRPCRequest NOTHING_REQUEST = new DRPCRequest("", ""); - private static final DRPCExecutionException TIMED_OUT = new WrappedDRPCExecutionException("Timed Out"); - private static final DRPCExecutionException SHUT_DOWN = new WrappedDRPCExecutionException("Server Shutting Down"); - private static final DRPCExecutionException DEFAULT_FAILED = new WrappedDRPCExecutionException("Request failed"); + private static final DRPCExecutionException TIMED_OUT = + new WrappedDRPCExecutionException("Timed Out"); + private static final DRPCExecutionException SHUT_DOWN = + new WrappedDRPCExecutionException("Server Shutting Down"); + private static final DRPCExecutionException DEFAULT_FAILED = + new WrappedDRPCExecutionException("Request failed"); private final Meter meterServerTimedOut; private final Meter meterExecuteCalls; @@ -67,10 +70,10 @@ public class DRPC implements AutoCloseable { DEFAULT_FAILED.set_type(DRPCExceptionType.FAILED_REQUEST); } - //Waiting to be fetched + // Waiting to be fetched private final ConcurrentHashMap> queues = new ConcurrentHashMap<>(); - //Waiting to be returned + // Waiting to be returned private final ConcurrentHashMap requests = new ConcurrentHashMap<>(); private final Timer timer = new Timer("DRPC-CLEANUP-TIMER", true); @@ -78,13 +81,15 @@ public class DRPC implements AutoCloseable { private final IAuthorizer auth; public DRPC(StormMetricsRegistry metricsRegistry, Map conf) { - this(metricsRegistry, mkAuthorizationHandler((String) conf.get(DaemonConfig.DRPC_AUTHORIZER), conf), + this(metricsRegistry, mkAuthorizationHandler((String) conf + .get(DaemonConfig.DRPC_AUTHORIZER), conf), ObjectReader.getInt(conf.get(DaemonConfig.DRPC_REQUEST_TIMEOUT_SECS), 600) * 1000); } public DRPC(StormMetricsRegistry metricsRegistry, IAuthorizer auth, long timeoutMs) { this.auth = auth; - this.meterServerTimedOut = metricsRegistry.registerMeter("drpc:num-server-timedout-requests"); + this.meterServerTimedOut = metricsRegistry + .registerMeter("drpc:num-server-timedout-requests"); this.meterExecuteCalls = metricsRegistry.registerMeter("drpc:num-execute-calls"); this.meterResultCalls = metricsRegistry.registerMeter("drpc:num-result-calls"); this.meterFailRequestCalls = metricsRegistry.registerMeter("drpc:num-failRequest-calls"); @@ -110,17 +115,20 @@ private static void logAccess(String operation, String function) { } private static void logAccess(ReqContext reqContext, String operation, String function) { - ThriftAccessLogger.logAccessFunction(reqContext.requestID(), reqContext.remoteAddress(), reqContext.principal(), operation, + ThriftAccessLogger.logAccessFunction(reqContext.requestID(), reqContext.remoteAddress(), + reqContext.principal(), operation, function); } @VisibleForTesting - static void checkAuthorization(ReqContext reqContext, IAuthorizer auth, String operation, String function) + static void checkAuthorization(ReqContext reqContext, IAuthorizer auth, String operation, + String function) throws AuthorizationException { checkAuthorization(reqContext, auth, operation, function, true); } - private static void checkAuthorization(ReqContext reqContext, IAuthorizer auth, String operation, String function, boolean log) + private static void checkAuthorization(ReqContext reqContext, IAuthorizer auth, + String operation, String function, boolean log) throws AuthorizationException { if (reqContext != null && log) { logAccess(reqContext, operation, function); @@ -131,16 +139,19 @@ private static void checkAuthorization(ReqContext reqContext, IAuthorizer auth, if (!auth.permit(reqContext, operation, map)) { Principal principal = reqContext.principal(); String user = (principal != null) ? principal.getName() : "unknown"; - throw new WrappedAuthorizationException("DRPC request '" + operation + "' for '" + user + "' user is not authorized"); + throw new WrappedAuthorizationException("DRPC request '" + operation + "' for '" + + user + "' user is not authorized"); } } } - private void checkAuthorization(String operation, String function) throws AuthorizationException { + private void checkAuthorization(String operation, + String function) throws AuthorizationException { checkAuthorization(ReqContext.context(), auth, operation, function); } - private void checkAuthorizationNoLog(String operation, String function) throws AuthorizationException { + private void checkAuthorizationNoLog(String operation, + String function) throws AuthorizationException { checkAuthorization(ReqContext.context(), auth, operation, function, false); } @@ -151,7 +162,7 @@ private void cleanup(String id) { if (!req.wasFetched()) { queue.remove(req); } - //Drop the queue itself once nothing is waiting in it, otherwise the map keeps an + // Drop the queue itself once nothing is waiting in it, otherwise the map keeps an // entry for every function name a client has ever asked about. return queue.isEmpty() ? null : queue; }); @@ -198,7 +209,7 @@ public DRPCRequest fetchRequest(String functionName) throws AuthorizationExcepti meterFetchRequestCalls.mark(); checkAuthorizationNoLog("fetchRequest", functionName); checkFunctionName(functionName); - //Never create a queue here. A function name comes from the client, so a queue that no one + // Never create a queue here. A function name comes from the client, so a queue that no one // ever puts a request into would stay in the map forever. Poll and drop an emptied queue // under the same lock execute() adds under, so a request can never be left in a queue that // was just removed from the map. @@ -209,7 +220,7 @@ public DRPCRequest fetchRequest(String functionName) throws AuthorizationExcepti }); OutstandingRequest req = polled.get(); if (req != null) { - //Only log accesses that fetched something + // Only log accesses that fetched something logAccess("fetchRequest", functionName); req.fetched(); DRPCRequest ret = req.getRequest(); @@ -231,7 +242,8 @@ public void failRequest(String id, DRPCExecutionException e) throws Authorizatio } } - public T execute(String functionName, String funcArgs, RequestFactory factory) throws + public T execute(String functionName, String funcArgs, + RequestFactory factory) throws AuthorizationException { meterExecuteCalls.mark(); checkAuthorization("execute", functionName); @@ -250,8 +262,10 @@ public T execute(String functionName, String func return req; } - public String executeBlocking(String functionName, String funcArgs) throws DRPCExecutionException, AuthorizationException { - BlockingOutstandingRequest req = execute(functionName, funcArgs, BlockingOutstandingRequest.FACTORY); + public String executeBlocking(String functionName, + String funcArgs) throws DRPCExecutionException, AuthorizationException { + BlockingOutstandingRequest req = execute(functionName, funcArgs, + BlockingOutstandingRequest.FACTORY); try { LOG.debug("Waiting for result {} {}", functionName, funcArgs); return req.getResult(); diff --git a/storm-server/src/main/java/org/apache/storm/daemon/metrics/MetricsUtils.java b/storm-server/src/main/java/org/apache/storm/daemon/metrics/MetricsUtils.java index faec88438b3..2ea55bf163a 100644 --- a/storm-server/src/main/java/org/apache/storm/daemon/metrics/MetricsUtils.java +++ b/storm-server/src/main/java/org/apache/storm/daemon/metrics/MetricsUtils.java @@ -38,7 +38,8 @@ public class MetricsUtils { private static final Logger LOG = LoggerFactory.getLogger(MetricsUtils.class); public static List getPreparableReporters(Map daemonConf) { - List clazzes = (List) daemonConf.get(DaemonConfig.STORM_DAEMON_METRICS_REPORTER_PLUGINS); + List clazzes = (List) daemonConf + .get(DaemonConfig.STORM_DAEMON_METRICS_REPORTER_PLUGINS); List reporterList = new ArrayList<>(); if (clazzes != null) { @@ -63,10 +64,12 @@ private static PreparableReporter getPreparableReporter(String clazz) { public static File getCsvLogDir(Map daemonConf) { String csvMetricsLogDirectory = - ObjectReader.getString(daemonConf.get(DaemonConfig.STORM_DAEMON_METRICS_REPORTER_CSV_LOG_DIR), null); + ObjectReader.getString(daemonConf + .get(DaemonConfig.STORM_DAEMON_METRICS_REPORTER_CSV_LOG_DIR), null); if (csvMetricsLogDirectory == null) { csvMetricsLogDirectory = ConfigUtils.absoluteStormLocalDir(daemonConf); - csvMetricsLogDirectory = csvMetricsLogDirectory + ConfigUtils.FILE_SEPARATOR + "csvmetrics"; + csvMetricsLogDirectory = csvMetricsLogDirectory + ConfigUtils.FILE_SEPARATOR + + "csvmetrics"; } File csvMetricsDir = new File(csvMetricsLogDirectory); validateCreateOutputDir(csvMetricsDir); @@ -86,22 +89,26 @@ private static void validateCreateOutputDir(File dir) { } public static TimeUnit getMetricsRateUnit(Map daemonConf) { - return getTimeUnitForConfig(daemonConf, Config.STORM_DAEMON_METRICS_REPORTER_PLUGIN_RATE_UNIT); + return getTimeUnitForConfig(daemonConf, + Config.STORM_DAEMON_METRICS_REPORTER_PLUGIN_RATE_UNIT); } public static TimeUnit getMetricsDurationUnit(Map daemonConf) { - return getTimeUnitForConfig(daemonConf, Config.STORM_DAEMON_METRICS_REPORTER_PLUGIN_DURATION_UNIT); + return getTimeUnitForConfig(daemonConf, + Config.STORM_DAEMON_METRICS_REPORTER_PLUGIN_DURATION_UNIT); } public static Locale getMetricsReporterLocale(Map daemonConf) { - String languageTag = ObjectReader.getString(daemonConf.get(Config.STORM_DAEMON_METRICS_REPORTER_PLUGIN_LOCALE), null); + String languageTag = ObjectReader.getString(daemonConf + .get(Config.STORM_DAEMON_METRICS_REPORTER_PLUGIN_LOCALE), null); if (languageTag != null) { return Locale.forLanguageTag(languageTag); } return null; } - private static TimeUnit getTimeUnitForConfig(Map daemonConf, String configName) { + private static TimeUnit getTimeUnitForConfig(Map daemonConf, + String configName) { String timeUnitString = ObjectReader.getString(daemonConf.get(configName), null); if (timeUnitString != null) { return TimeUnit.valueOf(timeUnitString); diff --git a/storm-server/src/main/java/org/apache/storm/daemon/metrics/reporters/ConsolePreparableReporter.java b/storm-server/src/main/java/org/apache/storm/daemon/metrics/reporters/ConsolePreparableReporter.java index 8b81128d1aa..426d6399f73 100644 --- a/storm-server/src/main/java/org/apache/storm/daemon/metrics/reporters/ConsolePreparableReporter.java +++ b/storm-server/src/main/java/org/apache/storm/daemon/metrics/reporters/ConsolePreparableReporter.java @@ -1,12 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

      * http://www.apache.org/licenses/LICENSE-2.0 *

      - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -17,7 +22,6 @@ import java.util.Locale; import java.util.Map; import java.util.concurrent.TimeUnit; - import org.apache.storm.DaemonConfig; import org.apache.storm.daemon.metrics.MetricsUtils; import org.apache.storm.utils.ObjectReader; @@ -50,7 +54,8 @@ public void prepare(MetricRegistry metricsRegistry, Map daemonCo builder.convertDurationsTo(durationUnit); } reporter = builder.build(); - reportingIntervalSecs = ObjectReader.getInt(daemonConf.get(DaemonConfig.STORM_DAEMON_METRICS_REPORTER_INTERVAL_SECS), 10); + reportingIntervalSecs = ObjectReader.getInt(daemonConf + .get(DaemonConfig.STORM_DAEMON_METRICS_REPORTER_INTERVAL_SECS), 10); } @Override @@ -59,7 +64,8 @@ public void start() { LOG.debug("Starting..."); reporter.start(reportingIntervalSecs, TimeUnit.SECONDS); } else { - throw new IllegalStateException("Attempt to start without preparing " + getClass().getSimpleName()); + throw new IllegalStateException("Attempt to start without preparing " + getClass() + .getSimpleName()); } } @@ -70,7 +76,8 @@ public void stop() { reporter.report(); reporter.stop(); } else { - throw new IllegalStateException("Attempt to stop without preparing " + getClass().getSimpleName()); + throw new IllegalStateException("Attempt to stop without preparing " + getClass() + .getSimpleName()); } } } diff --git a/storm-server/src/main/java/org/apache/storm/daemon/metrics/reporters/CsvPreparableReporter.java b/storm-server/src/main/java/org/apache/storm/daemon/metrics/reporters/CsvPreparableReporter.java index a58ce19c8ff..4be1a6945b5 100644 --- a/storm-server/src/main/java/org/apache/storm/daemon/metrics/reporters/CsvPreparableReporter.java +++ b/storm-server/src/main/java/org/apache/storm/daemon/metrics/reporters/CsvPreparableReporter.java @@ -1,12 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

      * http://www.apache.org/licenses/LICENSE-2.0 *

      - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -18,7 +23,6 @@ import java.util.Locale; import java.util.Map; import java.util.concurrent.TimeUnit; - import org.apache.storm.DaemonConfig; import org.apache.storm.daemon.metrics.MetricsUtils; import org.apache.storm.utils.ObjectReader; @@ -52,7 +56,8 @@ public void prepare(MetricRegistry metricsRegistry, Map daemonCo File csvMetricsDir = MetricsUtils.getCsvLogDir(daemonConf); reporter = builder.build(csvMetricsDir); - reportingIntervalSecs = ObjectReader.getInt(daemonConf.get(DaemonConfig.STORM_DAEMON_METRICS_REPORTER_INTERVAL_SECS), 10); + reportingIntervalSecs = ObjectReader.getInt(daemonConf + .get(DaemonConfig.STORM_DAEMON_METRICS_REPORTER_INTERVAL_SECS), 10); } @Override @@ -61,7 +66,8 @@ public void start() { LOG.debug("Starting..."); reporter.start(reportingIntervalSecs, TimeUnit.SECONDS); } else { - throw new IllegalStateException("Attempt to start without preparing " + getClass().getSimpleName()); + throw new IllegalStateException("Attempt to start without preparing " + getClass() + .getSimpleName()); } } @@ -72,7 +78,8 @@ public void stop() { reporter.report(); reporter.stop(); } else { - throw new IllegalStateException("Attempt to stop without preparing " + getClass().getSimpleName()); + throw new IllegalStateException("Attempt to stop without preparing " + getClass() + .getSimpleName()); } } diff --git a/storm-server/src/main/java/org/apache/storm/daemon/metrics/reporters/JmxPreparableReporter.java b/storm-server/src/main/java/org/apache/storm/daemon/metrics/reporters/JmxPreparableReporter.java index 718df7ee7ee..c943ea2cbc3 100644 --- a/storm-server/src/main/java/org/apache/storm/daemon/metrics/reporters/JmxPreparableReporter.java +++ b/storm-server/src/main/java/org/apache/storm/daemon/metrics/reporters/JmxPreparableReporter.java @@ -1,12 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

      * http://www.apache.org/licenses/LICENSE-2.0 *

      - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -14,7 +19,6 @@ import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.jmx.JmxReporter; - import java.util.Map; import java.util.concurrent.TimeUnit; import org.apache.storm.DaemonConfig; @@ -31,7 +35,8 @@ public class JmxPreparableReporter implements PreparableReporter { public void prepare(MetricRegistry metricsRegistry, Map daemonConf) { LOG.info("Preparing..."); JmxReporter.Builder builder = JmxReporter.forRegistry(metricsRegistry); - String domain = ObjectReader.getString(daemonConf.get(DaemonConfig.STORM_DAEMON_METRICS_REPORTER_PLUGIN_DOMAIN), null); + String domain = ObjectReader.getString(daemonConf + .get(DaemonConfig.STORM_DAEMON_METRICS_REPORTER_PLUGIN_DOMAIN), null); if (domain != null) { builder.inDomain(domain); } @@ -48,7 +53,8 @@ public void start() { LOG.debug("Starting..."); reporter.start(); } else { - throw new IllegalStateException("Attempt to start without preparing " + getClass().getSimpleName()); + throw new IllegalStateException("Attempt to start without preparing " + getClass() + .getSimpleName()); } } @@ -58,7 +64,8 @@ public void stop() { LOG.debug("Stopping..."); reporter.stop(); } else { - throw new IllegalStateException("Attempt to stop without preparing " + getClass().getSimpleName()); + throw new IllegalStateException("Attempt to stop without preparing " + getClass() + .getSimpleName()); } } } diff --git a/storm-server/src/main/java/org/apache/storm/daemon/metrics/reporters/PreparableReporter.java b/storm-server/src/main/java/org/apache/storm/daemon/metrics/reporters/PreparableReporter.java index e3e6625dc6b..391f4e7c5ec 100644 --- a/storm-server/src/main/java/org/apache/storm/daemon/metrics/reporters/PreparableReporter.java +++ b/storm-server/src/main/java/org/apache/storm/daemon/metrics/reporters/PreparableReporter.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-server/src/main/java/org/apache/storm/daemon/nimbus/HeartbeatCache.java b/storm-server/src/main/java/org/apache/storm/daemon/nimbus/HeartbeatCache.java index 128b06da579..49c5ede10e7 100644 --- a/storm-server/src/main/java/org/apache/storm/daemon/nimbus/HeartbeatCache.java +++ b/storm-server/src/main/java/org/apache/storm/daemon/nimbus/HeartbeatCache.java @@ -27,7 +27,6 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.function.Function; - import org.apache.storm.generated.Assignment; import org.apache.storm.generated.ExecutorInfo; import org.apache.storm.generated.SupervisorWorkerHeartbeat; @@ -52,7 +51,8 @@ private static class ExecutorCache { ExecutorCache(Map newBeat) { if (newBeat != null) { - executorReportedTimeSecs = ((Number) newBeat.getOrDefault(ClientStatsUtil.TIME_SECS, 0L)).longValue(); + executorReportedTimeSecs = ((Number) newBeat.getOrDefault(ClientStatsUtil.TIME_SECS, + 0L)).longValue(); } else { executorReportedTimeSecs = 0L; } @@ -74,7 +74,8 @@ public synchronized void updateTimeout(Integer timeout) { } // Used for RPC heartbeats: nimbusTimeSecs is refreshed on every heartbeat so that - // idle-but-alive executors (whose stats TIME_SECS may not advance) are not falsely timed out. + // idle-but-alive executors (whose stats TIME_SECS may not advance) are not falsely timed + // out. public synchronized void updateFromRpcHb(Integer timeout) { nimbusTimeSecs = Time.currentTimeSecsLong(); updateTimeout(timeout); @@ -84,7 +85,8 @@ public synchronized void updateFromRpcHb(Integer timeout) { // TIME_SECS advances, preserving zombie detection for legacy topologies. public synchronized void updateFromZkHb(Integer timeout, Map newBeat) { if (newBeat != null) { - Long newReportedTime = ((Number) newBeat.getOrDefault(ClientStatsUtil.TIME_SECS, 0L)).longValue(); + Long newReportedTime = ((Number) newBeat.getOrDefault(ClientStatsUtil.TIME_SECS, + 0L)).longValue(); if (!newReportedTime.equals(executorReportedTimeSecs)) { nimbusTimeSecs = Time.currentTimeSecsLong(); } @@ -94,7 +96,7 @@ public synchronized void updateFromZkHb(Integer timeout, Map new } } - //Topology Id -> executor ids -> component -> stats(...) + // Topology Id -> executor ids -> component -> stats(...) private final ConcurrentHashMap, ExecutorCache>> cache; /** @@ -164,7 +166,8 @@ public void timeoutOldHeartbeats(String topoId, Integer taskTimeoutSecs) { * @param allExecutors the executors. * @param timeout the timeout. */ - public void updateFromZkHeartbeat(String topoId, Map, Map> executorBeats, + public void updateFromZkHeartbeat(String topoId, Map, Map> executorBeats, Set> allExecutors, Integer timeout) { Map, ExecutorCache> topoCache = cache.computeIfAbsent(topoId, MAKE_MAP); if (executorBeats == null) { @@ -173,7 +176,8 @@ public void updateFromZkHeartbeat(String topoId, Map, Map executor : allExecutors) { final Map newBeat = executorBeats.get(executor); - ExecutorCache currBeat = topoCache.computeIfAbsent(executor, (k) -> new ExecutorCache(newBeat)); + ExecutorCache currBeat = topoCache.computeIfAbsent(executor, + (k) -> new ExecutorCache(newBeat)); currBeat.updateFromZkHb(timeout, newBeat); } } @@ -184,15 +188,19 @@ public void updateFromZkHeartbeat(String topoId, Map, Map, Map> executorBeats = StatsUtil.convertWorkerBeats(workerHeartbeat); + public void updateHeartbeat(SupervisorWorkerHeartbeat workerHeartbeat, + Integer taskTimeoutSecs) { + Map, Map> executorBeats = StatsUtil + .convertWorkerBeats(workerHeartbeat); String topoId = workerHeartbeat.get_storm_id(); Map, ExecutorCache> topoCache = cache.computeIfAbsent(topoId, MAKE_MAP); for (ExecutorInfo executorInfo : workerHeartbeat.get_executors()) { - List executor = Arrays.asList(executorInfo.get_task_start(), executorInfo.get_task_end()); + List executor = Arrays.asList(executorInfo.get_task_start(), executorInfo + .get_task_end()); final Map newBeat = executorBeats.get(executor); - ExecutorCache currBeat = topoCache.computeIfAbsent(executor, (k) -> new ExecutorCache(newBeat)); + ExecutorCache currBeat = topoCache.computeIfAbsent(executor, + (k) -> new ExecutorCache(newBeat)); currBeat.updateFromRpcHb(taskTimeoutSecs); } } @@ -206,9 +214,11 @@ public void updateHeartbeat(SupervisorWorkerHeartbeat workerHeartbeat, Integer t * @param taskLaunchSecs timeout for right after a worker is launched. * @return the set of tasks that are alive. */ - public Set> getAliveExecutors(String topoId, Set> allExecutors, Assignment assignment, int taskLaunchSecs) { + public Set> getAliveExecutors(String topoId, Set> allExecutors, + Assignment assignment, int taskLaunchSecs) { Map, ExecutorCache> topoCache = cache.computeIfAbsent(topoId, MAKE_MAP); - LOG.debug("Computing alive executors for {}\nExecutors: {}\nAssignment: {}\nHeartbeat cache: {}", + LOG.debug("Computing alive executors for {}\nExecutors: {}\nAssignment: {}\nHeartbeat " + + "cache: {}", topoId, allExecutors, assignment, topoCache); Set> ret = new HashSet<>(); @@ -222,7 +232,7 @@ public Set> getAliveExecutors(String topoId, Set> al Long startTime = execToStartTimes.get(longExec); ExecutorCache executorCache = topoCache.get(exec); - //null isTimedOut means worker never reported any heartbeat + // null isTimedOut means worker never reported any heartbeat boolean isTimedOut = executorCache == null ? true : executorCache.isTimedOut(); Long delta = startTime == null ? null : Time.deltaSecsLong(startTime); if (startTime != null && ((delta < taskLaunchSecs) || !isTimedOut)) { diff --git a/storm-server/src/main/java/org/apache/storm/daemon/nimbus/Nimbus.java b/storm-server/src/main/java/org/apache/storm/daemon/nimbus/Nimbus.java index 75ef22c6a05..d4d7466cd2a 100644 --- a/storm-server/src/main/java/org/apache/storm/daemon/nimbus/Nimbus.java +++ b/storm-server/src/main/java/org/apache/storm/daemon/nimbus/Nimbus.java @@ -26,7 +26,6 @@ import com.codahale.metrics.MetricSet; import com.codahale.metrics.SlidingTimeWindowReservoir; import com.codahale.metrics.Timer; - import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; @@ -51,8 +50,8 @@ import java.util.HashSet; import java.util.Iterator; import java.util.List; -import java.util.Map; import java.util.Map.Entry; +import java.util.Map; import java.util.NavigableMap; import java.util.Set; import java.util.TreeSet; @@ -65,9 +64,7 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; import javax.security.auth.Subject; - import net.minidev.json.JSONValue; - import org.apache.storm.Config; import org.apache.storm.Constants; import org.apache.storm.DaemonConfig; @@ -156,8 +153,8 @@ import org.apache.storm.metric.ClusterMetricsConsumerExecutor; import org.apache.storm.metric.StormMetricsRegistry; import org.apache.storm.metric.api.DataPoint; -import org.apache.storm.metric.api.IClusterMetricsConsumer; import org.apache.storm.metric.api.IClusterMetricsConsumer.ClusterInfo; +import org.apache.storm.metric.api.IClusterMetricsConsumer; import org.apache.storm.metricstore.AggLevel; import org.apache.storm.metricstore.Metric; import org.apache.storm.metricstore.MetricStore; @@ -224,8 +221,8 @@ import org.apache.storm.utils.Time; import org.apache.storm.utils.TimeCacheMap; import org.apache.storm.utils.TupleUtils; -import org.apache.storm.utils.Utils; import org.apache.storm.utils.Utils.UptimeComputer; +import org.apache.storm.utils.Utils; import org.apache.storm.utils.VersionInfo; import org.apache.storm.utils.WrappedAlreadyAliveException; import org.apache.storm.utils.WrappedAuthorizationException; @@ -242,7 +239,8 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon { @VisibleForTesting public static final List ZK_ACLS = Arrays.asList(ZooDefs.Ids.CREATOR_ALL_ACL.get(0)); - public static final SimpleVersion MIN_VERSION_SUPPORT_RPC_HEARTBEAT = new SimpleVersion("2.0.0"); + public static final SimpleVersion MIN_VERSION_SUPPORT_RPC_HEARTBEAT = + new SimpleVersion("2.0.0"); private static final Logger LOG = LoggerFactory.getLogger(Nimbus.class); // Metrics private final Meter submitTopologyWithOptsCalls; @@ -285,10 +283,10 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon { private final Meter mkAssignmentsErrors; private final Meter sendAssignmentExceptions; // used in AssignmentDistributionService.java - //Timer + // Timer private final Timer fileUploadDuration; private final Timer schedulingDuration; - //Scheduler histogram + // Scheduler histogram private final Histogram numAddedExecPerScheduling; private final Histogram numAddedSlotPerScheduling; private final Histogram numRemovedExecPerScheduling; @@ -313,29 +311,37 @@ public static List getNimbusAcls(Map conf) { NIMBUS_SUBJECT.setReadOnly(); } - private static final TopologyStateTransition NOOP_TRANSITION = (arg, nimbus, topoId, base) -> null; - private static final TopologyStateTransition INACTIVE_TRANSITION = (arg, nimbus, topoId, base) -> Nimbus.make(TopologyStatus.INACTIVE); - private static final TopologyStateTransition ACTIVE_TRANSITION = (arg, nimbus, topoId, base) -> Nimbus.make(TopologyStatus.ACTIVE); - private static final TopologyStateTransition REMOVE_TRANSITION = (args, nimbus, topoId, base) -> { + private static final TopologyStateTransition NOOP_TRANSITION = (arg, nimbus, topoId, + base) -> null; + private static final TopologyStateTransition INACTIVE_TRANSITION = (arg, nimbus, topoId, + base) -> Nimbus.make(TopologyStatus.INACTIVE); + private static final TopologyStateTransition ACTIVE_TRANSITION = (arg, nimbus, topoId, + base) -> Nimbus.make(TopologyStatus.ACTIVE); + private static final TopologyStateTransition REMOVE_TRANSITION = (args, nimbus, topoId, + base) -> { LOG.info("Killing topology: {}", topoId); IStormClusterState state = nimbus.getStormClusterState(); Assignment oldAssignment = state.assignmentInfo(topoId, null); state.removeStorm(topoId); - notifySupervisorsAsKilled(state, oldAssignment, nimbus.getAssignmentsDistributer(), nimbus.getMetricsRegistry()); + notifySupervisorsAsKilled(state, oldAssignment, nimbus.getAssignmentsDistributer(), nimbus + .getMetricsRegistry()); nimbus.heartbeatsCache.removeTopo(topoId); nimbus.getIdToExecutors().getAndUpdate(new Dissoc<>(topoId)); return null; }; - private static final TopologyStateTransition DO_REBALANCE_TRANSITION = (args, nimbus, topoId, base) -> { + private static final TopologyStateTransition DO_REBALANCE_TRANSITION = (args, nimbus, topoId, + base) -> { nimbus.doRebalance(topoId, base); return Nimbus.make(base.get_prev_status()); }; - private static final TopologyStateTransition KILL_TRANSITION = (killTime, nimbus, topoId, base) -> { + private static final TopologyStateTransition KILL_TRANSITION = (killTime, nimbus, topoId, + base) -> { int delay = 0; if (killTime != null) { delay = ((Number) killTime).intValue(); } else { - delay = ObjectReader.getInt(Nimbus.readTopoConf(topoId, nimbus.getTopoCache()).get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS)); + delay = ObjectReader.getInt(Nimbus.readTopoConf(topoId, nimbus.getTopoCache()) + .get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS)); } nimbus.delayEvent(topoId, delay, TopologyActions.REMOVE, null); StormBase sb = new StormBase(); @@ -350,13 +356,15 @@ public static List getNimbusAcls(Map conf) { return sb; }; - private static final TopologyStateTransition REBALANCE_TRANSITION = (args, nimbus, topoId, base) -> { + private static final TopologyStateTransition REBALANCE_TRANSITION = (args, nimbus, topoId, + base) -> { RebalanceOptions rbo = ((RebalanceOptions) args).deepCopy(); int delay = 0; if (rbo.is_set_wait_secs()) { delay = rbo.get_wait_secs(); } else { - delay = ObjectReader.getInt(Nimbus.readTopoConf(topoId, nimbus.getTopoCache()).get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS)); + delay = ObjectReader.getInt(Nimbus.readTopoConf(topoId, nimbus.getTopoCache()) + .get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS)); } nimbus.delayEvent(topoId, delay, TopologyActions.DO_REBALANCE, null); @@ -376,46 +384,56 @@ public static List getNimbusAcls(Map conf) { return sb; }; - private static final TopologyStateTransition GAIN_LEADERSHIP_WHEN_KILLED_TRANSITION = (args, nimbus, topoId, base) -> { + private static final TopologyStateTransition GAIN_LEADERSHIP_WHEN_KILLED_TRANSITION = (args, + nimbus, topoId, base) -> { int delay = base.get_topology_action_options().get_kill_options().get_wait_secs(); nimbus.delayEvent(topoId, delay, TopologyActions.REMOVE, null); return null; }; - private static final TopologyStateTransition GAIN_LEADERSHIP_WHEN_REBALANCING_TRANSITION = (args, nimbus, topoId, base) -> { - int delay = base.get_topology_action_options().get_rebalance_options().get_wait_secs(); - nimbus.delayEvent(topoId, delay, TopologyActions.DO_REBALANCE, null); - return null; - }; + private static final TopologyStateTransition GAIN_LEADERSHIP_WHEN_REBALANCING_TRANSITION = + (args, nimbus, topoId, base) -> { + int delay = base.get_topology_action_options().get_rebalance_options().get_wait_secs(); + nimbus.delayEvent(topoId, delay, TopologyActions.DO_REBALANCE, null); + return null; + }; private static final Map> TOPO_STATE_TRANSITIONS = new ImmutableMap.Builder>() - .put(TopologyStatus.ACTIVE, new ImmutableMap.Builder() + .put(TopologyStatus.ACTIVE, new ImmutableMap.Builder() .put(TopologyActions.INACTIVATE, INACTIVE_TRANSITION) .put(TopologyActions.ACTIVATE, NOOP_TRANSITION) .put(TopologyActions.REBALANCE, REBALANCE_TRANSITION) .put(TopologyActions.KILL, KILL_TRANSITION) .build()) - .put(TopologyStatus.INACTIVE, new ImmutableMap.Builder() + .put(TopologyStatus.INACTIVE, new ImmutableMap.Builder() .put(TopologyActions.ACTIVATE, ACTIVE_TRANSITION) .put(TopologyActions.INACTIVATE, NOOP_TRANSITION) .put(TopologyActions.REBALANCE, REBALANCE_TRANSITION) .put(TopologyActions.KILL, KILL_TRANSITION) .build()) - .put(TopologyStatus.KILLED, new ImmutableMap.Builder() + .put(TopologyStatus.KILLED, new ImmutableMap.Builder() .put(TopologyActions.GAIN_LEADERSHIP, GAIN_LEADERSHIP_WHEN_KILLED_TRANSITION) .put(TopologyActions.KILL, KILL_TRANSITION) .put(TopologyActions.REMOVE, REMOVE_TRANSITION) .build()) - .put(TopologyStatus.REBALANCING, new ImmutableMap.Builder() + .put(TopologyStatus.REBALANCING, new ImmutableMap.Builder() .put(TopologyActions.GAIN_LEADERSHIP, GAIN_LEADERSHIP_WHEN_REBALANCING_TRANSITION) .put(TopologyActions.KILL, KILL_TRANSITION) .put(TopologyActions.DO_REBALANCE, DO_REBALANCE_TRANSITION) .build()) .build(); - private static final List EMPTY_STRING_LIST = Collections.unmodifiableList(Collections.emptyList()); - private static final Set EMPTY_STRING_SET = Collections.unmodifiableSet(Collections.emptySet()); - //A dependency blob key whose file name part ends with a canonical UUID, which a client splices in per upload. + private static final List EMPTY_STRING_LIST = Collections.unmodifiableList(Collections + .emptyList()); + private static final Set EMPTY_STRING_SET = Collections.unmodifiableSet(Collections + .emptySet()); + // A dependency blob key whose file name part ends with a canonical UUID, which a client splices + // in per upload. private static final Pattern UNIQUE_DEPENDENCY_KEY = Pattern.compile( - "^dep-.+-[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}(\\..+)?$"); + "^dep-.+-[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}(\\..+" + + ")?$"); private static final RotatingMap topologyCleanupDetected = new RotatingMap<>(2); private static long topologyCleanupRotationTime = 0L; @@ -440,7 +458,8 @@ public static List getNimbusAcls(Map conf) { private final TimeCacheMap uploaders; private final BlobStore blobStore; private final TopoCache topoCache; - //When a dependency blob was first seen unreferenced by any topology, only used by the cleanup pass. + // When a dependency blob was first seen unreferenced by any topology, only used by the cleanup + // pass. private final Map orphanedDependencyKeysDetectedMs = new HashMap<>(); @SuppressWarnings("deprecation") private final TimeCacheMap blobDownloaders; @@ -454,7 +473,7 @@ public static List getNimbusAcls(Map conf) { private final StormTimer cleanupTimer; private final IScheduler scheduler; private final IScheduler underlyingScheduler; - //Metrics related + // Metrics related private final AtomicReference schedulingStartTimeNs = new AtomicReference<>(null); private final AtomicLong longestSchedulingTime = new AtomicLong(); @@ -477,87 +496,119 @@ public static List getNimbusAcls(Map conf) { private final ClusterSummaryMetricSet clusterMetricSet; private MetricStore metricsStore; private IAuthorizer authorizationHandler; - //Cached CuratorFramework, mainly used for BlobStore. + // Cached CuratorFramework, mainly used for BlobStore. private final CuratorFramework zkClient; - //Cached topology -> executor ids, used for deciding timeout workers of heartbeatsCache. + // Cached topology -> executor ids, used for deciding timeout workers of heartbeatsCache. private AtomicReference>>> idToExecutors; - //May be null if worker tokens are not supported by the thrift transport. + // May be null if worker tokens are not supported by the thrift transport. private WorkerTokenManager workerTokenManager; private boolean wasLeader = false; - public Nimbus(Map conf, INimbus inimbus, StormMetricsRegistry metricsRegistry) throws Exception { + public Nimbus(Map conf, INimbus inimbus, + StormMetricsRegistry metricsRegistry) throws Exception { this(conf, inimbus, null, null, null, null, null, metricsRegistry); } - public Nimbus(Map conf, INimbus inimbus, IStormClusterState stormClusterState, NimbusInfo hostPortInfo, + public Nimbus(Map conf, INimbus inimbus, IStormClusterState stormClusterState, + NimbusInfo hostPortInfo, BlobStore blobStore, ILeaderElector leaderElector, IGroupMappingServiceProvider groupMapper, StormMetricsRegistry metricsRegistry) throws Exception { - this(conf, inimbus, stormClusterState, hostPortInfo, blobStore, null, leaderElector, groupMapper, metricsRegistry); + this(conf, inimbus, stormClusterState, hostPortInfo, blobStore, null, leaderElector, + groupMapper, metricsRegistry); } - public Nimbus(Map conf, INimbus inimbus, IStormClusterState stormClusterState, NimbusInfo hostPortInfo, + public Nimbus(Map conf, INimbus inimbus, IStormClusterState stormClusterState, + NimbusInfo hostPortInfo, BlobStore blobStore, TopoCache topoCache, ILeaderElector leaderElector, IGroupMappingServiceProvider groupMapper, StormMetricsRegistry metricsRegistry) throws Exception { this.conf = conf; this.metricsRegistry = metricsRegistry; this.resourceMetrics = new ResourceMetrics(metricsRegistry); - this.submitTopologyWithOptsCalls = metricsRegistry.registerMeter("nimbus:num-submitTopologyWithOpts-calls"); + this.submitTopologyWithOptsCalls = metricsRegistry + .registerMeter("nimbus:num-submitTopologyWithOpts-calls"); this.submitTopologyCalls = metricsRegistry.registerMeter("nimbus:num-submitTopology-calls"); - this.killTopologyWithOptsCalls = metricsRegistry.registerMeter("nimbus:num-killTopologyWithOpts-calls"); + this.killTopologyWithOptsCalls = metricsRegistry + .registerMeter("nimbus:num-killTopologyWithOpts-calls"); this.killTopologyCalls = metricsRegistry.registerMeter("nimbus:num-killTopology-calls"); this.rebalanceCalls = metricsRegistry.registerMeter("nimbus:num-rebalance-calls"); this.activateCalls = metricsRegistry.registerMeter("nimbus:num-activate-calls"); this.deactivateCalls = metricsRegistry.registerMeter("nimbus:num-deactivate-calls"); this.debugCalls = metricsRegistry.registerMeter("nimbus:num-debug-calls"); - this.setWorkerProfilerCalls = metricsRegistry.registerMeter("nimbus:num-setWorkerProfiler-calls"); + this.setWorkerProfilerCalls = metricsRegistry + .registerMeter("nimbus:num-setWorkerProfiler-calls"); this.getComponentPendingProfileActionsCalls = metricsRegistry.registerMeter( "nimbus:num-getComponentPendingProfileActions-calls"); this.setLogConfigCalls = metricsRegistry.registerMeter("nimbus:num-setLogConfig-calls"); - this.uploadNewCredentialsCalls = metricsRegistry.registerMeter("nimbus:num-uploadNewCredentials-calls"); - this.beginFileUploadCalls = metricsRegistry.registerMeter("nimbus:num-beginFileUpload-calls"); + this.uploadNewCredentialsCalls = metricsRegistry + .registerMeter("nimbus:num-uploadNewCredentials-calls"); + this.beginFileUploadCalls = metricsRegistry + .registerMeter("nimbus:num-beginFileUpload-calls"); this.uploadChunkCalls = metricsRegistry.registerMeter("nimbus:num-uploadChunk-calls"); - this.finishFileUploadCalls = metricsRegistry.registerMeter("nimbus:num-finishFileUpload-calls"); + this.finishFileUploadCalls = metricsRegistry + .registerMeter("nimbus:num-finishFileUpload-calls"); this.downloadChunkCalls = metricsRegistry.registerMeter("nimbus:num-downloadChunk-calls"); this.getNimbusConfCalls = metricsRegistry.registerMeter("nimbus:num-getNimbusConf-calls"); this.getLogConfigCalls = metricsRegistry.registerMeter("nimbus:num-getLogConfig-calls"); - this.getTopologyConfCalls = metricsRegistry.registerMeter("nimbus:num-getTopologyConf-calls"); + this.getTopologyConfCalls = metricsRegistry + .registerMeter("nimbus:num-getTopologyConf-calls"); this.getTopologyCalls = metricsRegistry.registerMeter("nimbus:num-getTopology-calls"); - this.getUserTopologyCalls = metricsRegistry.registerMeter("nimbus:num-getUserTopology-calls"); + this.getUserTopologyCalls = metricsRegistry + .registerMeter("nimbus:num-getUserTopology-calls"); this.getClusterInfoCalls = metricsRegistry.registerMeter("nimbus:num-getClusterInfo-calls"); - this.getTopologySummariesCalls = metricsRegistry.registerMeter("nimbus:num-getTopologySummaries-calls"); - this.getTopologySummaryCalls = metricsRegistry.registerMeter("nimbus:num-getTopologySummary-calls"); - this.getTopologySummaryByNameCalls = metricsRegistry.registerMeter("nimbus:num-getTopologySummaryByName-calls"); + this.getTopologySummariesCalls = metricsRegistry + .registerMeter("nimbus:num-getTopologySummaries-calls"); + this.getTopologySummaryCalls = metricsRegistry + .registerMeter("nimbus:num-getTopologySummary-calls"); + this.getTopologySummaryByNameCalls = metricsRegistry + .registerMeter("nimbus:num-getTopologySummaryByName-calls"); this.getLeaderCalls = metricsRegistry.registerMeter("nimbus:num-getLeader-calls"); - this.isTopologyNameAllowedCalls = metricsRegistry.registerMeter("nimbus:num-isTopologyNameAllowed-calls"); + this.isTopologyNameAllowedCalls = metricsRegistry + .registerMeter("nimbus:num-isTopologyNameAllowed-calls"); this.getTopologyInfoWithOptsCalls = metricsRegistry.registerMeter( "nimbus:num-getTopologyInfoWithOpts-calls"); - this.getTopologyInfoCalls = metricsRegistry.registerMeter("nimbus:num-getTopologyInfo-calls"); - this.getTopologyInfoByNameCalls = metricsRegistry.registerMeter("nimbus:num-getTopologyInfoByName-calls"); - this.getTopologyInfoByNameWithOptsCalls = metricsRegistry.registerMeter("nimbus:num-getTopologyInfoByNameWithOpts-calls"); - this.getTopologyPageInfoCalls = metricsRegistry.registerMeter("nimbus:num-getTopologyPageInfo-calls"); - this.getSupervisorPageInfoCalls = metricsRegistry.registerMeter("nimbus:num-getSupervisorPageInfo-calls"); - this.getComponentPageInfoCalls = metricsRegistry.registerMeter("nimbus:num-getComponentPageInfo-calls"); + this.getTopologyInfoCalls = metricsRegistry + .registerMeter("nimbus:num-getTopologyInfo-calls"); + this.getTopologyInfoByNameCalls = metricsRegistry + .registerMeter("nimbus:num-getTopologyInfoByName-calls"); + this.getTopologyInfoByNameWithOptsCalls = metricsRegistry + .registerMeter("nimbus:num-getTopologyInfoByNameWithOpts-calls"); + this.getTopologyPageInfoCalls = metricsRegistry + .registerMeter("nimbus:num-getTopologyPageInfo-calls"); + this.getSupervisorPageInfoCalls = metricsRegistry + .registerMeter("nimbus:num-getSupervisorPageInfo-calls"); + this.getComponentPageInfoCalls = metricsRegistry + .registerMeter("nimbus:num-getComponentPageInfo-calls"); this.getOwnerResourceSummariesCalls = metricsRegistry.registerMeter( "nimbus:num-getOwnerResourceSummaries-calls"); this.shutdownCalls = metricsRegistry.registerMeter("nimbus:num-shutdown-calls"); - this.processWorkerMetricsCalls = metricsRegistry.registerMeter("nimbus:process-worker-metric-calls"); + this.processWorkerMetricsCalls = metricsRegistry + .registerMeter("nimbus:process-worker-metric-calls"); this.mkAssignmentsErrors = metricsRegistry.registerMeter("nimbus:mkAssignments-Errors"); - this.sendAssignmentExceptions = metricsRegistry.registerMeter(Constants.NIMBUS_SEND_ASSIGNMENT_EXCEPTIONS); + this.sendAssignmentExceptions = metricsRegistry + .registerMeter(Constants.NIMBUS_SEND_ASSIGNMENT_EXCEPTIONS); this.fileUploadDuration = metricsRegistry.registerTimer("nimbus:files-upload-duration-ms"); - this.schedulingDuration = metricsRegistry.registerTimer("nimbus:topology-scheduling-duration-ms"); - this.numAddedExecPerScheduling = metricsRegistry.registerHistogram("nimbus:num-added-executors-per-scheduling"); - this.numAddedSlotPerScheduling = metricsRegistry.registerHistogram("nimbus:num-added-slots-per-scheduling"); - this.numRemovedExecPerScheduling = metricsRegistry.registerHistogram("nimbus:num-removed-executors-per-scheduling"); - this.numRemovedSlotPerScheduling = metricsRegistry.registerHistogram("nimbus:num-removed-slots-per-scheduling"); - this.numNetExecIncreasePerScheduling = metricsRegistry.registerHistogram("nimbus:num-net-executors-increase-per-scheduling"); - this.numNetSlotIncreasePerScheduling = metricsRegistry.registerHistogram("nimbus:num-net-slots-increase-per-scheduling"); + this.schedulingDuration = metricsRegistry + .registerTimer("nimbus:topology-scheduling-duration-ms"); + this.numAddedExecPerScheduling = metricsRegistry + .registerHistogram("nimbus:num-added-executors-per-scheduling"); + this.numAddedSlotPerScheduling = metricsRegistry + .registerHistogram("nimbus:num-added-slots-per-scheduling"); + this.numRemovedExecPerScheduling = metricsRegistry + .registerHistogram("nimbus:num-removed-executors-per-scheduling"); + this.numRemovedSlotPerScheduling = metricsRegistry + .registerHistogram("nimbus:num-removed-slots-per-scheduling"); + this.numNetExecIncreasePerScheduling = metricsRegistry + .registerHistogram("nimbus:num-net-executors-increase-per-scheduling"); + this.numNetSlotIncreasePerScheduling = metricsRegistry + .registerHistogram("nimbus:num-net-slots-increase-per-scheduling"); this.metricsStore = null; try { this.metricsStore = MetricStoreConfig.configure(conf, metricsRegistry); } catch (Exception e) { - // the metrics store is not critical to the operation of the cluster, allow Nimbus to come up + // the metrics store is not critical to the operation of the cluster, allow Nimbus to + // come up LOG.error("Failed to initialize metric store", e); } @@ -570,9 +621,11 @@ public Nimbus(Map conf, INimbus inimbus, IStormClusterState stor } this.inimbus = inimbus; - this.authorizationHandler = StormCommon.mkAuthorizationHandler((String) conf.get(DaemonConfig.NIMBUS_AUTHORIZER), conf); + this.authorizationHandler = StormCommon.mkAuthorizationHandler((String) conf + .get(DaemonConfig.NIMBUS_AUTHORIZER), conf); this.impersonationAuthorizationHandler = - StormCommon.mkAuthorizationHandler((String) conf.get(DaemonConfig.NIMBUS_IMPERSONATION_AUTHORIZER), conf); + StormCommon.mkAuthorizationHandler((String) conf + .get(DaemonConfig.NIMBUS_IMPERSONATION_AUTHORIZER), conf); this.submittedCount = new AtomicLong(0); if (stormClusterState == null) { stormClusterState = makeStormClusterState(conf); @@ -588,7 +641,8 @@ public Nimbus(Map conf, INimbus inimbus, IStormClusterState stor this.blobListers = makeBlobListCacheMap(conf); this.uptime = Utils.makeUptimeComputer(); this.validator = ReflectionUtils - .newInstance((String) conf.getOrDefault(DaemonConfig.NIMBUS_TOPOLOGY_VALIDATOR, DefaultTopologyValidator.class.getName())); + .newInstance((String) conf.getOrDefault(DaemonConfig.NIMBUS_TOPOLOGY_VALIDATOR, + DefaultTopologyValidator.class.getName())); this.timer = new StormTimer(null, (t, e) -> { LOG.error("Error while processing event", e); Utils.exitProcess(20, "Error while processing event"); @@ -611,14 +665,16 @@ public Nimbus(Map conf, INimbus inimbus, IStormClusterState stor topoCache = new TopoCache(blobStore, conf); } if (leaderElector == null) { - leaderElector = Zookeeper.zkLeaderElector(conf, zkClient, blobStore, topoCache, stormClusterState, getNimbusAcls(conf), + leaderElector = Zookeeper.zkLeaderElector(conf, zkClient, blobStore, topoCache, + stormClusterState, getNimbusAcls(conf), metricsRegistry, submitLock); } this.leaderElector = leaderElector; this.blobStore.setLeaderElector(this.leaderElector); this.topoCache = topoCache; - this.assignmentsDistributer = AssignmentDistributionService.getInstance(conf, this.scheduler); + this.assignmentsDistributer = AssignmentDistributionService.getInstance(conf, + this.scheduler); this.idToSchedStatus = new AtomicReference<>(new HashMap<>()); this.nodeIdToResources = new AtomicReference<>(new HashMap<>()); this.idToResources = new AtomicReference<>(new HashMap<>()); @@ -635,7 +691,8 @@ public Nimbus(Map conf, INimbus inimbus, IStormClusterState stor this.groupMapper = groupMapper; this.principalToLocal = ClientAuthUtils.getPrincipalToLocalPlugin(conf); // We don't use the classpath part of this, so just an empty list - this.supervisorClasspaths = Collections.unmodifiableNavigableMap(Utils.getConfiguredClasspathVersions(conf, EMPTY_STRING_LIST)); + this.supervisorClasspaths = Collections.unmodifiableNavigableMap(Utils + .getConfiguredClasspathVersions(conf, EMPTY_STRING_LIST)); clusterMetricSet = new ClusterSummaryMetricSet(metricsRegistry); } @@ -643,15 +700,17 @@ public Nimbus(Map conf, INimbus inimbus, IStormClusterState stor private static StormBase make(TopologyStatus status) { StormBase ret = new StormBase(); ret.set_status(status); - //The following are required for backwards compatibility with clojure code + // The following are required for backwards compatibility with clojure code ret.set_component_executors(Collections.emptyMap()); ret.set_component_debug(Collections.emptyMap()); return ret; } @SuppressWarnings("deprecation") - private static TimeCacheMap fileCacheMap(Map conf) { - return new TimeCacheMap<>(ObjectReader.getInt(conf.get(DaemonConfig.NIMBUS_FILE_COPY_EXPIRATION_SECS), 600), + private static TimeCacheMap fileCacheMap(Map conf) { + return new TimeCacheMap<>(ObjectReader.getInt(conf + .get(DaemonConfig.NIMBUS_FILE_COPY_EXPIRATION_SECS), 600), (id, stream) -> { try { stream.close(); @@ -661,8 +720,9 @@ private static TimeCacheMap fileCacheMap(Ma }); } - //Not symmetric difference. Performing A.entrySet() - B.entrySet() - private static Map mapDiff(Map first, Map second) { + // Not symmetric difference. Performing A.entrySet() - B.entrySet() + private static Map mapDiff(Map first, Map second) { Map ret = new HashMap<>(); for (Entry entry : second.entrySet()) { if (!entry.getValue().equals(first.get(entry.getKey()))) { @@ -672,7 +732,8 @@ private static Map mapDiff(Map first, Map return ret; } - private static IScheduler wrapAsBlacklistScheduler(Map conf, IScheduler scheduler, + private static IScheduler wrapAsBlacklistScheduler(Map conf, + IScheduler scheduler, StormMetricsRegistry metricsRegistry) { BlacklistScheduler blacklistWrappedScheduler = new BlacklistScheduler(scheduler); blacklistWrappedScheduler.prepare(conf, metricsRegistry); @@ -695,15 +756,18 @@ private static IScheduler makeScheduler(Map conf, INimbus inimbu } /** - * Constructs a TimeCacheMap instance with a blob store timeout whose expiration callback invokes cancel on the value held by an expired + * Constructs a TimeCacheMap instance with a blob store timeout whose expiration callback + * invokes cancel on the value held by an expired * entry when that value is an AtomicOutputStream and calls close otherwise. * * @param conf the config to use * @return the newly created map */ @SuppressWarnings("deprecation") - private static TimeCacheMap makeBlobCacheMap(Map conf) { - return new TimeCacheMap<>(ObjectReader.getInt(conf.get(DaemonConfig.NIMBUS_BLOBSTORE_EXPIRATION_SECS), 600), + private static TimeCacheMap makeBlobCacheMap(Map conf) { + return new TimeCacheMap<>(ObjectReader.getInt(conf + .get(DaemonConfig.NIMBUS_BLOBSTORE_EXPIRATION_SECS), 600), (id, stream) -> { try { if (stream instanceof AtomicOutputStream) { @@ -724,11 +788,14 @@ private static TimeCacheMap makeBlobCacheMa * @return the newly created TimeCacheMap */ @SuppressWarnings("deprecation") - private static TimeCacheMap> makeBlobListCacheMap(Map conf) { - return new TimeCacheMap<>(ObjectReader.getInt(conf.get(DaemonConfig.NIMBUS_BLOBSTORE_EXPIRATION_SECS), 600)); + private static TimeCacheMap> makeBlobListCacheMap(Map conf) { + return new TimeCacheMap<>(ObjectReader.getInt(conf + .get(DaemonConfig.NIMBUS_BLOBSTORE_EXPIRATION_SECS), 600)); } - private static ITopologyActionNotifierPlugin createTopologyActionNotifier(Map conf) { + private static ITopologyActionNotifierPlugin createTopologyActionNotifier(Map conf) { String clazz = (String) conf.get(DaemonConfig.NIMBUS_TOPOLOGY_ACTION_NOTIFIER_PLUGIN); ITopologyActionNotifierPlugin ret = null; if (clazz != null && !clazz.isEmpty()) { @@ -750,7 +817,8 @@ private static List makeClusterMetricsConsumerEx List ret = new ArrayList<>(); if (consumers != null) { for (Map consumer : consumers) { - ret.add(new ClusterMetricsConsumerExecutor((String) consumer.get("class"), consumer.get("argument"))); + ret.add(new ClusterMetricsConsumerExecutor((String) consumer.get("class"), consumer + .get("argument"))); } } return ret; @@ -760,7 +828,8 @@ private static Subject getSubject() { return ReqContext.context().subject(); } - static Map readTopoConf(String topoId, TopoCache tc) throws KeyNotFoundException, + static Map readTopoConf(String topoId, + TopoCache tc) throws KeyNotFoundException, AuthorizationException, IOException { return tc.readTopoConf(topoId, getSubject()); } @@ -791,26 +860,33 @@ public static int getVersionForKey(String key, NimbusInfo nimbusInfo, return kseq.getKeySequenceNumber(zkClient, mayCreateKey); } - private static StormTopology readStormTopology(String topoId, TopoCache tc) throws KeyNotFoundException, AuthorizationException, + private static StormTopology readStormTopology(String topoId, + TopoCache tc) throws KeyNotFoundException, AuthorizationException, IOException { return tc.readTopology(topoId, getSubject()); } - private static Map readTopoConfAsNimbus(String topoId, TopoCache tc) throws KeyNotFoundException, + private static Map readTopoConfAsNimbus(String topoId, + TopoCache tc) throws KeyNotFoundException, AuthorizationException, IOException { return tc.readTopoConf(topoId, NIMBUS_SUBJECT); } - private static StormTopology readStormTopologyAsNimbus(String topoId, TopoCache tc) throws KeyNotFoundException, + private static StormTopology readStormTopologyAsNimbus(String topoId, + TopoCache tc) throws KeyNotFoundException, AuthorizationException, IOException { return tc.readTopology(topoId, NIMBUS_SUBJECT); } /** - * Mask the credential values in a config map that is about to be serialized to a client. Nimbus serves config over - * several read-only operations, and a caller authorized for those is not necessarily authorized to hold the cluster's - * or the topology's secrets. This covers what {@link ConfigUtils#maskCredentials(Map)} covers, plus the Blowfish - * tuple-serializer key, whose constant lives outside the config classes so the annotation scan cannot see it and + * Mask the credential values in a config map that is about to be serialized to a client. Nimbus + * serves config over + * several read-only operations, and a caller authorized for those is not necessarily authorized + * to hold the cluster's + * or the topology's secrets. This covers what {@link ConfigUtils#maskCredentials(Map)} covers, + * plus the Blowfish + * tuple-serializer key, whose constant lives outside the config classes so the annotation scan + * cannot see it and * whose name matches no credential pattern. * * @param conf the config about to be served @@ -825,7 +901,7 @@ private static Map maskCredentialsForApi(Map con } /** - * convert {topology-id -> SchedulerAssignment} to {topology-id -> {executor [node port]}}. + * Convert {topology-id -> SchedulerAssignment} to {topology-id -> {executor [node port]}}. * * @return {topology-id -> {executor [node port]}} mapping */ @@ -834,7 +910,8 @@ private static Map, List>> computeTopoToExecToNod Map, List>> ret = new HashMap<>(); for (Entry schedEntry : schedAssignments.entrySet()) { Map, List> execToNodePort = new HashMap<>(); - for (Entry execAndNodePort : schedEntry.getValue().getExecutorToSlot().entrySet()) { + for (Entry execAndNodePort : schedEntry.getValue() + .getExecutorToSlot().entrySet()) { ExecutorDetails exec = execAndNodePort.getKey(); WorkerSlot slot = execAndNodePort.getValue(); execToNodePort.put(exec.toList(), slot.toList()); @@ -855,7 +932,8 @@ private static int numUsedWorkers(SchedulerAssignment assignment) { } /** - * Convert {topology-id -> SchedulerAssignment} to {topology-id -> {WorkerSlot WorkerResources}}. Make sure this can deal with other + * Convert {topology-id -> SchedulerAssignment} to {topology-id -> {WorkerSlot + * WorkerResources}}. Make sure this can deal with other * non-RAS schedulers later we may further support map-for-any-resources. * * @param schedAssignments the assignments @@ -886,7 +964,8 @@ private boolean auditAssignmentChanges(Map existingAssignmen long numAddedSlot = 0; if (existingAssignments.isEmpty()) { for (Entry entry : newAssignments.entrySet()) { - final Map, NodeInfo> execToPort = entry.getValue().get_executor_node_port(); + final Map, NodeInfo> execToPort = entry.getValue() + .get_executor_node_port(); final long count = new HashSet<>(execToPort.values()).size(); LOG.info("Assigning {} to {} slots", entry.getKey(), count); LOG.info("Assign executors: {}", execToPort.keySet()); @@ -895,7 +974,8 @@ private boolean auditAssignmentChanges(Map existingAssignmen } } else if (newAssignments.isEmpty()) { for (Entry entry : existingAssignments.entrySet()) { - final Map, NodeInfo> execToPort = entry.getValue().get_executor_node_port(); + final Map, NodeInfo> execToPort = entry.getValue() + .get_executor_node_port(); final long count = new HashSet<>(execToPort.values()).size(); LOG.info("Removing {} from {} slots", entry.getKey(), count); LOG.info("Remove executors: {}", execToPort.keySet()); @@ -903,10 +983,12 @@ private boolean auditAssignmentChanges(Map existingAssignmen numRemovedExec += execToPort.size(); } } else { - MapDifference difference = Maps.difference(existingAssignments, newAssignments); + MapDifference difference = Maps.difference(existingAssignments, + newAssignments); if (anyChanged = !difference.areEqual()) { for (Entry entry : difference.entriesOnlyOnLeft().entrySet()) { - final Map, NodeInfo> execToPort = entry.getValue().get_executor_node_port(); + final Map, NodeInfo> execToPort = entry.getValue() + .get_executor_node_port(); final long count = new HashSet<>(execToPort.values()).size(); LOG.info("Removing {} from {} slots", entry.getKey(), count); LOG.info("Remove executors: {}", execToPort.keySet()); @@ -914,20 +996,24 @@ private boolean auditAssignmentChanges(Map existingAssignmen numRemovedExec += execToPort.size(); } for (Entry entry : difference.entriesOnlyOnRight().entrySet()) { - final Map, NodeInfo> execToPort = entry.getValue().get_executor_node_port(); + final Map, NodeInfo> execToPort = entry.getValue() + .get_executor_node_port(); final long count = new HashSet<>(execToPort.values()).size(); LOG.info("Assigning {} to {} slots", entry.getKey(), count); LOG.info("Assign executors: {}", execToPort.keySet()); numAddedSlot += count; numAddedExec += execToPort.size(); } - for (Entry> entry : difference.entriesDiffering().entrySet()) { - final Map, NodeInfo> execToSlot = entry.getValue().rightValue().get_executor_node_port(); + for (Entry> entry : difference + .entriesDiffering().entrySet()) { + final Map, NodeInfo> execToSlot = entry.getValue().rightValue() + .get_executor_node_port(); final Set slots = new HashSet<>(execToSlot.values()); LOG.info("Reassigning {} to {} slots", entry.getKey(), slots.size()); LOG.info("Reassign executors: {}", execToSlot.keySet()); - final Map, NodeInfo> oldExecToSlot = entry.getValue().leftValue().get_executor_node_port(); + final Map, NodeInfo> oldExecToSlot = entry.getValue().leftValue() + .get_executor_node_port(); long commonExecCount = 0; Set commonSlots = new HashSet<>(execToSlot.size()); @@ -939,14 +1025,16 @@ private boolean auditAssignmentChanges(Map existingAssignmen } long commonSlotCount = commonSlots.size(); - //Treat reassign as remove and add - numRemovedSlot += new HashSet<>(oldExecToSlot.values()).size() - commonSlotCount; + // Treat reassign as remove and add + numRemovedSlot += new HashSet<>(oldExecToSlot.values()) + .size() - commonSlotCount; numRemovedExec += oldExecToSlot.size() - commonExecCount; numAddedSlot += slots.size() - commonSlotCount; numAddedExec += execToSlot.size() - commonExecCount; } } - LOG.debug("{} assignments unchanged: {}", difference.entriesInCommon().size(), difference.entriesInCommon().keySet()); + LOG.debug("{} assignments unchanged: {}", difference.entriesInCommon().size(), + difference.entriesInCommon().keySet()); } numAddedExecPerScheduling.update(numAddedExec); numAddedSlotPerScheduling.update(numAddedSlot); @@ -956,7 +1044,8 @@ private boolean auditAssignmentChanges(Map existingAssignmen numNetSlotIncreasePerScheduling.update(numAddedSlot - numRemovedSlot); if (anyChanged) { - LOG.info("Fragmentation after scheduling is: {} MB, {} PCore CPUs", fragmentedMemory(), fragmentedCpu()); + LOG.info("Fragmentation after scheduling is: {} MB, {} PCore CPUs", fragmentedMemory(), + fragmentedCpu()); nodeIdToResources.get().forEach((id, node) -> { final double availableMem = node.getAvailableMem(); if (availableMem < 0) { @@ -967,7 +1056,8 @@ private boolean auditAssignmentChanges(Map existingAssignmen LOG.warn("CPU over-scheduled on {}", id, availableCpu); } LOG.info( - "Node Id: {} Total Mem: {}, Used Mem: {}, Available Mem: {}, Total CPU: {}, Used " + "Node Id: {} Total Mem: {}, Used Mem: {}, Available Mem: {}, Total CPU: {}, " + + "Used " + "CPU: {}, Available CPU: {}, fragmented: {}", id, node.getTotalMem(), node.getUsedMem(), availableMem, node.getTotalCpu(), node.getUsedCpu(), availableCpu, isFragmented(node)); @@ -978,7 +1068,8 @@ private boolean auditAssignmentChanges(Map existingAssignmen private static List> changedExecutors(Map, NodeInfo> map, Map, List> newExecToNodePort) { - HashMap>> tmpSlotAssigned = map == null ? new HashMap<>() : Utils.reverseMap(map); + HashMap>> tmpSlotAssigned = map == null ? new HashMap<>() : Utils + .reverseMap(map); HashMap, List>> slotAssigned = new HashMap<>(); for (Entry>> entry : tmpSlotAssigned.entrySet()) { NodeInfo ni = entry.getKey(); @@ -989,7 +1080,8 @@ private static List> changedExecutors(Map, NodeInfo> map, value.sort(Comparator.comparing(a -> a.get(0))); slotAssigned.put(key, value); } - HashMap, List>> tmpNewSlotAssigned = newExecToNodePort == null ? new HashMap<>() : + HashMap, List>> tmpNewSlotAssigned = newExecToNodePort == null + ? new HashMap<>() : Utils.reverseMap(newExecToNodePort); HashMap, List>> newSlotAssigned = new HashMap<>(); for (Entry, List>> entry : tmpNewSlotAssigned.entrySet()) { @@ -1022,22 +1114,27 @@ private static Map basicSupervisorDetailsMap(IStormCl String id = entry.getKey(); SupervisorInfo info = entry.getValue(); ret.put(id, new SupervisorDetails(id, info.get_server_port(), info.get_hostname(), - info.get_scheduler_meta(), null, info.get_resources_map(), + info.get_scheduler_meta(), null, info + .get_resources_map(), supervisorUptimeSecs(info))); } return ret; } private static long supervisorUptimeSecs(SupervisorInfo info) { - // An unset uptime maps to 0L (not the Long.MAX_VALUE default the bare SupervisorDetails constructors use) so a - // freshly (re)registered supervisor is treated as just-returned and must accrue real uptime before - // Cluster#hasMinimumIdleSupervisorStability lets the idle rebalance place workers on it -- the conservative + // An unset uptime maps to 0L (not the Long.MAX_VALUE default the bare SupervisorDetails + // constructors use) so a + // freshly (re)registered supervisor is treated as just-returned and must accrue real uptime + // before + // Cluster#hasMinimumIdleSupervisorStability lets the idle rebalance place workers on it -- + // the conservative // choice on the production path. return info.is_set_uptime_secs() ? info.get_uptime_secs() : 0L; } /** - * NOTE: this can return false when a topology has just been activated. The topology may still be + * NOTE: this can return false when a topology has just been activated. The topology may still + * be * in the STORMS_SUBTREE. */ private static boolean isTopologyActive(IStormClusterState state, String topoName) { @@ -1050,12 +1147,14 @@ private static boolean isTopologyActiveOrActivating(IStormClusterState state, St /** * Returns the topologyId of the topology using this blob. + * * @param state the cluster state * @param topoCache the topology cache * @param key the blob key * @return null or id */ - private static String topologyUsingThisBlob(IStormClusterState state, TopoCache topoCache, String key) { + private static String topologyUsingThisBlob(IStormClusterState state, TopoCache topoCache, + String key) { for (String topologyId : state.activeStorms()) { Map topoConf = null; try { @@ -1067,7 +1166,8 @@ private static String topologyUsingThisBlob(IStormClusterState state, TopoCache continue; } @SuppressWarnings("unchecked") - Map> blobstoreMap = (Map>) topoConf.get(Config.TOPOLOGY_BLOBSTORE_MAP); + Map> blobstoreMap = (Map>) topoConf.get(Config.TOPOLOGY_BLOBSTORE_MAP); if (blobstoreMap != null && blobstoreMap.containsKey(key)) { return topologyId; } @@ -1079,7 +1179,8 @@ private static Map tryReadTopoConf(String topoId, TopoCache tc) throws NotAliveException, AuthorizationException, IOException { try { return readTopoConfAsNimbus(topoId, tc); - //Was a try-cause but I looked at the code around this and key not found is not wrapped in runtime, + // Was a try-cause but I looked at the code around this and key not found is not wrapped + // in runtime, // so it is not needed } catch (KeyNotFoundException e) { if (topoId == null) { @@ -1106,10 +1207,14 @@ private static long getTopologyCleanupDetectedTime(String topologyId) { } /** - * From a set of topologies that have been found to cleanup, return a set that has been detected for a minimum - * amount of time. Topology entries first detected less than NIMBUS_TOPOLOGY_BLOBSTORE_DELETION_DELAY_MS ago are - * ignored. The delay is to prevent a race conditions such as when a blobstore is created and when the topology - * is submitted. It is possible the Nimbus cleanup timer task will find entries to delete between these two events. + * From a set of topologies that have been found to cleanup, return a set that has been detected + * for a minimum + * amount of time. Topology entries first detected less than + * NIMBUS_TOPOLOGY_BLOBSTORE_DELETION_DELAY_MS ago are + * ignored. The delay is to prevent a race conditions such as when a blobstore is created and + * when the topology + * is submitted. It is possible the Nimbus cleanup timer task will find entries to delete + * between these two events. * *

      Tracked topology entries are rotated out of the stored map periodically. * @@ -1122,7 +1227,8 @@ static Set getExpiredTopologyIds(Set toposToClean, Map= topologyDeletionDelay) { + if (Math.max(0, Time + .currentTimeMillis() - getTopologyCleanupDetectedTime(topologyId)) >= topologyDeletionDelay) { idleTopologies.add(topologyId); } } @@ -1133,7 +1239,8 @@ static Set getExpiredTopologyIds(Set toposToClean, Map topoIdsToClean(IStormClusterState state, BlobStore store, Map conf) { + public static Set topoIdsToClean(IStormClusterState state, BlobStore store, Map conf) { Set cleanable = new HashSet<>(); cleanable.addAll(Utils.OR(state.heartbeatStorms(), EMPTY_STRING_LIST)); cleanable.addAll(Utils.OR(state.errorTopologies(), EMPTY_STRING_LIST)); @@ -1156,12 +1263,14 @@ private static String extractStatusStr(StormBase base) { return ret; } - private static StormTopology normalizeTopology(Map topoConf, StormTopology topology) + private static StormTopology normalizeTopology(Map topoConf, + StormTopology topology) throws InvalidTopologyException { StormTopology ret = topology.deepCopy(); for (Object comp : StormCommon.allComponents(ret).values()) { Map mergedConf = StormCommon.componentConf(comp); - mergedConf.put(Config.TOPOLOGY_TASKS, ServerUtils.getComponentParallelism(topoConf, comp)); + mergedConf.put(Config.TOPOLOGY_TASKS, ServerUtils.getComponentParallelism(topoConf, + comp)); String jsonConf = JSONValue.toJSONString(mergedConf); StormCommon.getComponentCommon(comp).set_json_conf(jsonConf); } @@ -1195,12 +1304,13 @@ private static void addToSerializers(Map ser, List conf) * @param topology the Storm topology */ @SuppressWarnings("unchecked") - static Map normalizeConf(Map conf, Map topoConf, StormTopology topology) { + static Map normalizeConf(Map conf, Map topoConf, + StormTopology topology) { // clear any values from the topoConf that it should not be setting. topoConf.remove(Config.STORM_WORKERS_ARTIFACTS_DIR); - //ensure that serializations are same for all tasks no matter what's on + // ensure that serializations are same for all tasks no matter what's on // the supervisors. this also allows you to declare the serializations as a sequence List> allConfs = new ArrayList<>(); for (Object comp : StormCommon.allComponents(topology).values()) { @@ -1208,15 +1318,17 @@ static Map normalizeConf(Map conf, Map decorators = new HashSet<>(); - //Yes we are putting in a config that is not the same type we pulled out. + // Yes we are putting in a config that is not the same type we pulled out. Map serializers = new HashMap<>(); for (Map c : allConfs) { addToDecorators(decorators, (List) c.get(Config.TOPOLOGY_KRYO_DECORATORS)); addToSerializers(serializers, (List) c.get(Config.TOPOLOGY_KRYO_REGISTER)); } - addToDecorators(decorators, (List) topoConf.getOrDefault(Config.TOPOLOGY_KRYO_DECORATORS, + addToDecorators(decorators, (List) topoConf + .getOrDefault(Config.TOPOLOGY_KRYO_DECORATORS, conf.get(Config.TOPOLOGY_KRYO_DECORATORS))); - addToSerializers(serializers, (List) topoConf.getOrDefault(Config.TOPOLOGY_KRYO_REGISTER, + addToSerializers(serializers, (List) topoConf + .getOrDefault(Config.TOPOLOGY_KRYO_REGISTER, conf.get(Config.TOPOLOGY_KRYO_REGISTER))); Map mergedConf = Utils.merge(conf, topoConf); @@ -1224,8 +1336,10 @@ static Map normalizeConf(Map conf, Map(decorators)); ret.put(Config.TOPOLOGY_ACKER_EXECUTORS, mergedConf.get(Config.TOPOLOGY_ACKER_EXECUTORS)); - ret.put(Config.TOPOLOGY_EVENTLOGGER_EXECUTORS, mergedConf.get(Config.TOPOLOGY_EVENTLOGGER_EXECUTORS)); - ret.put(Config.TOPOLOGY_MAX_TASK_PARALLELISM, mergedConf.get(Config.TOPOLOGY_MAX_TASK_PARALLELISM)); + ret.put(Config.TOPOLOGY_EVENTLOGGER_EXECUTORS, mergedConf + .get(Config.TOPOLOGY_EVENTLOGGER_EXECUTORS)); + ret.put(Config.TOPOLOGY_MAX_TASK_PARALLELISM, mergedConf + .get(Config.TOPOLOGY_MAX_TASK_PARALLELISM)); // storm.messaging.netty.authentication is about inter-worker communication // enforce netty authentication when either topo or daemon set it to true @@ -1236,43 +1350,53 @@ static Map normalizeConf(Map conf, Map> reporters = (List>) - ret.computeIfAbsent(Config.TOPOLOGY_METRICS_REPORTERS, (key) -> new ArrayList<>()); + ret.computeIfAbsent(Config.TOPOLOGY_METRICS_REPORTERS, + (key) -> new ArrayList<>()); List> systemReporters = (List>) conf.get(Config.STORM_TOPOLOGY_METRICS_SYSTEM_REPORTERS); reporters.addAll(systemReporters); } // Don't allow topoConf to override various cluster-specific properties. - // Specifically adding the cluster settings to the topoConf here will make sure these settings + // Specifically adding the cluster settings to the topoConf here will make sure these + // settings // also override the subsequently generated conf picked up locally on the classpath. // // We will be dealing with 3 confs: @@ -1280,18 +1404,22 @@ static Map normalizeConf(Map conf, Map workerMaxTimeoutSecs) { ret.put(Config.TOPOLOGY_WORKER_TIMEOUT_SECS, workerMaxTimeoutSecs); String topoId = (String) mergedConf.get(Config.STORM_ID); - LOG.warn("Topology {} topology.worker.timeout.secs is too large. Reducing from {} to {}", + LOG.warn("Topology {} topology.worker.timeout.secs is too large. Reducing from {} " + + "to {}", topoId, workerTimeoutSecs, workerMaxTimeoutSecs); } } @@ -1302,7 +1430,7 @@ private static void rmBlobKey(BlobStore store, String key, IStormClusterState st try { store.deleteBlob(key, NIMBUS_SUBJECT); } catch (Exception e) { - //Yes eat the exception + // Yes eat the exception LOG.info("Exception {}", e); } } @@ -1318,7 +1446,8 @@ public static void cleanInbox(String dirLoc, int seconds) { final long now = Time.currentTimeMillis(); final long ms = Time.secsToMillis(seconds); File dir = new File(dirLoc); - for (File f : dir.listFiles((file) -> file.isFile() && ((file.lastModified() + ms) <= now))) { + for (File f : dir.listFiles((file) -> file.isFile() && ((file.lastModified() + + ms) <= now))) { if (f.delete()) { LOG.info("Cleaning inbox ... deleted: {}", f.getName()); } else { @@ -1340,35 +1469,49 @@ private static void validateTopologyName(String name) throws InvalidTopologyExce } /** - * Check that a submitted topology only claims blobs that are topology dependencies, and that every one of them - * exists. The dependency lists of a submitted topology are filled in by the client, so without this a submission - * could name any blob at all, for example another topology's code or configuration blob, and nimbus would delete - * it as its own dependency once the submitted topology is cleaned up. A key that exists nowhere is just as - * damaging: on gaining leadership a nimbus compares the dependencies of every active topology against its - * blobstore and gives up leadership when one is missing, so a single unresolvable key on a single active topology + * Check that a submitted topology only claims blobs that are topology dependencies, and that + * every one of them + * exists. The dependency lists of a submitted topology are filled in by the client, so without + * this a submission + * could name any blob at all, for example another topology's code or configuration blob, and + * nimbus would delete + * it as its own dependency once the submitted topology is cleaned up. A key that exists nowhere + * is just as + * damaging: on gaining leadership a nimbus compares the dependencies of every active topology + * against its + * blobstore and gives up leadership when one is missing, so a single unresolvable key on a + * single active topology * leaves the cluster without a leader for as long as that topology is active. * *

      Existence is probed with {@code getBlobMeta} as the submitter, exactly as * {@link Utils#validateTopologyBlobStoreMap(Map, BlobStore)} probes the blobs of - * {@link Config#TOPOLOGY_BLOBSTORE_MAP}, so a submitter can only claim a dependency it is allowed to read. The - * client uploads the dependency blobs before it submits, so they are present by the time this runs. + * {@link Config#TOPOLOGY_BLOBSTORE_MAP}, so a submitter can only claim a dependency it is + * allowed to read. The + * client uploads the dependency blobs before it submits, so they are present by the time this + * runs. * * @param topology the submitted topology * @param blobStore the blobstore to look the keys up in * @param subject the subject to look the keys up as, i.e. the submitter - * @throws InvalidTopologyException if a dependency list holds something that is not a dependency blob key, or + * @throws InvalidTopologyException if a dependency list holds something that is not a + * dependency blob key, or * names a dependency blob that does not exist - * @throws AuthorizationException if the submitter may not read one of the dependency blobs it named + * @throws AuthorizationException if the submitter may not read one of the dependency blobs it + * named */ @VisibleForTesting - static void validateDependencyBlobKeys(StormTopology topology, BlobStore blobStore, Subject subject) + static void validateDependencyBlobKeys(StormTopology topology, BlobStore blobStore, + Subject subject) throws InvalidTopologyException, AuthorizationException { Set checked = new HashSet<>(); - validateDependencyBlobKeys(topology.get_dependency_jars(), "dependency_jars", blobStore, subject, checked); - validateDependencyBlobKeys(topology.get_dependency_artifacts(), "dependency_artifacts", blobStore, subject, checked); + validateDependencyBlobKeys(topology.get_dependency_jars(), "dependency_jars", blobStore, + subject, checked); + validateDependencyBlobKeys(topology.get_dependency_artifacts(), "dependency_artifacts", + blobStore, subject, checked); } - private static void validateDependencyBlobKeys(List keys, String fieldName, BlobStore blobStore, Subject subject, + private static void validateDependencyBlobKeys(List keys, String fieldName, + BlobStore blobStore, Subject subject, Set checked) throws InvalidTopologyException, AuthorizationException { if (keys == null) { return; @@ -1387,8 +1530,10 @@ private static void validateDependencyBlobKeys(List keys, String fieldNa blobStore.getBlobMeta(key, subject); } catch (KeyNotFoundException keyNotFound) { throw new WrappedInvalidTopologyException("Topology " + fieldName + " lists [" + key - + "], which is not in the blobstore; upload the dependency before submitting the topology, and if it " - + "was uploaded earlier note that a dependency blob is deleted once no topology uses it any more, so " + + "], which is not in the blobstore; upload the dependency before submitting " + + "the topology, and if it " + + "was uploaded earlier note that a dependency blob is deleted once no " + + "topology uses it any more, so " + "it has to be uploaded again"); } } @@ -1403,14 +1548,17 @@ private static StormTopology tryReadTopology(String topoId, TopoCache tc) } } - private static void validateTopologySize(Map topoConf, Map nimbusConf, + private static void validateTopologySize(Map topoConf, Map nimbusConf, StormTopology topology) throws InvalidTopologyException { // check allowedWorkers only if the scheduler is not the Resource Aware Scheduler if (!ServerUtils.isRas(nimbusConf)) { int workerCount = ObjectReader.getInt(topoConf.get(Config.TOPOLOGY_WORKERS), 1); - Integer allowedWorkers = ObjectReader.getInt(nimbusConf.get(DaemonConfig.NIMBUS_SLOTS_PER_TOPOLOGY), null); + Integer allowedWorkers = ObjectReader.getInt(nimbusConf + .get(DaemonConfig.NIMBUS_SLOTS_PER_TOPOLOGY), null); if (allowedWorkers != null && workerCount > allowedWorkers) { - throw new WrappedInvalidTopologyException("Failed to submit topology. Topology requests more than " + throw new WrappedInvalidTopologyException("Failed to submit topology. Topology " + + "requests more than " + allowedWorkers + " workers."); } } @@ -1418,9 +1566,11 @@ private static void validateTopologySize(Map topoConf, Map allowedExecutors) { - throw new WrappedInvalidTopologyException("Failed to submit topology. Topology requests more than " + throw new WrappedInvalidTopologyException("Failed to submit topology. Topology " + + "requests more than " + allowedExecutors + " executors."); } } @@ -1428,14 +1578,16 @@ private static void validateTopologySize(Map topoConf, Map 0) { - level.set_reset_log_level_timeout_epoch(Time.currentTimeMillis() + Time.secsToMillis(timeoutSecs)); + level.set_reset_log_level_timeout_epoch(Time.currentTimeMillis() + Time + .secsToMillis(timeoutSecs)); } else { level.unset_reset_log_level_timeout_epoch(); } } @VisibleForTesting - public static List topologiesOnSupervisor(Map assignments, String supervisorId) { + public static List topologiesOnSupervisor(Map assignments, + String supervisorId) { Set ret = new HashSet<>(); for (Entry entry : assignments.entrySet()) { Assignment assignment = entry.getValue(); @@ -1486,18 +1638,22 @@ private static Map> extr List metrics = new ArrayList<>(); metrics.add(new DataPoint("slotsTotal", sup.get_num_workers())); metrics.add(new DataPoint("slotsUsed", sup.get_num_used_workers())); - metrics.add(new DataPoint("totalMem", sup.get_total_resources().get(Constants.COMMON_TOTAL_MEMORY_RESOURCE_NAME))); - metrics.add(new DataPoint("totalCpu", sup.get_total_resources().get(Constants.COMMON_CPU_RESOURCE_NAME))); + metrics.add(new DataPoint("totalMem", sup.get_total_resources() + .get(Constants.COMMON_TOTAL_MEMORY_RESOURCE_NAME))); + metrics.add(new DataPoint("totalCpu", sup.get_total_resources() + .get(Constants.COMMON_CPU_RESOURCE_NAME))); metrics.add(new DataPoint("usedMem", sup.get_used_mem())); metrics.add(new DataPoint("usedCpu", sup.get_used_cpu())); IClusterMetricsConsumer.SupervisorInfo info = - new IClusterMetricsConsumer.SupervisorInfo(sup.get_host(), sup.get_supervisor_id(), Time.currentTimeSecs()); + new IClusterMetricsConsumer.SupervisorInfo(sup.get_host(), sup + .get_supervisor_id(), Time.currentTimeSecs()); ret.put(info, metrics); } return ret; } - private static void setResourcesDefaultIfNotSet(Map compResourcesMap, String compId, + private static void setResourcesDefaultIfNotSet(Map compResourcesMap, String compId, Map topoConf) { NormalizedResourceRequest resources = compResourcesMap.get(compId); if (resources == null) { @@ -1508,9 +1664,10 @@ private static void setResourcesDefaultIfNotSet(Map conf) throws IOException { int port = ObjectReader.getInt(conf.get(Config.NIMBUS_THRIFT_PORT)); try (ServerSocket socket = new ServerSocket(port)) { - //Nothing + // Nothing } catch (BindException e) { - LOG.error("{} is not available. Check if another process is already listening on {}", port, port); + LOG.error("{} is not available. Check if another process is already listening on {}", + port, port); System.exit(0); } } @@ -1524,8 +1681,9 @@ public void launchServer() throws Exception { IStormClusterState state = stormClusterState; NimbusInfo hpi = nimbusHostPortInfo; - //add to nimbuses - NimbusSummary nimbusSummary = new NimbusSummary(hpi.getHost(), hpi.getPort(), Time.currentTimeSecs(), false, STORM_VERSION); + // add to nimbuses + NimbusSummary nimbusSummary = new NimbusSummary(hpi.getHost(), hpi.getPort(), Time + .currentTimeSecs(), false, STORM_VERSION); nimbusSummary.set_tlsPort(hpi.getTlsPort()); state.addNimbusHost(hpi.getHost(), nimbusSummary); leaderElector.addToLeaderLockQueue(); @@ -1535,9 +1693,12 @@ public void launchServer() throws Exception { exec.prepare(); } - // Leadership coordination may be incomplete when launchServer is called. Previous behavior did a one time check - // which could cause Nimbus to not process TopologyActions.GAIN_LEADERSHIP transitions. Similar problem exists for - // HA Nimbus on being newly elected as leader. Change to a recurring pattern addresses these problems. + // Leadership coordination may be incomplete when launchServer is called. Previous + // behavior did a one time check + // which could cause Nimbus to not process TopologyActions.GAIN_LEADERSHIP transitions. + // Similar problem exists for + // HA Nimbus on being newly elected as leader. Change to a recurring pattern addresses + // these problems. timer.scheduleRecurring(3, 5, () -> { try { @@ -1554,8 +1715,10 @@ public void launchServer() throws Exception { } }); - final boolean doNotReassign = (Boolean) conf.getOrDefault(ServerConfigUtils.NIMBUS_DO_NOT_REASSIGN, false); - timer.scheduleRecurring(0, ObjectReader.getInt(conf.get(DaemonConfig.NIMBUS_MONITOR_FREQ_SECS)), + final boolean doNotReassign = (Boolean) conf + .getOrDefault(ServerConfigUtils.NIMBUS_DO_NOT_REASSIGN, false); + timer.scheduleRecurring(0, ObjectReader.getInt(conf + .get(DaemonConfig.NIMBUS_MONITOR_FREQ_SECS)), () -> { try { if (!doNotReassign) { @@ -1566,13 +1729,16 @@ public void launchServer() throws Exception { } }); // Schedule topology cleanup - cleanupTimer.scheduleRecurring(0, ObjectReader.getInt(conf.get(DaemonConfig.NIMBUS_MONITOR_FREQ_SECS)), + cleanupTimer.scheduleRecurring(0, ObjectReader.getInt(conf + .get(DaemonConfig.NIMBUS_MONITOR_FREQ_SECS)), () -> { cleanupTimer.schedule(0, () -> doCleanup()); }); // Schedule Nimbus inbox cleaner - final int jarExpSecs = ObjectReader.getInt(conf.get(DaemonConfig.NIMBUS_INBOX_JAR_EXPIRATION_SECS)); - timer.scheduleRecurring(0, ObjectReader.getInt(conf.get(DaemonConfig.NIMBUS_CLEANUP_INBOX_FREQ_SECS)), + final int jarExpSecs = ObjectReader.getInt(conf + .get(DaemonConfig.NIMBUS_INBOX_JAR_EXPIRATION_SECS)); + timer.scheduleRecurring(0, ObjectReader.getInt(conf + .get(DaemonConfig.NIMBUS_CLEANUP_INBOX_FREQ_SECS)), () -> { try { cleanInbox(getInbox(), jarExpSecs); @@ -1583,9 +1749,11 @@ public void launchServer() throws Exception { // Schedule topology history cleaner - Integer interval = ObjectReader.getInt(conf.get(DaemonConfig.LOGVIEWER_CLEANUP_INTERVAL_SECS), null); + Integer interval = ObjectReader.getInt(conf + .get(DaemonConfig.LOGVIEWER_CLEANUP_INTERVAL_SECS), null); if (interval != null) { - final int lvCleanupAgeMins = ObjectReader.getInt(conf.get(DaemonConfig.LOGVIEWER_CLEANUP_AGE_MINS)); + final int lvCleanupAgeMins = ObjectReader.getInt(conf + .get(DaemonConfig.LOGVIEWER_CLEANUP_AGE_MINS)); timer.scheduleRecurring(0, interval, () -> { try { @@ -1596,7 +1764,8 @@ public void launchServer() throws Exception { }); } - timer.scheduleRecurring(0, ObjectReader.getInt(conf.get(DaemonConfig.NIMBUS_CREDENTIAL_RENEW_FREQ_SECS)), + timer.scheduleRecurring(0, ObjectReader.getInt(conf + .get(DaemonConfig.NIMBUS_CREDENTIAL_RENEW_FREQ_SECS)), () -> { try { renewCredentials(); @@ -1605,8 +1774,10 @@ public void launchServer() throws Exception { } }); - // Periodically make sure the blobstore update time is up to date. This could have failed if Nimbus encountered - // an exception updating the update time, or due to bugs causing a missed update of the blobstore mod time on a blob + // Periodically make sure the blobstore update time is up to date. This could have + // failed if Nimbus encountered + // an exception updating the update time, or due to bugs causing a missed update of the + // blobstore mod time on a blob // update. timer.scheduleRecurring(30, ServerConfigUtils.getLocalizerUpdateBlobInterval(conf) * 5, () -> { @@ -1617,15 +1788,20 @@ public void launchServer() throws Exception { } }); - metricsRegistry.registerGauge("nimbus:total-available-memory-non-negative", () -> nodeIdToResources.get().values() + metricsRegistry.registerGauge("nimbus:total-available-memory-non-negative", + () -> nodeIdToResources.get().values() .parallelStream() - .mapToDouble(supervisorResources -> Math.max(supervisorResources.getAvailableMem(), 0)) + .mapToDouble(supervisorResources -> Math.max(supervisorResources + .getAvailableMem(), 0)) .sum()); - metricsRegistry.registerGauge("nimbus:available-cpu-non-negative", () -> nodeIdToResources.get().values() + metricsRegistry.registerGauge("nimbus:available-cpu-non-negative", + () -> nodeIdToResources.get().values() .parallelStream() - .mapToDouble(supervisorResources -> Math.max(supervisorResources.getAvailableCpu(), 0)) + .mapToDouble(supervisorResources -> Math.max(supervisorResources + .getAvailableCpu(), 0)) .sum()); - metricsRegistry.registerGauge("nimbus:total-memory", () -> nodeIdToResources.get().values() + metricsRegistry.registerGauge("nimbus:total-memory", () -> nodeIdToResources.get() + .values() .parallelStream() .mapToDouble(SupervisorResources::getTotalMem) .sum()); @@ -1634,7 +1810,8 @@ public void launchServer() throws Exception { .mapToDouble(SupervisorResources::getTotalCpu) .sum()); metricsRegistry.registerGauge("nimbus:longest-scheduling-time-ms", () -> { - //We want to update longest scheduling time in real time in case scheduler get stuck + // We want to update longest scheduling time in real time in case scheduler get + // stuck // Get current time before startTime to avoid potential race with scheduler's Timer Long currTime = Time.nanoTime(); Long startTime = schedulingStartTimeNs.get(); @@ -1644,7 +1821,8 @@ public void launchServer() throws Exception { }); metricsRegistry.registerMeter("nimbus:num-launched").mark(); - timer.scheduleRecurring(0, ObjectReader.getInt(conf.get(DaemonConfig.STORM_CLUSTER_METRICS_CONSUMER_PUBLISH_INTERVAL_SECS)), + timer.scheduleRecurring(0, ObjectReader.getInt(conf + .get(DaemonConfig.STORM_CLUSTER_METRICS_CONSUMER_PUBLISH_INTERVAL_SECS)), () -> { try { if (isLeader()) { @@ -1677,13 +1855,16 @@ private static Nimbus launchServer(Map conf, INimbus inimbus) th final Nimbus nimbus = new Nimbus(conf, inimbus, metricsRegistry); nimbus.launchServer(); - MultiThriftServer multiThriftServer = new MultiThriftServer<>("nimbus-thrift-server"); + MultiThriftServer multiThriftServer = + new MultiThriftServer<>("nimbus-thrift-server"); if (!ObjectReader.getBoolean(conf.get(Config.NIMBUS_THRIFT_TLS_SERVER_ONLY), false)) { - multiThriftServer.add(new ThriftServer(conf, new Processor<>(nimbus), ThriftConnectionType.NIMBUS)); + multiThriftServer.add(new ThriftServer(conf, new Processor<>(nimbus), + ThriftConnectionType.NIMBUS)); } int tlsPort = ObjectReader.getInt(conf.get(Config.NIMBUS_THRIFT_TLS_PORT)); if (tlsPort > 0) { - multiThriftServer.add(new ThriftServer(conf, new Processor<>(nimbus), ThriftConnectionType.NIMBUS_TLS)); + multiThriftServer.add(new ThriftServer(conf, new Processor<>(nimbus), + ThriftConnectionType.NIMBUS_TLS)); } metricsRegistry.startMetricsReporters(conf); @@ -1702,9 +1883,11 @@ private static Nimbus launchServer(Map conf, INimbus inimbus) th public static Nimbus launch(INimbus inimbus) throws Exception { Map conf = Utils.merge(ConfigUtils.readStormConfig(), - ConfigUtils.readYamlConfig("storm-cluster-auth.yaml", false)); + ConfigUtils.readYamlConfig("storm-cluster-auth.yaml", + false)); boolean fixupAcl = (boolean) conf.get(DaemonConfig.STORM_NIMBUS_ZOOKEEPER_ACLS_FIXUP); - boolean checkAcl = fixupAcl || (boolean) conf.get(DaemonConfig.STORM_NIMBUS_ZOOKEEPER_ACLS_CHECK); + boolean checkAcl = fixupAcl || (boolean) conf + .get(DaemonConfig.STORM_NIMBUS_ZOOKEEPER_ACLS_CHECK); if (checkAcl) { AclEnforcement.verifyAcls(conf, fixupAcl); } @@ -1723,13 +1906,16 @@ private static CuratorFramework makeZKClient(Map conf) { String root = (String) conf.get(Config.STORM_ZOOKEEPER_ROOT); CuratorFramework ret = null; if (servers != null && port != null) { - ret = ClientZookeeper.mkClient(conf, servers, port, root, new DefaultWatcherCallBack(), conf, DaemonType.NIMBUS); + ret = ClientZookeeper.mkClient(conf, servers, port, root, new DefaultWatcherCallBack(), + conf, DaemonType.NIMBUS); } return ret; } - private static IStormClusterState makeStormClusterState(Map conf) throws Exception { - return ClusterUtils.mkStormClusterState(conf, new ClusterStateContext(DaemonType.NIMBUS, conf)); + private static IStormClusterState makeStormClusterState(Map conf) throws Exception { + return ClusterUtils.mkStormClusterState(conf, new ClusterStateContext(DaemonType.NIMBUS, + conf)); } private static List asIntExec(List exec) { @@ -1746,7 +1932,8 @@ private static List asIntExec(List exec) { * @param newAss new assigned assignment * @return nodeId -> host map of assignments changed nodes */ - private static Map assignmentChangedNodes(Assignment oldAss, Assignment newAss) { + private static Map assignmentChangedNodes(Assignment oldAss, + Assignment newAss) { Map, NodeInfo> oldExecutorNodePort = null; Map, NodeInfo> newExecutorNodePort = null; Map allNodeHost = new HashMap<>(); @@ -1758,7 +1945,7 @@ private static Map assignmentChangedNodes(Assignment oldAss, Ass newExecutorNodePort = newAss.get_executor_node_port(); allNodeHost.putAll(newAss.get_node_host()); } - //kill or newly submit + // kill or newly submit if (oldAss == null || newAss == null) { return allNodeHost; } else { @@ -1782,16 +1969,20 @@ private static Map assignmentChangedNodes(Assignment oldAss, Ass } /** - * Pick out assignments for a specific host from all assignments. This could include multiple NUMA + * Pick out assignments for a specific host from all assignments. This could include multiple + * NUMA * supervisors on an individual host. + * * @param assignmentMap stormId -> assignment map * @param hostname hostname * @return stormId -> assignment map for the node */ - private static Map assignmentsForHost(Map assignmentMap, String hostname) { + private static Map assignmentsForHost(Map assignmentMap, + String hostname) { Map ret = new HashMap<>(); - assignmentMap.entrySet().stream().filter(assignmentEntry -> assignmentEntry.getValue().get_node_host().values() + assignmentMap.entrySet().stream().filter(assignmentEntry -> assignmentEntry.getValue() + .get_node_host().values() .contains(hostname)) .forEach(assignmentEntry -> { ret.put(assignmentEntry.getKey(), assignmentEntry.getValue()); @@ -1807,10 +1998,12 @@ private static Map assignmentsForHost(Map assignment map for the node */ - private static Map assignmentsForNodeId(Map assignmentMap, String nodeId) { + private static Map assignmentsForNodeId(Map assignmentMap, String nodeId) { Map ret = new HashMap<>(); - assignmentMap.entrySet().stream().filter(assignmentEntry -> assignmentEntry.getValue().get_node_host().keySet() + assignmentMap.entrySet().stream().filter(assignmentEntry -> assignmentEntry.getValue() + .get_node_host().keySet() .contains(nodeId)) .forEach(assignmentEntry -> { @@ -1820,12 +2013,12 @@ private static Map assignmentsForNodeId(Map host map * @param supervisorDetails nodeId -> {@link SupervisorDetails} map */ @@ -1838,18 +2031,22 @@ private static void notifySupervisorsAssignments(Map assignm String nodeId = nodeEntry.getKey(); String hostname = nodeEntry.getValue(); SupervisorAssignments supervisorAssignments = new SupervisorAssignments(); - supervisorAssignments.set_storm_assignment(assignmentsForHost(assignments, hostname)); + supervisorAssignments.set_storm_assignment(assignmentsForHost(assignments, + hostname)); SupervisorDetails details = supervisorDetails.get(nodeId); Integer serverPort = details != null ? details.getServerPort() : null; - service.addAssignmentsForNode(nodeId, nodeEntry.getValue(), serverPort, supervisorAssignments, metricsRegistry); + service.addAssignmentsForNode(nodeId, nodeEntry.getValue(), serverPort, + supervisorAssignments, metricsRegistry); } catch (Throwable tr1) { - //just skip when any error happens wait for next round assignments reassign - LOG.error("Exception when add assignments distribution task for node {}", nodeEntry.getKey()); + // just skip when any error happens wait for next round assignments reassign + LOG.error("Exception when add assignments distribution task for node {}", nodeEntry + .getKey()); } } } - private static void notifySupervisorsAsKilled(IStormClusterState clusterState, Assignment oldAss, + private static void notifySupervisorsAsKilled(IStormClusterState clusterState, + Assignment oldAss, AssignmentDistributionService service, StormMetricsRegistry metricsRegistry) { Map nodeHost = assignmentChangedNodes(oldAss, null); notifySupervisorsAssignments(clusterState.assignmentsInfo(), service, nodeHost, @@ -1886,7 +2083,8 @@ public AtomicReference>>> getIdToExecutors() { return idToExecutors; } - private Set> getOrUpdateExecutors(String topoId, StormBase base, Map topoConf, + private Set> getOrUpdateExecutors(String topoId, StormBase base, Map topoConf, StormTopology topology) throws InvalidTopologyException { Set> executors = idToExecutors.get().get(topoId); @@ -1928,8 +2126,10 @@ private String getInbox() throws IOException { } /** - * Check that a client supplied jar location names a file inside the nimbus inbox, i.e. one that was handed out by - * beginFileUpload and written through uploadChunk/finishFileUpload. Both paths are canonicalized first so that + * Check that a client supplied jar location names a file inside the nimbus inbox, i.e. one that + * was handed out by + * beginFileUpload and written through uploadChunk/finishFileUpload. Both paths are + * canonicalized first so that * ".." segments and symlinks cannot point outside of the inbox. * * @param inboxLocation the nimbus inbox directory @@ -1944,7 +2144,8 @@ static void validateUploadedJarLocation(String inboxLocation, String uploadedJar Path uploadedJar = new File(uploadedJarLocation).getCanonicalFile().toPath(); if (uploadedJar.equals(inboxDir) || !uploadedJar.startsWith(inboxDir)) { throw new WrappedAuthorizationException("uploadedJarLocation " + uploadedJarLocation - + " is not inside the nimbus inbox. Topology jars must be uploaded through beginFileUpload/uploadChunk" + + " is not inside the nimbus inbox. Topology jars must be uploaded through " + + "beginFileUpload/uploadChunk" + "/finishFileUpload before the topology is submitted."); } } @@ -1984,24 +2185,29 @@ void doRebalance(String topoId, StormBase stormBase) throws Exception { } stormClusterState.updateStorm(topoId, updated); updateBlobStore(topoId, rbo, ServerUtils.principalNameToSubject(rbo.get_principal())); - idToExecutors.getAndUpdate(new Dissoc<>(topoId)); // remove the executors cache to let it recompute. + idToExecutors + .getAndUpdate(new Dissoc<>(topoId)); // remove the executors cache to let it recompute. mkAssignments(topoId); } private String toTopoId(String topoName) throws NotAliveException { return stormClusterState.getTopoId(topoName) - .orElseThrow(() -> new WrappedNotAliveException(topoName + " is not alive")); + .orElseThrow(() -> new WrappedNotAliveException(topoName + + " is not alive")); } - private void transitionName(String topoName, TopologyActions event, Object eventArg, boolean errorOnNoTransition) throws Exception { + private void transitionName(String topoName, TopologyActions event, Object eventArg, + boolean errorOnNoTransition) throws Exception { transition(toTopoId(topoName), event, eventArg, errorOnNoTransition); } - private void transition(String topoId, TopologyActions event, Object eventArg) throws Exception { + private void transition(String topoId, TopologyActions event, + Object eventArg) throws Exception { transition(topoId, event, eventArg, false); } - private void transition(String topoId, TopologyActions event, Object eventArg, boolean errorOnNoTransition) + private void transition(String topoId, TopologyActions event, Object eventArg, + boolean errorOnNoTransition) throws Exception { LOG.info("TRANSITION: {} {} {} {}", topoId, event, eventArg, errorOnNoTransition); assertIsLeader(); @@ -2009,18 +2215,20 @@ private void transition(String topoId, TopologyActions event, Object eventArg, b IStormClusterState clusterState = stormClusterState; StormBase base = clusterState.stormBase(topoId, null); if (base == null || base.get_status() == null) { - LOG.info("Cannot apply event {} to {} because topology no longer exists", event, topoId); + LOG.info("Cannot apply event {} to {} because topology no longer exists", event, + topoId); } else { TopologyStatus status = base.get_status(); TopologyStateTransition transition = TOPO_STATE_TRANSITIONS.get(status).get(event); if (transition == null) { - String message = "No transition for event: " + event + ", status: " + status + " storm-id: " + topoId; + String message = "No transition for event: " + event + ", status: " + status + + " storm-id: " + topoId; if (errorOnNoTransition) { throw new RuntimeException(message); } if (TopologyActions.GAIN_LEADERSHIP != event) { - //GAIN_LEADERSHIP is a system event so don't log an issue + // GAIN_LEADERSHIP is a system event so don't log an issue LOG.info(message); } transition = NOOP_TRANSITION; @@ -2040,10 +2248,11 @@ private void setupStormCode(Map conf, String topoId, String tmpJ BlobStore store = blobStore; String jarKey = ConfigUtils.masterStormJarKey(topoId); if (tmpJarLocation != null) { - //in local mode there is no jar + // in local mode there is no jar validateUploadedJarLocation(getInbox(), tmpJarLocation); try (FileInputStream fin = new FileInputStream(tmpJarLocation)) { - store.createBlob(jarKey, fin, new SettableBlobMeta(BlobStoreAclHandler.DEFAULT), subject); + store.createBlob(jarKey, fin, new SettableBlobMeta(BlobStoreAclHandler.DEFAULT), + subject); } } @@ -2051,7 +2260,8 @@ private void setupStormCode(Map conf, String topoId, String tmpJ topoCache.addTopology(topoId, subject, topology); } - private void updateTopologyResources(String topoId, Map> resourceOverrides, Subject subject) + private void updateTopologyResources(String topoId, Map> resourceOverrides, Subject subject) throws AuthorizationException, IOException, KeyNotFoundException { StormTopology topo = topoCache.readTopology(topoId, subject); topo = topo.deepCopy(); @@ -2059,9 +2269,11 @@ private void updateTopologyResources(String topoId, Map configOverride, Subject subject) + private void updateTopologyConf(String topoId, Map configOverride, + Subject subject) throws AuthorizationException, IOException, KeyNotFoundException { - Map topoConf = new HashMap<>(topoCache.readTopoConf(topoId, subject)); //Copy the data + Map topoConf = new HashMap<>(topoCache.readTopoConf(topoId, + subject)); // Copy the data topoConf.putAll(configOverride); topoCache.updateTopoConf(topoId, subject, topoConf); } @@ -2086,9 +2298,12 @@ private Integer getBlobReplicationCount(String key) throws Exception { return null; } - private void waitForDesiredCodeReplication(Map topoConf, String topoId) throws Exception { - int minReplicationCount = ObjectReader.getInt(topoConf.get(Config.TOPOLOGY_MIN_REPLICATION_COUNT)); - int maxWaitTime = ObjectReader.getInt(topoConf.get(Config.TOPOLOGY_MAX_REPLICATION_WAIT_TIME_SEC)); + private void waitForDesiredCodeReplication(Map topoConf, + String topoId) throws Exception { + int minReplicationCount = ObjectReader.getInt(topoConf + .get(Config.TOPOLOGY_MIN_REPLICATION_COUNT)); + int maxWaitTime = ObjectReader.getInt(topoConf + .get(Config.TOPOLOGY_MAX_REPLICATION_WAIT_TIME_SEC)); int jarCount = minReplicationCount; if (!ConfigUtils.isLocalMode(topoConf)) { jarCount = getBlobReplicationCount(ConfigUtils.masterStormJarKey(topoId)); @@ -2096,20 +2311,23 @@ private void waitForDesiredCodeReplication(Map topoConf, String int codeCount = getBlobReplicationCount(ConfigUtils.masterStormCodeKey(topoId)); int confCount = getBlobReplicationCount(ConfigUtils.masterStormConfKey(topoId)); long totalWaitTime = 0; - //When is this ever null? + // When is this ever null? if (blobStore != null) { while (jarCount < minReplicationCount && codeCount < minReplicationCount && confCount < minReplicationCount) { if (maxWaitTime > 0 && totalWaitTime > maxWaitTime) { - LOG.info("desired replication count of {} not achieved for {} but we have hit the max wait time {}" - + " so moving on with replication count for conf key = {} for code key = {} for jar key = ", + LOG.info("desired replication count of {} not achieved for {} but we have hit " + + "the max wait time {}" + + " so moving on with replication count for conf key = {} for code " + + "key = {} for jar key = ", minReplicationCount, topoId, maxWaitTime, confCount, codeCount, jarCount); return; } LOG.debug("Checking if I am still the leader"); assertIsLeader(); - LOG.info("WAITING... storm-id {}, {} topoConf, String confCount = getBlobReplicationCount(ConfigUtils.masterStormConfKey(topoId)); } } - LOG.info("desired replication count {} achieved for topology {}, current-replication-count for conf key = {}," - + " current-replication-count for code key = {}, current-replication-count for jar key = {}", + LOG.info("desired replication count {} achieved for topology {}, " + + "current-replication-count for conf key = {}," + + " current-replication-count for code key = {}, current-replication-count for " + + "jar key = {}", minReplicationCount, topoId, confCount, codeCount, jarCount); } - private TopologyDetails readTopologyDetails(String topoId, StormBase base) throws KeyNotFoundException, + private TopologyDetails readTopologyDetails(String topoId, + StormBase base) throws KeyNotFoundException, AuthorizationException, IOException, InvalidTopologyException { if (base == null) { - throw new InvalidTopologyException("Cannot readTopologyDetails: StormBase parameter value is null"); + throw new InvalidTopologyException("Cannot readTopologyDetails: StormBase parameter " + + "value is null"); } if (topoId == null) { - throw new InvalidTopologyException("Cannot readTopologyDetails: topoId parameter value is null"); + throw new InvalidTopologyException("Cannot readTopologyDetails: topoId parameter " + + "value is null"); } Map topoConf = readTopoConfAsNimbus(topoId, topoCache); @@ -2140,7 +2363,8 @@ private TopologyDetails readTopologyDetails(String topoId, StormBase base) throw fixupBase(base, topoConf); stormClusterState.updateStorm(topoId, base); } - Map, String> rawExecToComponent = computeExecutorToComponent(topoId, base, topoConf, topo); + Map, String> rawExecToComponent = computeExecutorToComponent(topoId, base, + topoConf, topo); Map executorsToComponent = new HashMap<>(); for (Entry, String> entry : rawExecToComponent.entrySet()) { List execs = entry.getKey(); @@ -2148,16 +2372,20 @@ private TopologyDetails readTopologyDetails(String topoId, StormBase base) throw executorsToComponent.put(execDetails, entry.getValue()); } - return new TopologyDetails(topoId, topoConf, topo, base.get_num_workers(), executorsToComponent, + return new TopologyDetails(topoId, topoConf, topo, base.get_num_workers(), + executorsToComponent, base.get_launch_time_secs(), base.get_owner()); } - private void updateHeartbeatsFromZkHeartbeat(String topoId, Set> allExecutors, Assignment existingAssignment) { + private void updateHeartbeatsFromZkHeartbeat(String topoId, Set> allExecutors, + Assignment existingAssignment) { LOG.debug("Updating heartbeats for {} {} (from ZK heartbeat)", topoId, allExecutors); IStormClusterState state = stormClusterState; Map, Map> executorBeats = - StatsUtil.convertExecutorBeats(state.executorBeats(topoId, existingAssignment.get_executor_node_port())); - heartbeatsCache.updateFromZkHeartbeat(topoId, executorBeats, allExecutors, getTopologyHeartbeatTimeoutSecs(topoId)); + StatsUtil.convertExecutorBeats(state.executorBeats(topoId, existingAssignment + .get_executor_node_port())); + heartbeatsCache.updateFromZkHeartbeat(topoId, executorBeats, allExecutors, + getTopologyHeartbeatTimeoutSecs(topoId)); } /** @@ -2171,15 +2399,18 @@ private void updateAllHeartbeats(Map existingAssignments, for (Entry entry : existingAssignments.entrySet()) { String topoId = entry.getKey(); if (zkHeartbeatTopologies.contains(topoId)) { - updateHeartbeatsFromZkHeartbeat(topoId, topologyToExecutors.get(topoId), entry.getValue()); + updateHeartbeatsFromZkHeartbeat(topoId, topologyToExecutors.get(topoId), entry + .getValue()); } else { LOG.debug("Timing out old heartbeats for {}", topoId); - heartbeatsCache.timeoutOldHeartbeats(topoId, getTopologyHeartbeatTimeoutSecs(topoId)); + heartbeatsCache.timeoutOldHeartbeats(topoId, + getTopologyHeartbeatTimeoutSecs(topoId)); } } } - private void updateCachedHeartbeatsFromWorker(SupervisorWorkerHeartbeat workerHeartbeat, int heartbeatTimeoutSecs) { + private void updateCachedHeartbeatsFromWorker(SupervisorWorkerHeartbeat workerHeartbeat, + int heartbeatTimeoutSecs) { heartbeatsCache.updateHeartbeat(workerHeartbeat, heartbeatTimeoutSecs); } @@ -2189,13 +2420,15 @@ private void updateCachedHeartbeatsFromSupervisor(SupervisorWorkerHeartbeats wor int heartbeatTimeoutSecs = getTopologyHeartbeatTimeoutSecs(topoId); updateCachedHeartbeatsFromWorker(hb, heartbeatTimeoutSecs); } - if (!heartbeatsReadyFlag.get() && !Strings.isNullOrEmpty(workerHeartbeats.get_supervisor_id())) { + if (!heartbeatsReadyFlag.get() && !Strings.isNullOrEmpty(workerHeartbeats + .get_supervisor_id())) { heartbeatsRecoveryStrategy.reportNodeId(workerHeartbeats.get_supervisor_id()); } } /** - * Decide if the heartbeats is recovered for a master, will wait for all the assignments nodes to recovery, every node will take care + * Decide if the heartbeats is recovered for a master, will wait for all the assignments nodes + * to recovery, every node will take care * its node heartbeats reporting. * * @return true if all nodes have reported heartbeats or exceeds max-time-out @@ -2205,7 +2438,8 @@ private boolean isHeartbeatsRecovered() { return true; } Set allNodes = new HashSet<>(); - for (Map.Entry assignmentEntry : stormClusterState.assignmentsInfo().entrySet()) { + for (Map.Entry assignmentEntry : stormClusterState.assignmentsInfo() + .entrySet()) { allNodes.addAll(assignmentEntry.getValue().get_node_host().keySet()); } boolean isReady = heartbeatsRecoveryStrategy.isReady(allNodes); @@ -2224,8 +2458,10 @@ private boolean isAssignmentsRecovered() { return stormClusterState.isAssignmentsBackendSynchronized(); } - private Set> aliveExecutors(String topoId, Set> allExecutors, Assignment assignment) { - return heartbeatsCache.getAliveExecutors(topoId, allExecutors, assignment, getTopologyLaunchHeartbeatTimeoutSec(topoId)); + private Set> aliveExecutors(String topoId, Set> allExecutors, + Assignment assignment) { + return heartbeatsCache.getAliveExecutors(topoId, allExecutors, assignment, + getTopologyLaunchHeartbeatTimeoutSec(topoId)); } private List> computeExecutors(StormBase base, Map topoConf, @@ -2233,7 +2469,8 @@ private List> computeExecutors(StormBase base, Map throws InvalidTopologyException { if (base == null) { - throw new InvalidTopologyException("Cannot computeExecutors: StormBase parameter value is null"); + throw new InvalidTopologyException("Cannot computeExecutors: StormBase parameter " + + "value is null"); } Map compToExecutors = base.get_component_executors(); @@ -2249,7 +2486,8 @@ private List> computeExecutors(StormBase base, Map if (numExecutors != null) { List> partitioned = Utils.partitionFixed(numExecutors, tasks); for (List partition : partitioned) { - ret.add(Arrays.asList(partition.get(0), partition.get(partition.size() - 1))); + ret.add(Arrays.asList(partition.get(0), partition.get(partition + .size() - 1))); } } } @@ -2260,7 +2498,8 @@ private List> computeExecutors(StormBase base, Map private Map, String> computeExecutorToComponent(String topoId, StormBase base, Map topoConf, StormTopology topology) throws InvalidTopologyException { - List> executors = new ArrayList<>(getOrUpdateExecutors(topoId, base, topoConf, topology)); + List> executors = new ArrayList<>(getOrUpdateExecutors(topoId, base, topoConf, + topology)); Map taskToComponent = StormCommon.stormTaskInfo(topology, topoConf); Map, String> ret = new HashMap<>(); for (List executor : executors) { @@ -2288,14 +2527,15 @@ private Map>> computeTopologyToExecutors(Map alive executors map. + * Compute a topology-id -> alive executors map. * * @param existingAssignment the current assignments * @param topologyToExecutors the executors for the current topologies * @param scratchTopologyId the topology being rebalanced and should be excluded * @return the map of topology id to alive executors */ - private Map>> computeTopologyToAliveExecutors(Map existingAssignment, + private Map>> computeTopologyToAliveExecutors(Map existingAssignment, Map>> topologyToExecutors, String scratchTopologyId) { Map>> ret = new HashMap<>(); @@ -2314,7 +2554,8 @@ private Map>> computeTopologyToAliveExecutors(Map> computeSupervisorToDeadPorts(Map existingAssignments, + private Map> computeSupervisorToDeadPorts(Map existingAssignments, Map>> topologyToExecutors, Map>> topologyToAliveExecutors) { Map> ret = new HashMap<>(); @@ -2343,13 +2584,15 @@ private Map> computeSupervisorToDeadPorts(Map computeTopologyToSchedulerAssignment(Map existingAssignments, + private Map computeTopologyToSchedulerAssignment(Map existingAssignments, Map>> topologyToAliveExecutors) { Map ret = new HashMap<>(); @@ -2373,7 +2616,8 @@ private Map computeTopologyToSchedulerAssignmen List exec = asIntExec(execAndNodePort.getKey()); NodeInfo info = execAndNodePort.getValue(); if (aliveExecutors.contains(exec)) { - execToSlot.put(new ExecutorDetails(exec.get(0), exec.get(1)), nodePortToSlot.get(info)); + execToSlot.put(new ExecutorDetails(exec.get(0), exec.get(1)), nodePortToSlot + .get(info)); } } ret.put(topoId, new SchedulerAssignmentImpl(topoId, execToSlot, slotToResources, null)); @@ -2389,7 +2633,8 @@ private Map computeTopologyToSchedulerAssignmen * @param missingAssignmentTopologies topologies that need assignments * @return a map: {supervisor-id SupervisorDetails} */ - private Map readAllSupervisorDetails(Map> superToDeadPorts, + private Map readAllSupervisorDetails(Map> superToDeadPorts, Topologies topologies, Collection missingAssignmentTopologies) { Map ret = new HashMap<>(); IStormClusterState state = stormClusterState; @@ -2397,7 +2642,8 @@ private Map readAllSupervisorDetails(Map superDetails = new ArrayList<>(); for (Entry entry : superInfos.entrySet()) { SupervisorInfo info = entry.getValue(); - superDetails.add(new SupervisorDetails(entry.getKey(), info.get_meta(), info.get_resources_map(), + superDetails.add(new SupervisorDetails(entry.getKey(), info.get_meta(), info + .get_resources_map(), supervisorUptimeSecs(info))); } // Note that allSlotsAvailableForScheduling @@ -2431,33 +2677,42 @@ private Map readAllSupervisorDetails(Map supervisorResources.getAvailableMem() || minCpu > supervisorResources.getAvailableCpu(); + return minMemory > supervisorResources.getAvailableMem() || minCpu > supervisorResources + .getAvailableCpu(); } private double fragmentedMemory() { Double res = nodeIdToResources.get().values().parallelStream().filter(this::isFragmented) - .mapToDouble(SupervisorResources::getAvailableMem).filter(x -> x > 0).sum(); + .mapToDouble(SupervisorResources::getAvailableMem) + .filter(x -> x > 0).sum(); return res.intValue(); } private int fragmentedCpu() { Double res = nodeIdToResources.get().values().parallelStream().filter(this::isFragmented) - .mapToDouble(SupervisorResources::getAvailableCpu).filter(x -> x > 0).sum(); + .mapToDouble(SupervisorResources::getAvailableCpu) + .filter(x -> x > 0).sum(); return res.intValue(); } - private Map computeNewSchedulerAssignments(Map existingAssignments, + private Map computeNewSchedulerAssignments(Map existingAssignments, Topologies topologies, Map bases, String scratchTopologyId) throws KeyNotFoundException, AuthorizationException, InvalidTopologyException, IOException { @@ -2471,11 +2726,14 @@ private Map computeNewSchedulerAssignments(Map>> topoToAliveExecutors = computeTopologyToAliveExecutors(existingAssignments, topoToExec, + Map>> topoToAliveExecutors = + computeTopologyToAliveExecutors(existingAssignments, topoToExec, scratchTopologyId); - Map> supervisorToDeadPorts = computeSupervisorToDeadPorts(existingAssignments, topoToExec, + Map> supervisorToDeadPorts = + computeSupervisorToDeadPorts(existingAssignments, topoToExec, topoToAliveExecutors); - Map topoToSchedAssignment = computeTopologyToSchedulerAssignment(existingAssignments, + Map topoToSchedAssignment = + computeTopologyToSchedulerAssignment(existingAssignments, topoToAliveExecutors); Set missingAssignmentTopologies = new HashSet<>(); for (TopologyDetails topo : topologies.getTopologies()) { @@ -2484,26 +2742,31 @@ private Map computeNewSchedulerAssignments(Map> aliveExecs = topoToAliveExecutors.get(id); int numDesiredWorkers = topo.getNumWorkers(); int numAssignedWorkers = numUsedWorkers(topoToSchedAssignment.get(id)); - if (allExecs == null || allExecs.isEmpty() || !allExecs.equals(aliveExecs) || numDesiredWorkers > numAssignedWorkers) { - //We have something to schedule... + if (allExecs == null || allExecs.isEmpty() || !allExecs.equals(aliveExecs) + || numDesiredWorkers > numAssignedWorkers) { + // We have something to schedule... missingAssignmentTopologies.add(id); } } Map supervisors = - readAllSupervisorDetails(supervisorToDeadPorts, topologies, missingAssignmentTopologies); - Cluster cluster = new Cluster(inimbus, resourceMetrics, supervisors, topoToSchedAssignment, topologies, conf); + readAllSupervisorDetails(supervisorToDeadPorts, topologies, + missingAssignmentTopologies); + Cluster cluster = new Cluster(inimbus, resourceMetrics, supervisors, topoToSchedAssignment, + topologies, conf); cluster.setStatusMap(idToSchedStatus.get()); schedulingStartTimeNs.set(Time.nanoTime()); scheduler.schedule(topologies, cluster); - //Get and set the start time before getting current time in order to avoid potential race with the longest-scheduling-time-ms gauge + // Get and set the start time before getting current time in order to avoid potential race + // with the longest-scheduling-time-ms gauge final Long startTime = schedulingStartTimeNs.getAndSet(null); long elapsedNs = Time.nanoTime() - startTime; longestSchedulingTime.accumulateAndGet(elapsedNs, Math::max); schedulingDuration.update(elapsedNs, TimeUnit.NANOSECONDS); - LOG.debug("Scheduling took {} ms for {} topologies", TimeUnit.NANOSECONDS.toMillis(elapsedNs), topologies.getTopologies().size()); + LOG.debug("Scheduling took {} ms for {} topologies", TimeUnit.NANOSECONDS + .toMillis(elapsedNs), topologies.getTopologies().size()); - //merge with existing statuses + // merge with existing statuses idToSchedStatus.set(Utils.merge(idToSchedStatus.get(), cluster.getStatusMap())); nodeIdToResources.set(cluster.getSupervisorsResourcesMap()); @@ -2512,15 +2775,18 @@ private Map computeNewSchedulerAssignments(Map Utils.merge(orig, update)); Map> workerResources = new HashMap<>(); - for (Entry> uglyWorkerResources : cluster.getWorkerResourcesMap().entrySet()) { + for (Entry> uglyWorkerResources : cluster + .getWorkerResourcesMap().entrySet()) { Map slotToResources = new HashMap<>(); - for (Entry uglySlotToResources : uglyWorkerResources.getValue().entrySet()) { + for (Entry uglySlotToResources : uglyWorkerResources + .getValue().entrySet()) { WorkerResources wr = uglySlotToResources.getValue(); slotToResources.put(uglySlotToResources.getKey(), wr); } workerResources.put(uglyWorkerResources.getKey(), slotToResources); } - idToWorkerResources.getAndAccumulate(workerResources, (orig, update) -> Utils.merge(orig, update)); + idToWorkerResources.getAndAccumulate(workerResources, (orig, update) -> Utils.merge(orig, + update)); return cluster.getAssignments(); } @@ -2552,7 +2818,7 @@ private TopologyResources getResourcesForTopology(String topoId, StormBase base) Assignment assignment = state.assignmentInfo(topoId, null); ret = new TopologyResources(details, assignment); } catch (KeyNotFoundException e) { - //This can happen when a topology is first coming up + // This can happen when a topology is first coming up // It's thrown by the blobstore code LOG.error("Failed to get topology details", e); ret = new TopologyResources(); @@ -2568,7 +2834,8 @@ private Map getWorkerResourcesForTopology(String to ret = new HashMap<>(); Assignment assignment = state.assignmentInfo(topoId, null); if (assignment != null && assignment.is_set_worker_resources()) { - for (Entry entry : assignment.get_worker_resources().entrySet()) { + for (Entry entry : assignment.get_worker_resources() + .entrySet()) { NodeInfo ni = entry.getKey(); WorkerSlot slot = new WorkerSlot(ni.get_node(), ni.get_port_iterator().next()); ret.put(slot, entry.getValue()); @@ -2606,28 +2873,33 @@ private void mkAssignments(String scratchTopoId) throws Exception { } // get existing assignment (just the topologyToExecutorToNodePort map) -> default to {} // filter out ones which have a executor timeout - // figure out available slots on cluster. add to that the used valid slots to get total slots. figure out how many executors + // figure out available slots on cluster. add to that the used valid slots to get total + // slots. figure out how many executors // should be in each slot (e.g., 4, 4, 4, 5) - // only keep existing slots that satisfy one of those slots. for rest, reassign them across remaining slots - // edge case for slots with no executor timeout but with supervisor timeout... just treat these as valid slots that can be - // reassigned to. worst comes to worse the executor will timeout and won't assign here next time around + // only keep existing slots that satisfy one of those slots. for rest, reassign them + // across remaining slots + // edge case for slots with no executor timeout but with supervisor timeout... just + // treat these as valid slots that can be + // reassigned to. worst comes to worse the executor will timeout and won't assign here + // next time around IStormClusterState state = stormClusterState; - //read all the topologies + // read all the topologies Map bases; Map tds = new HashMap<>(); synchronized (submitLock) { // should promote: only fetch storm bases of topologies that need scheduling. bases = state.topologyBases(); - for (Iterator> it = bases.entrySet().iterator(); it.hasNext(); ) { + for (Iterator> it = bases.entrySet().iterator(); it + .hasNext(); ) { Entry entry = it.next(); String id = entry.getKey(); StormBase base = entry.getValue(); try { tds.put(id, readTopologyDetails(id, base)); } catch (KeyNotFoundException e) { - //A race happened and it is probably not running + // A race happened and it is probably not running it.remove(); } } @@ -2635,7 +2907,7 @@ private void mkAssignments(String scratchTopoId) throws Exception { List assignedTopologyIds = state.assignments(null); Map existingAssignments = new HashMap<>(); for (String id : assignedTopologyIds) { - //for the topology which wants rebalance (specified by the scratchTopoId) + // for the topology which wants rebalance (specified by the scratchTopoId) // we exclude its assignment, meaning that all the slots occupied by its assignment // will be treated as free slot in the scheduler code. if (!id.equals(scratchTopoId)) { @@ -2652,31 +2924,36 @@ private void mkAssignments(String scratchTopoId) throws Exception { } // make the new assignments for topologies - lockingMkAssignments(existingAssignments, bases, scratchTopoId, assignedTopologyIds, state, tds); + lockingMkAssignments(existingAssignments, bases, scratchTopoId, assignedTopologyIds, + state, tds); } catch (Exception e) { this.mkAssignmentsErrors.mark(); throw e; } } - private void lockingMkAssignments(Map existingAssignments, Map bases, + private void lockingMkAssignments(Map existingAssignments, Map bases, String scratchTopoId, List assignedTopologyIds, IStormClusterState state, Map tds) throws Exception { Topologies topologies = new Topologies(tds); synchronized (schedLock) { Map newSchedulerAssignments = - computeNewSchedulerAssignments(existingAssignments, topologies, bases, scratchTopoId); + computeNewSchedulerAssignments(existingAssignments, topologies, bases, + scratchTopoId); Map, List>> topologyToExecutorToNodePort = computeTopoToExecToNodePort(newSchedulerAssignments, assignedTopologyIds); Map> newAssignedWorkerToResources = computeTopoToNodePortToResources(newSchedulerAssignments); int nowSecs = Time.currentTimeSecs(); - Map basicSupervisorDetailsMap = basicSupervisorDetailsMap(state); - //construct the final Assignments by adding start-times etc into it + Map basicSupervisorDetailsMap = + basicSupervisorDetailsMap(state); + // construct the final Assignments by adding start-times etc into it Map newAssignments = new HashMap<>(); - for (Entry, List>> entry : topologyToExecutorToNodePort.entrySet()) { + for (Entry, List>> entry : topologyToExecutorToNodePort + .entrySet()) { String topoId = entry.getKey(); Map, List> execToNodePort = entry.getValue(); if (execToNodePort == null) { @@ -2709,16 +2986,18 @@ private void lockingMkAssignments(Map existingAssignments, M for (List id : reassignExecutors) { startTimes.put(id, (long) nowSecs); } - Map workerToResources = newAssignedWorkerToResources.get(topoId); + Map workerToResources = newAssignedWorkerToResources + .get(topoId); if (workerToResources == null) { workerToResources = new HashMap<>(); } - Assignment newAssignment = new Assignment((String) conf.get(Config.STORM_LOCAL_DIR)); + Assignment newAssignment = new Assignment((String) conf + .get(Config.STORM_LOCAL_DIR)); Map justAssignedKeys = new HashMap<>(allNodeHost); - //Modifies justAssignedKeys + // Modifies justAssignedKeys justAssignedKeys.keySet().retainAll(allNodes); newAssignment.set_node_host(justAssignedKeys); - //convert NodePort to NodeInfo (again!!!). + // convert NodePort to NodeInfo (again!!!). Map, NodeInfo> execToNodeInfo = new HashMap<>(); for (Entry, List> execAndNodePort : execToNodePort.entrySet()) { List nodePort = execAndNodePort.getValue(); @@ -2729,7 +3008,7 @@ private void lockingMkAssignments(Map existingAssignments, M } newAssignment.set_executor_node_port(execToNodeInfo); newAssignment.set_executor_start_time_secs(startTimes); - //do another conversion (lets just make this all common) + // do another conversion (lets just make this all common) Map workerResources = new HashMap<>(); for (Entry wr : workerToResources.entrySet()) { WorkerSlot nodePort = wr.getKey(); @@ -2752,7 +3031,7 @@ private void lockingMkAssignments(Map existingAssignments, M idToWorkerResources.set(new HashMap<>()); } - //tasks figure out what tasks to talk to by looking at topology at runtime + // tasks figure out what tasks to talk to by looking at topology at runtime // only log/set when there's been a change to the assignment for (Entry entry : newAssignments.entrySet()) { String topoId = entry.getKey(); @@ -2767,17 +3046,20 @@ private void lockingMkAssignments(Map existingAssignments, M } } - //grouping assignment by node to see the nodes diff, then notify nodes/supervisors to synchronize its owned assignment - //because the number of existing assignments is small for every scheduling round, - //we expect to notify supervisors at almost the same time + // grouping assignment by node to see the nodes diff, then notify nodes/supervisors to + // synchronize its owned assignment + // because the number of existing assignments is small for every scheduling round, + // we expect to notify supervisors at almost the same time Map totalAssignmentsChangedNodes = new HashMap<>(); for (Entry entry : newAssignments.entrySet()) { String topoId = entry.getKey(); Assignment assignment = entry.getValue(); Assignment existingAssignment = existingAssignments.get(topoId); - totalAssignmentsChangedNodes.putAll(assignmentChangedNodes(existingAssignment, assignment)); + totalAssignmentsChangedNodes.putAll(assignmentChangedNodes(existingAssignment, + assignment)); } - notifySupervisorsAssignments(newAssignments, assignmentsDistributer, totalAssignmentsChangedNodes, + notifySupervisorsAssignments(newAssignments, assignmentsDistributer, + totalAssignmentsChangedNodes, basicSupervisorDetailsMap, getMetricsRegistry()); Map> addedSlots = new HashMap<>(); @@ -2803,7 +3085,8 @@ private void notifyTopologyActionListener(String topoId, String action) { try { notifier.notify(topoId, action); } catch (Exception e) { - LOG.warn("Ignoring exception from Topology action notifier for storm-Id {}", topoId, e); + LOG.warn("Ignoring exception from Topology action notifier for storm-Id {}", topoId, + e); } } } @@ -2815,9 +3098,11 @@ private void fixupBase(StormBase base, Map topoConf) { // Topology may set custom heartbeat timeout. private int getTopologyHeartbeatTimeoutSecs(Map topoConf) { - int defaultNimbusTimeout = ObjectReader.getInt(conf.get(DaemonConfig.NIMBUS_TASK_TIMEOUT_SECS)); + int defaultNimbusTimeout = ObjectReader.getInt(conf + .get(DaemonConfig.NIMBUS_TASK_TIMEOUT_SECS)); if (topoConf.containsKey(Config.TOPOLOGY_WORKER_TIMEOUT_SECS)) { - int topoTimeout = ObjectReader.getInt(topoConf.get(Config.TOPOLOGY_WORKER_TIMEOUT_SECS)); + int topoTimeout = ObjectReader.getInt(topoConf + .get(Config.TOPOLOGY_WORKER_TIMEOUT_SECS)); topoTimeout = Math.max(topoTimeout, defaultNimbusTimeout); return topoTimeout; } @@ -2840,16 +3125,19 @@ private int getTopologyHeartbeatTimeoutSecs(String topoId) { } private int getTopologyLaunchHeartbeatTimeoutSec(String topoId) { - int nimbusLaunchTimeout = ObjectReader.getInt(conf.get(DaemonConfig.NIMBUS_TASK_LAUNCH_SECS)); + int nimbusLaunchTimeout = ObjectReader.getInt(conf + .get(DaemonConfig.NIMBUS_TASK_LAUNCH_SECS)); int topoHeartbeatTimeoutSecs = getTopologyHeartbeatTimeoutSecs(topoId); return Math.max(nimbusLaunchTimeout, topoHeartbeatTimeoutSecs); } - private void startTopology(String topoName, String topoId, TopologyStatus initStatus, String owner, + private void startTopology(String topoName, String topoId, TopologyStatus initStatus, + String owner, String principal, Map topoConf, StormTopology stormTopology) throws InvalidTopologyException { if (TopologyStatus.ACTIVE != initStatus && TopologyStatus.INACTIVE != initStatus) { - throw new InvalidTopologyException("Cannot startTopology: initStatus should be ACTIVE or INACTIVE, not " + initStatus.name()); + throw new InvalidTopologyException("Cannot startTopology: initStatus should be ACTIVE " + + "or INACTIVE, not " + initStatus.name()); } Map numExecutors = new HashMap<>(); StormTopology topology = StormCommon.systemTopology(topoConf, stormTopology); @@ -2860,7 +3148,8 @@ private void startTopology(String topoName, String topoId, TopologyStatus initSt StormBase base = new StormBase(); base.set_name(topoName); if (topoConf.containsKey(Config.TOPOLOGY_VERSION)) { - base.set_topology_version(ObjectReader.getString(topoConf.get(Config.TOPOLOGY_VERSION))); + base.set_topology_version(ObjectReader.getString(topoConf + .get(Config.TOPOLOGY_VERSION))); } base.set_launch_time_secs(Time.currentTimeSecs()); base.set_status(initStatus); @@ -2876,7 +3165,8 @@ private void startTopology(String topoName, String topoId, TopologyStatus initSt notifyTopologyActionListener(topoName, "activate"); } - private void assertTopoActive(String topoName, boolean expectActive) throws NotAliveException, AlreadyAliveException { + private void assertTopoActive(String topoName, + boolean expectActive) throws NotAliveException, AlreadyAliveException { if (isTopologyActive(stormClusterState, topoName) != expectActive) { if (expectActive) { throw new WrappedNotAliveException(topoName + " is not alive"); @@ -2902,7 +3192,8 @@ public void checkAuthorization(String topoName, Map topoConf, St } @VisibleForTesting - public void checkAuthorization(String topoName, Map topoConf, String operation, ReqContext context) + public void checkAuthorization(String topoName, Map topoConf, String operation, + ReqContext context) throws AuthorizationException { @@ -2919,19 +3210,24 @@ public void checkAuthorization(String topoName, Map topoConf, St } if (context.isImpersonating()) { - LOG.info("principal: {} is trying to impersonate principal: {}", context.realPrincipal(), context.principal()); + LOG.info("principal: {} is trying to impersonate principal: {}", context + .realPrincipal(), context.principal()); if (impersonationAuthorizer == null) { - LOG.warn("impersonation attempt but {} has no authorizer configured. potential security risk, " - + "please see SECURITY.MD to learn how to configure impersonation authorizer.", + LOG.warn("impersonation attempt but {} has no authorizer configured. potential " + + "security risk, " + + "please see SECURITY.MD to learn how to configure impersonation " + + "authorizer.", DaemonConfig.NIMBUS_IMPERSONATION_AUTHORIZER); } else { if (!impersonationAuthorizer.permit(context, operation, checkConf)) { ThriftAccessLogger.logAccess(context.requestID(), context.remoteAddress(), context.principal(), operation, topoName, "access-denied"); throw new WrappedAuthorizationException("principal " + context.realPrincipal() - + " is not authorized to impersonate principal " + context.principal() + + " is not authorized to impersonate " + + "principal " + context.principal() + " from host " + context.remoteAddress() - + " Please see SECURITY.MD to learn how to configure impersonation acls."); + + " Please see SECURITY.MD to learn how to " + + "configure impersonation acls."); } } } @@ -2939,18 +3235,22 @@ public void checkAuthorization(String topoName, Map topoConf, St IAuthorizer aclHandler = authorizationHandler; if (aclHandler != null) { if (!aclHandler.permit(context, operation, checkConf)) { - ThriftAccessLogger.logAccess(context.requestID(), context.remoteAddress(), context.principal(), operation, + ThriftAccessLogger.logAccess(context.requestID(), context.remoteAddress(), context + .principal(), operation, topoName, "access-denied"); - throw new WrappedAuthorizationException(operation + (topoName != null ? " on topology " + topoName : "") + throw new WrappedAuthorizationException(operation + (topoName != null + ? " on topology " + topoName : "") + " is not authorized"); } else { - ThriftAccessLogger.logAccess(context.requestID(), context.remoteAddress(), context.principal(), + ThriftAccessLogger.logAccess(context.requestID(), context.remoteAddress(), context + .principal(), operation, topoName, "access-granted"); } } } - private boolean isAuthorized(String operation, String topoId) throws NotAliveException, AuthorizationException, IOException { + private boolean isAuthorized(String operation, + String topoId) throws NotAliveException, AuthorizationException, IOException { Map topoConf = tryReadTopoConf(topoId, topoCache); topoConf = Utils.merge(conf, topoConf); String topoName = (String) topoConf.get(Config.TOPOLOGY_NAME); @@ -2963,7 +3263,8 @@ private boolean isAuthorized(String operation, String topoId) throws NotAliveExc } @VisibleForTesting - public Set filterAuthorized(String operation, Collection topoIds) throws NotAliveException, + public Set filterAuthorized(String operation, + Collection topoIds) throws NotAliveException, AuthorizationException, IOException { Set ret = new HashSet<>(); for (String topoId : topoIds) { @@ -2975,23 +3276,33 @@ public Set filterAuthorized(String operation, Collection topoIds } /** - * Collect the dependency blob keys that are still referenced by a topology other than the ones being cleaned up. + * Collect the dependency blob keys that are still referenced by a topology other than the ones + * being cleaned up. * - *

      Dependency blobs uploaded by a current client are unique to a single submission, but a cluster upgraded from an - * older release can still hold artifact blobs whose key is derived from the maven coordinate alone, and those are - * listed by every topology that was submitted with the same artifact. Deleting one of those while a topology still - * needs it breaks worker launch and makes every nimbus refuse leadership, so the reference set has to be exact. + *

      Dependency blobs uploaded by a current client are unique to a single submission, but a + * cluster upgraded from an + * older release can still hold artifact blobs whose key is derived from the maven coordinate + * alone, and those are + * listed by every topology that was submitted with the same artifact. Deleting one of those + * while a topology still + * needs it breaks worker launch and makes every nimbus refuse leadership, so the reference set + * has to be exact. * - *

      This deliberately does not swallow read failures: a topology whose code blob cannot be read has unknown - * references, and treating it as referencing nothing would allow a live topology's dependencies to be deleted. Only - * a missing code blob is treated as contributing no references, because such a topology cannot be launched anyway. + *

      This deliberately does not swallow read failures: a topology whose code blob cannot be + * read has unknown + * references, and treating it as referencing nothing would allow a live topology's dependencies + * to be deleted. Only + * a missing code blob is treated as contributing no references, because such a topology cannot + * be launched anyway. * - * @param excludedTopoIds the topologies being cleaned up, which must not count as referencing anything + * @param excludedTopoIds the topologies being cleaned up, which must not count as referencing + * anything * @return every dependency blob key referenced by a topology that is not being cleaned up */ @VisibleForTesting Set referencedDependencyKeys(Set excludedTopoIds) throws Exception { - Set candidateTopoIds = new HashSet<>(Utils.OR(blobStore.storedTopoIds(), EMPTY_STRING_SET)); + Set candidateTopoIds = new HashSet<>(Utils.OR(blobStore.storedTopoIds(), + EMPTY_STRING_SET)); candidateTopoIds.addAll(Utils.OR(stormClusterState.activeStorms(), EMPTY_STRING_LIST)); candidateTopoIds.removeAll(excludedTopoIds); @@ -3001,8 +3312,10 @@ Set referencedDependencyKeys(Set excludedTopoIds) throws Excepti try { topo = readStormTopologyAsNimbus(topoId, topoCache); } catch (KeyNotFoundException e) { - //The topology has no code blob, so it references no dependencies and cannot be launched. - LOG.debug("No code found for {} while collecting dependency blob references", topoId); + // The topology has no code blob, so it references no dependencies and cannot be + // launched. + LOG.debug("No code found for {} while collecting dependency blob references", + topoId); continue; } if (topo.is_set_dependency_jars()) { @@ -3016,18 +3329,24 @@ Set referencedDependencyKeys(Set excludedTopoIds) throws Excepti } /** - * Tell whether a dependency blob key proves, by its shape alone, that no other topology can refer to it. + * Tell whether a dependency blob key proves, by its shape alone, that no other topology can + * refer to it. * - *

      A client that uploads a dependency splices a freshly generated UUID into the file name before prefixing it + *

      A client that uploads a dependency splices a freshly generated UUID into the file name + * before prefixing it * with {@code dep-}, so a key of that shape belongs to exactly one upload and therefore to exactly one topology. * Artifact keys written by older clients are derived from the maven coordinate alone * ({@code dep---.jar}) and are listed by every topology built against that artifact, so - * they do not match: a coordinate would have to end in five dash separated hexadecimal groups of exactly + * they do not match: a coordinate would have to end in five dash separated hexadecimal groups + * of exactly * 8-4-4-4-12 characters for that. * - *

      This lives here rather than next to the key generator on purpose. It is not a restatement of what the current - * client writes, but nimbus' own list of the key shapes it is willing to treat as unique, and it has to keep - * recognising the shapes written by every client version whose blobs may still be in the store even if the + *

      This lives here rather than next to the key generator on purpose. It is not a restatement + * of what the current + * client writes, but nimbus' own list of the key shapes it is willing to treat as unique, and + * it has to keep + * recognising the shapes written by every client version whose blobs may still be in the store + * even if the * generator changes. * * @param key a dependency blob key listed by a topology @@ -3039,16 +3358,21 @@ static boolean isProvablyUniqueDependencyKey(String key) { } /** - * Remove the dependency blobs of a topology that is being cleaned up, keeping the ones another topology still uses. + * Remove the dependency blobs of a topology that is being cleaned up, keeping the ones another + * topology still uses. * *

      When {@code stillReferenced} is null the cluster-wide reference scan failed, so it is unknown which blobs - * other topologies use. The topology's code blob is deleted right after this call and a dependency blob key - * carries no topology id, so anything left behind now can never be found again. Instead of giving up, the keys - * whose shape proves that no other topology can refer to them, that is the ones carrying a generated UUID, are + * other topologies use. The topology's code blob is deleted right after this call and a + * dependency blob key + * carries no topology id, so anything left behind now can never be found again. Instead of + * giving up, the keys + * whose shape proves that no other topology can refer to them, that is the ones carrying a + * generated UUID, are * still reclaimed; keys that could be shared are kept. * * @param topoId the topology being cleaned up - * @param stillReferenced the dependency blob keys referenced by topologies that are not being cleaned up, or null + * @param stillReferenced the dependency blob keys referenced by topologies that are not being + * cleaned up, or null * if that could not be determined */ @VisibleForTesting @@ -3070,18 +3394,22 @@ public void rmDependencyBlobsInTopology(String topoId, Set stillReferenc if (isProvablyUniqueDependencyKey(key)) { rmBlobKey(store, key, state); } else { - LOG.warn("Keeping dependency blob {} of {}, another topology may refer to it and the references " + LOG.warn("Keeping dependency blob {} of {}, another topology may refer to " + + "it and the references " + "could not be read", key, topoId); } } else if (stillReferenced.contains(key)) { - LOG.info("Keeping dependency blob {} of {}, it is still referenced by another topology", key, topoId); + LOG.info("Keeping dependency blob {} of {}, it is still referenced by another " + + "topology", key, topoId); } else { rmBlobKey(store, key, state); } } } catch (Exception e) { - //Yes eat the exception, cleaning up the rest of the topology matters more, but this leaves blobs behind. - LOG.warn("Could not remove the dependency blobs of {}, they will be left in the blob store", topoId, e); + // Yes eat the exception, cleaning up the rest of the topology matters more, but this + // leaves blobs behind. + LOG.warn("Could not remove the dependency blobs of {}, they will be left in the blob " + + "store", topoId, e); } } @@ -3092,12 +3420,12 @@ public void rmTopologyKeys(String topoId) { try { topoCache.deleteTopoConf(topoId, NIMBUS_SUBJECT); } catch (Exception e) { - //Just go on and try to delete the others + // Just go on and try to delete the others } try { topoCache.deleteTopology(topoId, NIMBUS_SUBJECT); } catch (Exception e) { - //Just go on and try to delte the others + // Just go on and try to delte the others } rmBlobKey(store, ConfigUtils.masterStormJarKey(topoId), state); } @@ -3108,20 +3436,30 @@ public void forceDeleteTopoDistDir(String topoId) throws IOException { } /** - * Remove the dependency blobs that no topology has referred to for longer than the inbox jar expiration. + * Remove the dependency blobs that no topology has referred to for longer than the inbox jar + * expiration. * - *

      {@link #rmDependencyBlobsInTopology} only runs in the pass that cleans up the owning topology, and a dependency - * blob key carries no topology id, so a blob that outlives that pass can never be traced back to it. That happens - * when a submission fails after its dependencies were uploaded, or when another nimbus still holds a copy of the blob + *

      {@link #rmDependencyBlobsInTopology} only runs in the pass that cleans up the owning + * topology, and a dependency + * blob key carries no topology id, so a blob that outlives that pass can never be traced back + * to it. That happens + * when a submission fails after its dependencies were uploaded, or when another nimbus still + * holds a copy of the blob * and it is downloaded back after it was removed here. This sweep reclaims those. * - *

      Only keys that are provably unique to one topology are considered: an older client that finds a shareable key - * in the store does not upload it again but refers to it, so such a key may be about to be used. A client uploads the - * dependencies before it submits the topology, so a key is only removed once it has been seen unreferenced for - * {@link DaemonConfig#NIMBUS_INBOX_JAR_EXPIRATION_SECS}, the time nimbus gives an uploaded topology jar to be - * submitted. When a key was first seen unreferenced is only kept in memory, so a new leader starts the wait over. + *

      Only keys that are provably unique to one topology are considered: an older client that + * finds a shareable key + * in the store does not upload it again but refers to it, so such a key may be about to be + * used. A client uploads the + * dependencies before it submits the topology, so a key is only removed once it has been seen + * unreferenced for + * {@link DaemonConfig#NIMBUS_INBOX_JAR_EXPIRATION_SECS}, the time nimbus gives an uploaded + * topology jar to be + * submitted. When a key was first seen unreferenced is only kept in memory, so a new leader + * starts the wait over. * - * @param referenced the dependency blob keys referenced by topologies that are not being cleaned up + * @param referenced the dependency blob keys referenced by topologies that are not being + * cleaned up */ @VisibleForTesting void sweepOrphanedDependencyBlobs(Set referenced) { @@ -3130,13 +3468,17 @@ void sweepOrphanedDependencyBlobs(Set referenced) { ObjectReader.getInt(conf.get(DaemonConfig.NIMBUS_INBOX_JAR_EXPIRATION_SECS), 3600)); long nowMs = Time.currentTimeMillis(); Set orphaned = blobStore.filterAndListKeys( - key -> isProvablyUniqueDependencyKey(key) && !referenced.contains(key) ? key : null); - //Forget the keys that are gone or referenced again, so that being orphaned later waits the full time again. + key -> isProvablyUniqueDependencyKey(key) && !referenced.contains(key) + ? key : null); + // Forget the keys that are gone or referenced again, so that being orphaned later waits + // the full time again. orphanedDependencyKeysDetectedMs.keySet().retainAll(orphaned); for (String key : orphaned) { - long unreferencedMs = nowMs - orphanedDependencyKeysDetectedMs.computeIfAbsent(key, k -> nowMs); + long unreferencedMs = nowMs - orphanedDependencyKeysDetectedMs.computeIfAbsent(key, + k -> nowMs); if (unreferencedMs >= graceMs) { - LOG.info("Removing dependency blob {}, no topology has referred to it for {} ms", key, unreferencedMs); + LOG.info("Removing dependency blob {}, no topology has referred to it for {} " + + "ms", key, unreferencedMs); rmBlobKey(blobStore, key, stormClusterState); orphanedDependencyKeysDetectedMs.remove(key); } @@ -3161,15 +3503,19 @@ public void doCleanup() { Set toClean = new HashSet<>(topoIdsToClean(state, blobStore, this.conf)); long topoIdSelectionDurationMs = Time.deltaMs(cleanupStartMs); - //Computed once for the whole pass, with the dying topologies excluded so that two of them sharing a - //dependency blob do not keep it alive for each other. A null result means the references are unknown and - //only the blobs that are provably unique to one topology are reclaimed below. + // Computed once for the whole pass, with the dying topologies excluded so that two of + // them sharing a + // dependency blob do not keep it alive for each other. A null result means the + // references are unknown and + // only the blobs that are provably unique to one topology are reclaimed below. Set stillReferenced = null; try { stillReferenced = referencedDependencyKeys(toClean); } catch (Exception e) { - LOG.warn("Could not determine which dependency blobs are still in use, only the dependency blobs whose " - + "key proves they belong to a single topology are reclaimed in this pass, the others are kept", e); + LOG.warn("Could not determine which dependency blobs are still in use, only the " + + "dependency blobs whose " + + "key proves they belong to a single topology are reclaimed in this pass, " + + "the others are kept", e); } for (String topoId : toClean) { @@ -3185,15 +3531,17 @@ public void doCleanup() { idToExecutors.getAndUpdate(new Dissoc<>(topoId)); } - //Catches the dependency blobs that outlived the pass that cleaned up their topology. Without knowing the - //references nothing can be told to be unused, so nothing is swept then. + // Catches the dependency blobs that outlived the pass that cleaned up their topology. + // Without knowing the + // references nothing can be told to be unused, so nothing is swept then. if (stillReferenced != null) { sweepOrphanedDependencyBlobs(stillReferenced); } long cleanupDurationMs = Time.deltaMs(cleanupStartMs); if (cleanupDurationMs > 10000) { - LOG.warn("doCleanup is taking too long, topoIdSelectionDurationMs={}, cleanupDurationMs={}", + LOG.warn("doCleanup is taking too long, topoIdSelectionDurationMs={}, " + + "cleanupDurationMs={}", topoIdSelectionDurationMs, cleanupDurationMs); } } catch (Exception ex) { @@ -3220,7 +3568,8 @@ private void addTopoToHistoryLog(String topoId, Map topoConf) { List users = ServerConfigUtils.getTopoLogsUsers(topoConf); List groups = ServerConfigUtils.getTopoLogsGroups(topoConf); synchronized (topologyHistoryLock) { - state.addTopologyHistory(new LSTopoHistory(topoId, Time.currentTimeSecs(), users, groups)); + state.addTopologyHistory(new LSTopoHistory(topoId, Time.currentTimeSecs(), users, + groups)); } } @@ -3249,8 +3598,10 @@ private boolean isUserPartOf(String user, Collection groupsToCheck) thro /** * Get the user whose topology history is to be returned. * - *

      The history is filtered for the caller authenticated on this request, not for the user named in the RPC - * argument. Only an admin (the ui daemon is expected to be one, see SECURITY.md) may ask for the history of + *

      The history is filtered for the caller authenticated on this request, not for the user + * named in the RPC + * argument. Only an admin (the ui daemon is expected to be one, see SECURITY.md) may ask for + * the history of * somebody else, because it serves the endpoint on behalf of its own authenticated web users. * * @param user the user asked for by the caller @@ -3265,26 +3616,29 @@ private String topologyHistoryUser(String user, Collection adminUsers, Collection adminGroups) throws AuthorizationException, IOException { Principal principal = ReqContext.context().principal(); if (principal == null) { - //security is off, there is no caller to filter by + // security is off, there is no caller to filter by return user; } String callerPrincipal = principal.getName(); String callerUser = principalToLocal.toLocal(principal); - if (adminUsers.contains(callerPrincipal) || adminUsers.contains(callerUser) || isUserPartOf(callerUser, adminGroups)) { + if (adminUsers.contains(callerPrincipal) || adminUsers.contains(callerUser) + || isUserPartOf(callerUser, adminGroups)) { return user; } if (user != null && !user.equals(callerPrincipal) && !user.equals(callerUser)) { - //Only an admin may ask for somebody else's history. Fall back to the caller's own - //rather than failing the call: a UI that is not in nimbus.admins asks on behalf of - //its web users, and answering with the caller's history keeps that page working. - LOG.warn("{} is not an admin and asked for the topology history of {}, returning its own history instead. " + // Only an admin may ask for somebody else's history. Fall back to the caller's own + // rather than failing the call: a UI that is not in nimbus.admins asks on behalf of + // its web users, and answering with the caller's history keeps that page working. + LOG.warn("{} is not an admin and asked for the topology history of {}, returning its " + + "own history instead. " + "Add {} to {} if it should be able to read the history of other users.", callerUser, user, callerPrincipal, Config.NIMBUS_ADMINS); } return callerUser; } - private List readTopologyHistory(String user, Collection adminUsers) throws IOException { + private List readTopologyHistory(String user, + Collection adminUsers) throws IOException { LocalState state = topologyHistoryState; List topoHistoryList = state.getTopoHistoryList(); if (topoHistoryList == null || topoHistoryList.isEmpty()) { @@ -3293,10 +3647,10 @@ private List readTopologyHistory(String user, Collection adminUs List ret = new ArrayList<>(); for (LSTopoHistory history : topoHistoryList) { - if (user == null || //Security off - adminUsers.contains(user) || //is admin - isUserPartOf(user, history.get_groups()) || //is in allowed group - history.get_users().contains(user)) { //is an allowed user + if (user == null // Security off + || adminUsers.contains(user) // is admin + || isUserPartOf(user, history.get_groups()) // is in allowed group + || history.get_users().contains(user)) { // is an allowed user ret.add(history.get_topology_id()); } } @@ -3315,17 +3669,19 @@ private void renewCredentials() throws Exception { for (Entry entry : assignedBases.entrySet()) { String id = entry.getKey(); String ownerPrincipal = entry.getValue().get_principal(); - Map topoConf = Collections.unmodifiableMap(Utils.merge(conf, tryReadTopoConf(id, topoCache))); + Map topoConf = Collections.unmodifiableMap(Utils.merge(conf, + tryReadTopoConf(id, topoCache))); synchronized (credUpdateLock) { Credentials origCreds = state.credentials(id, null); if (origCreds != null) { Map origCredsMap = origCreds.get_creds(); Map newCredsMap = new HashMap<>(origCredsMap); for (ICredentialsRenewer renewer : renewers) { - LOG.info("Renewing Creds For {} with {} owned by {}", id, renewer, ownerPrincipal); + LOG.info("Renewing Creds For {} with {} owned by {}", id, renewer, + ownerPrincipal); renewer.renew(newCredsMap, topoConf, ownerPrincipal); } - //Update worker tokens if needed + // Update worker tokens if needed upsertWorkerTokensInCreds(newCredsMap, ownerPrincipal, id); if (!newCredsMap.equals(origCredsMap)) { state.setCredentials(id, new Credentials(newCredsMap), topoConf); @@ -3354,7 +3710,8 @@ private SupervisorSummary makeSupervisorSummary(String supervisorId, SupervisorI } LOG.debug("NUM PORTS: {}", numPorts); SupervisorSummary ret = new SupervisorSummary(info.get_hostname(), - (int) info.get_uptime_secs(), numPorts, numUsedPorts, supervisorId); + (int) info + .get_uptime_secs(), numPorts, numUsedPorts, supervisorId); ret.set_total_resources(info.get_resources_map()); SupervisorResources resources = nodeIdToResources.get().get(supervisorId); if (resources != null && underlyingScheduler instanceof ResourceAwareScheduler) { @@ -3396,14 +3753,17 @@ private ClusterSummary getClusterInfoImpl() throws Exception { } int uptime = this.uptime.upTime(); List nimbuses = state.nimbuses(); - //update the isLeader field for each nimbus summary + // update the isLeader field for each nimbus summary NimbusInfo leader = leaderElector.getLeader(); for (NimbusSummary nimbusSummary : nimbuses) { nimbusSummary.set_uptime_secs(Time.deltaSecs(nimbusSummary.get_uptime_secs())); - // sometimes Leader election indicates the current nimbus is leader, but the host was recently restarted, + // sometimes Leader election indicates the current nimbus is leader, but the host was + // recently restarted, // and is currently not a leader. - boolean isLeader = leader.getHost().equals(nimbusSummary.get_host()) && leader.getPort() == nimbusSummary.get_port(); - if (isLeader && this.nimbusHostPortInfo.getHost().equals(leader.getHost()) && !this.isLeader()) { + boolean isLeader = leader.getHost().equals(nimbusSummary.get_host()) && leader + .getPort() == nimbusSummary.get_port(); + if (isLeader && this.nimbusHostPortInfo.getHost().equals(leader.getHost()) && !this + .isLeader()) { isLeader = false; } nimbusSummary.set_isLeader(isLeader); @@ -3431,7 +3791,8 @@ private List getTopologySummariesImpl() throws IOException, TEx return topologySummaries; } - private TopologySummary getTopologySummaryImpl(String topoId, StormBase base) throws IOException, TException { + private TopologySummary getTopologySummaryImpl(String topoId, + StormBase base) throws IOException, TException { IStormClusterState state = stormClusterState; Assignment assignment = state.assignmentInfo(topoId, null); @@ -3447,15 +3808,17 @@ private TopologySummary getTopologySummaryImpl(String topoId, StormBase base) th numWorkers = new HashSet<>(assignment.get_executor_node_port().values()).size(); } - TopologySummary summary = new TopologySummary(topoId, base.get_name(), numTasks, numExecutors, numWorkers, - Time.deltaSecs(base.get_launch_time_secs()), extractStatusStr(base)); + TopologySummary summary = new TopologySummary(topoId, base.get_name(), numTasks, + numExecutors, numWorkers, + Time.deltaSecs(base + .get_launch_time_secs()), extractStatusStr(base)); try { StormTopology topo = tryReadTopology(topoId, topoCache); if (topo != null && topo.is_set_storm_version()) { summary.set_storm_version(topo.get_storm_version()); } } catch (NotAliveException e) { - //Ignored it is not set + // Ignored it is not set } if (base.is_set_owner()) { @@ -3482,7 +3845,8 @@ private TopologySummary getTopologySummaryImpl(String topoId, StormBase base) th summary.set_assigned_generic_resources(resources.getAssignedGenericResources()); } try { - summary.set_replication_count(getBlobReplicationCount(ConfigUtils.masterStormCodeKey(topoId))); + summary.set_replication_count(getBlobReplicationCount(ConfigUtils + .masterStormCodeKey(topoId))); } catch (Exception e) { // This could fail if a blob gets deleted by mistake. Don't crash nimbus. LOG.error("Unable to find blob entry", e); @@ -3494,16 +3858,19 @@ private void sendClusterMetricsToExecutors() throws Exception { ClusterInfo clusterInfo = mkClusterInfo(); ClusterSummary clusterSummary = getClusterInfoImpl(); List clusterMetrics = extractClusterMetrics(clusterSummary); - Map> supervisorMetrics = extractSupervisorMetrics(clusterSummary); + Map> supervisorMetrics = + extractSupervisorMetrics(clusterSummary); for (ClusterMetricsConsumerExecutor consumerExecutor : clusterConsumerExceutors) { consumerExecutor.handleDataPoints(clusterInfo, clusterMetrics); - for (Entry> entry : supervisorMetrics.entrySet()) { + for (Entry> entry : supervisorMetrics.entrySet()) { consumerExecutor.handleDataPoints(entry.getKey(), entry.getValue()); } } } - private CommonTopoInfo getCommonTopoInfo(String topoId, String operation) throws NotAliveException, + private CommonTopoInfo getCommonTopoInfo(String topoId, + String operation) throws NotAliveException, AuthorizationException, IOException, InvalidTopologyException { CommonTopoInfo ret = new CommonTopoInfo(); ret.topoConf = tryReadTopoConf(topoId, topoCache); @@ -3520,8 +3887,10 @@ private CommonTopoInfo getCommonTopoInfo(String topoId, String operation) throws ret.launchTimeSecs = 0; } ret.assignment = state.assignmentInfo(topoId, null); - //get it from cluster state/zookeeper every time to collect the UI stats, may replace it with other StateStore later - ret.beats = ret.assignment != null ? StatsUtil.convertExecutorBeats(state.executorBeats(topoId, + // get it from cluster state/zookeeper every time to collect the UI stats, may replace it + // with other StateStore later + ret.beats = ret.assignment != null ? StatsUtil.convertExecutorBeats(state + .executorBeats(topoId, ret.assignment .get_executor_node_port())) : Collections @@ -3536,17 +3905,20 @@ public boolean awaitLeadership(long timeout, TimeUnit timeUnit) throws Interrupt } @Override - public void submitTopology(String name, String uploadedJarLocation, String jsonConf, StormTopology topology) + public void submitTopology(String name, String uploadedJarLocation, String jsonConf, + StormTopology topology) throws AlreadyAliveException, InvalidTopologyException, AuthorizationException, TException { submitTopologyCalls.mark(); - submitTopologyWithOpts(name, uploadedJarLocation, jsonConf, topology, new SubmitOptions(TopologyInitialStatus.ACTIVE)); + submitTopologyWithOpts(name, uploadedJarLocation, jsonConf, topology, + new SubmitOptions(TopologyInitialStatus.ACTIVE)); } - private void upsertWorkerTokensInCreds(Map creds, String user, String topologyId) { + private void upsertWorkerTokensInCreds(Map creds, String user, + String topologyId) { if (workerTokenManager != null) { workerTokenManager.upsertWorkerTokensInCredsForTopo(creds, user, topologyId); } - //Remove any expired keys after possibly inserting new ones. + // Remove any expired keys after possibly inserting new ones. stormClusterState.removeExpiredPrivateWorkerKeys(topologyId); } @@ -3558,7 +3930,8 @@ public void submitTopologyWithOpts(String topoName, String uploadedJarLocation, submitTopologyWithOptsCalls.mark(); assertIsLeader(); if (options == null) { - throw new InvalidTopologyException("Cannot submitTopologyWithOpts: SubmitOptions parameter value is null"); + throw new InvalidTopologyException("Cannot submitTopologyWithOpts: SubmitOptions " + + "parameter value is null"); } validateTopologyName(topoName); checkAuthorization(topoName, null, "submitTopology"); @@ -3574,14 +3947,18 @@ public void submitTopologyWithOpts(String topoName, String uploadedJarLocation, validateDependencyBlobKeys(topology, blobStore, getSubject()); if ((boolean) conf.getOrDefault(Config.DISABLE_SYMLINKS, false)) { @SuppressWarnings("unchecked") - Map blobMap = (Map) topoConf.get(Config.TOPOLOGY_BLOBSTORE_MAP); + Map blobMap = (Map) topoConf + .get(Config.TOPOLOGY_BLOBSTORE_MAP); if (blobMap != null && !blobMap.isEmpty()) { - throw new WrappedInvalidTopologyException("symlinks are disabled so blobs are not supported but " - + Config.TOPOLOGY_BLOBSTORE_MAP + " = " + blobMap); + throw new WrappedInvalidTopologyException("symlinks are disabled so blobs are " + + "not supported but " + + Config.TOPOLOGY_BLOBSTORE_MAP + " = " + + blobMap); } } ServerUtils.validateTopologyWorkerMaxHeapSizeConfigs(topoConf, topology, - ObjectReader.getDouble(conf.get(Config.TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB))); + ObjectReader.getDouble(conf + .get(Config.TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB))); Utils.validateTopologyBlobStoreMap(topoConf, blobStore); long uniqueNum = submittedCount.incrementAndGet(); String topoId = topoName + "-" + uniqueNum + "-" + Time.currentTimeSecs(); @@ -3597,8 +3974,10 @@ public void submitTopologyWithOpts(String topoName, String uploadedJarLocation, ReqContext req = ReqContext.context(); Principal principal = req.principal(); - String submitterPrincipal = principal == null ? null : principalToLocal.toLocal(principal); - Set topoAcl = new HashSet<>(ObjectReader.getStrings(topoConf.get(Config.TOPOLOGY_USERS))); + String submitterPrincipal = principal == null ? null : principalToLocal + .toLocal(principal); + Set topoAcl = new HashSet<>(ObjectReader.getStrings(topoConf + .get(Config.TOPOLOGY_USERS))); topoAcl.add(submitterPrincipal); String submitterUser = principalToLocal.toLocal(principal); topoAcl.add(submitterUser); @@ -3607,27 +3986,35 @@ public void submitTopologyWithOpts(String topoName, String uploadedJarLocation, topoConf.put(Config.TOPOLOGY_SUBMITTER_PRINCIPAL, topologyPrincipal); String systemUser = System.getProperty("user.name"); String topologyOwner = Utils.OR(submitterUser, systemUser); - topoConf.put(Config.TOPOLOGY_SUBMITTER_USER, topologyOwner); //Don't let the user set who we launch as + topoConf.put(Config.TOPOLOGY_SUBMITTER_USER, + topologyOwner); // Don't let the user set who we launch as topoConf.put(Config.TOPOLOGY_USERS, new ArrayList<>(topoAcl)); - topoConf.put(Config.STORM_ZOOKEEPER_SUPERACL, conf.get(Config.STORM_ZOOKEEPER_SUPERACL)); + topoConf.put(Config.STORM_ZOOKEEPER_SUPERACL, conf + .get(Config.STORM_ZOOKEEPER_SUPERACL)); if (!Utils.isZkAuthenticationConfiguredStormServer(conf)) { topoConf.remove(Config.STORM_ZOOKEEPER_TOPOLOGY_AUTH_SCHEME); topoConf.remove(Config.STORM_ZOOKEEPER_TOPOLOGY_AUTH_PAYLOAD); } - if (!(Boolean) conf.getOrDefault(DaemonConfig.STORM_TOPOLOGY_CLASSPATH_BEGINNING_ENABLED, false)) { + if (!(Boolean) conf + .getOrDefault(DaemonConfig.STORM_TOPOLOGY_CLASSPATH_BEGINNING_ENABLED, false)) { topoConf.remove(Config.TOPOLOGY_CLASSPATH_BEGINNING); } String topoVersionString = topology.get_storm_version(); if (topoVersionString == null) { - topoVersionString = (String) conf.getOrDefault(Config.SUPERVISOR_WORKER_DEFAULT_VERSION, VersionInfo.getVersion()); + topoVersionString = (String) conf + .getOrDefault(Config.SUPERVISOR_WORKER_DEFAULT_VERSION, VersionInfo + .getVersion()); } - //Check if we can run a topology with that version of storm. + // Check if we can run a topology with that version of storm. SimpleVersion topoVersion = new SimpleVersion(topoVersionString); - List cp = Utils.getCompatibleVersion(supervisorClasspaths, topoVersion, "classpath", null); + List cp = Utils.getCompatibleVersion(supervisorClasspaths, topoVersion, + "classpath", null); if (cp == null) { - throw new WrappedInvalidTopologyException("Topology submitted with storm version " + topoVersionString - + " but could not find a configured compatible version to use " + throw new WrappedInvalidTopologyException("Topology submitted with storm version " + + topoVersionString + + " but could not find a configured compatible " + + "version to use " + supervisorClasspaths.keySet()); } Map otherConf = Utils.getConfigFromClasspath(cp, conf); @@ -3635,32 +4022,38 @@ public void submitTopologyWithOpts(String topoName, String uploadedJarLocation, Map totalConf = Utils.merge(conf, totalConfToSave); - //When reading the conf in nimbus we want to fall back to our own settings + // When reading the conf in nimbus we want to fall back to our own settings // if the other config does not have it set. topology = normalizeTopology(totalConf, topology); // if the Resource Aware Scheduler is used, - // we might need to set the number of acker executors and eventlogger executors to be the estimated number of workers. + // we might need to set the number of acker executors and eventlogger executors to be + // the estimated number of workers. if (ServerUtils.isRas(conf)) { - int estimatedNumWorker = ServerUtils.getEstimatedWorkerCountForRasTopo(totalConf, topology); + int estimatedNumWorker = ServerUtils.getEstimatedWorkerCountForRasTopo(totalConf, + topology); setUpAckerExecutorConfigs(topoName, totalConfToSave, totalConf, estimatedNumWorker); - ServerUtils.validateTopologyAckerBundleResource(totalConfToSave, topology, topoName); + ServerUtils.validateTopologyAckerBundleResource(totalConfToSave, topology, + topoName); - int numEventLoggerExecs = ObjectReader.getInt(totalConf.get(Config.TOPOLOGY_EVENTLOGGER_EXECUTORS), estimatedNumWorker); + int numEventLoggerExecs = ObjectReader.getInt(totalConf + .get(Config.TOPOLOGY_EVENTLOGGER_EXECUTORS), estimatedNumWorker); totalConfToSave.put(Config.TOPOLOGY_EVENTLOGGER_EXECUTORS, numEventLoggerExecs); LOG.debug("Config {} set to: {} for topology: {}", Config.TOPOLOGY_EVENTLOGGER_EXECUTORS, numEventLoggerExecs, topoName); } - //Remove any configs that are specific to a host that might mess with the running topology. - totalConfToSave.remove(Config.STORM_LOCAL_HOSTNAME); //Don't override the host name, or everything looks like it is on nimbus + // Remove any configs that are specific to a host that might mess with the running + // topology. + totalConfToSave + .remove(Config.STORM_LOCAL_HOSTNAME); // Don't override the host name, or everything looks like it is on nimbus IStormClusterState state = stormClusterState; if (creds == null && workerTokenManager != null) { - //Make sure we can store the worker tokens even if no creds are provided. + // Make sure we can store the worker tokens even if no creds are provided. creds = new HashMap<>(); } if (creds != null) { @@ -3673,22 +4066,27 @@ public void submitTopologyWithOpts(String topoName, String uploadedJarLocation, if (ObjectReader.getBoolean(conf.get(Config.SUPERVISOR_RUN_WORKER_AS_USER), false) && (submitterUser == null || submitterUser.isEmpty())) { - throw new WrappedAuthorizationException("Could not determine the user to run this topology as."); + throw new WrappedAuthorizationException("Could not determine the user to run this " + + "topology as."); } - StormCommon.systemTopology(totalConf, topology); //this validates the structure of the topology + StormCommon.systemTopology(totalConf, + topology); // this validates the structure of the topology validateTopologySize(topoConf, conf, topology); if (Utils.isZkAuthenticationConfiguredStormServer(conf) && !Utils.isZkAuthenticationConfiguredTopology(topoConf)) { - throw new IllegalArgumentException("The cluster is configured for zookeeper authentication, but no payload was provided."); + throw new IllegalArgumentException("The cluster is configured for zookeeper " + + "authentication, but no payload was provided."); } LOG.info("Received topology submission for {} (storm-{} JDK-{}) with conf {}", topoName, - topoVersionString, topology.get_jdk_version(), ConfigUtils.maskPasswords(topoConf)); + topoVersionString, topology.get_jdk_version(), ConfigUtils + .maskPasswords(topoConf)); // lock protects against multiple topologies being submitted at once and // cleanup thread killing topology in b/w assignment and starting the topology synchronized (submitLock) { assertTopoActive(topoName, false); - //cred-update-lock is not needed here because creds are being added for the first time. + // cred-update-lock is not needed here because creds are being added for the first + // time. if (creds != null) { state.setCredentials(topoId, new Credentials(creds), topoConf); } @@ -3697,7 +4095,8 @@ public void submitTopologyWithOpts(String topoName, String uploadedJarLocation, waitForDesiredCodeReplication(totalConf, topoId); state.setupHeatbeats(topoId, topoConf); state.setupErrors(topoId, topoConf); - if (ObjectReader.getBoolean(totalConf.get(Config.TOPOLOGY_BACKPRESSURE_ENABLE), false)) { + if (ObjectReader.getBoolean(totalConf.get(Config.TOPOLOGY_BACKPRESSURE_ENABLE), + false)) { state.setupBackpressure(topoId, topoConf); } notifyTopologyActionListener(topoName, "submitTopology"); @@ -3710,10 +4109,12 @@ public void submitTopologyWithOpts(String topoName, String uploadedJarLocation, status = TopologyStatus.ACTIVE; break; default: - throw new IllegalArgumentException("Inital Status of " + options.get_initial_status() + " is not allowed."); + throw new IllegalArgumentException("Inital Status of " + options + .get_initial_status() + " is not allowed."); } - startTopology(topoName, topoId, status, topologyOwner, topologyPrincipal, totalConfToSave, topology); + startTopology(topoName, topoId, status, topologyOwner, topologyPrincipal, + totalConfToSave, topology); } } catch (Exception e) { LOG.warn("Topology submission exception. (topology name='{}')", topoName, e); @@ -3725,7 +4126,8 @@ public void submitTopologyWithOpts(String topoName, String uploadedJarLocation, } @VisibleForTesting - public static void setUpAckerExecutorConfigs(String topoName, Map totalConfToSave, + public static void setUpAckerExecutorConfigs(String topoName, Map totalConfToSave, Map totalConf, int estimatedNumWorker) { int numAckerExecs; @@ -3740,7 +4142,8 @@ public static void setUpAckerExecutorConfigs(String topoName, Map comps = new TreeSet<>(); comps.addAll(stormTopology.get_spouts().keySet()); comps.addAll(stormTopology.get_bolts().keySet()); - Map execOverrides = options.is_set_num_executors() ? options.get_num_executors() : Collections.emptyMap(); + Map execOverrides = options.is_set_num_executors() ? options + .get_num_executors() : Collections.emptyMap(); for (Map.Entry e : execOverrides.entrySet()) { String comp = e.getKey(); // validate non-system component ids if (!Utils.isSystemId(comp) && !comps.contains(comp)) { throw new WrappedInvalidTopologyException( - String.format("Invalid component %s for topology %s, valid values are %s", + String.format("Invalid component %s for topology %s, valid values are " + + "%s", comp, topoName, String.join(",", comps)) ); } // validate executor count for component Integer value = e.getValue(); if (value == null || value <= 0) { - throw new WrappedInvalidTopologyException("Number of executors must be greater than 0"); + throw new WrappedInvalidTopologyException("Number of executors must be " + + "greater than 0"); } } if (options.is_set_topology_conf_overrides()) { - Map topoConfigOverrides = Utils.parseJson(options.get_topology_conf_overrides()); - //Clean up some things the user should not set. (Not a security issue, just might confuse the topology) + Map topoConfigOverrides = Utils.parseJson(options + .get_topology_conf_overrides()); + // Clean up some things the user should not set. (Not a security issue, just might + // confuse the topology) topoConfigOverrides.remove(Config.TOPOLOGY_SUBMITTER_PRINCIPAL); topoConfigOverrides.remove(Config.TOPOLOGY_SUBMITTER_USER); topoConfigOverrides.remove(Config.STORM_ZOOKEEPER_SUPERACL); topoConfigOverrides.remove(Config.STORM_ZOOKEEPER_TOPOLOGY_AUTH_SCHEME); topoConfigOverrides.remove(Config.STORM_ZOOKEEPER_TOPOLOGY_AUTH_PAYLOAD); - if ((boolean) conf.getOrDefault(DaemonConfig.STORM_TOPOLOGY_CLASSPATH_BEGINNING_ENABLED, false)) { + if ((boolean) conf + .getOrDefault(DaemonConfig.STORM_TOPOLOGY_CLASSPATH_BEGINNING_ENABLED, + false)) { topoConfigOverrides.remove(Config.TOPOLOGY_CLASSPATH_BEGINNING); } topoConfigOverrides.remove(Config.STORM_LOCAL_HOSTNAME); - //Blobs referenced by the overrides have to be readable by the one asking for the rebalance, + // Blobs referenced by the overrides have to be readable by the one asking for the + // rebalance, // just like at submit time. Utils.validateTopologyBlobStoreMap(topoConfigOverrides, blobStore); options.set_topology_conf_overrides(JSONValue.toJSONString(topoConfigOverrides)); @@ -3917,7 +4328,8 @@ public void setLogConfig(String topoId, LogConfig config) throws TException { String loggerName = entry.getKey(); LogLevelAction action = logConfig.get_action(); if (loggerName.isEmpty()) { - throw new RuntimeException("Named loggers need a valid name. Use ROOT for the root logger"); + throw new RuntimeException("Named loggers need a valid name. Use ROOT for " + + "the root logger"); } switch (action) { case UPDATE: @@ -3931,7 +4343,7 @@ public void setLogConfig(String topoId, LogConfig config) throws TException { } break; default: - //NOOP + // NOOP break; } } @@ -3971,7 +4383,8 @@ public LogConfig getLogConfig(String topoId) throws TException { } @Override - public void debug(String topoName, String componentId, boolean enable, double samplingPercentage) + public void debug(String topoName, String componentId, boolean enable, + double samplingPercentage) throws NotAliveException, AuthorizationException, TException { debugCalls.mark(); try { @@ -3993,13 +4406,14 @@ public void debug(String topoName, String componentId, boolean enable, double sa options.set_samplingpct(spct); } StormBase updates = new StormBase(); - //For backwards compatability + // For backwards compatability updates.set_component_executors(Collections.emptyMap()); boolean hasCompId = componentId != null && !componentId.isEmpty(); String key = hasCompId ? componentId : topoId; updates.put_to_component_debug(key, options); - LOG.info("Nimbus setting debug to {} for storm-name '{}' storm-id '{}' sanpling pct '{}'" + LOG.info("Nimbus setting debug to {} for storm-name '{}' storm-id '{}' sanpling pct " + + "'{}'" + (hasCompId ? " component-id '" + componentId + "'" : ""), enable, topoName, topoId, spct); synchronized (submitLock) { @@ -4034,7 +4448,8 @@ public void setWorkerProfiler(String topoId, ProfileRequest profileRequest) thro } @Override - public List getComponentPendingProfileActions(String id, String componentId, ProfileAction action) + public List getComponentPendingProfileActions(String id, String componentId, + ProfileAction action) throws TException { try { getComponentPendingProfileActionsCalls.mark(); @@ -4042,14 +4457,17 @@ public List getComponentPendingProfileActions(String id, String Map, List> exec2hostPort = new HashMap<>(); if (info.assignment != null) { Map nodeToHost = info.assignment.get_node_host(); - for (Entry, NodeInfo> entry : info.assignment.get_executor_node_port().entrySet()) { + for (Entry, NodeInfo> entry : info.assignment.get_executor_node_port() + .entrySet()) { NodeInfo ni = entry.getValue(); - List hostPort = Arrays.asList(nodeToHost.get(ni.get_node()), ni.get_port_iterator().next().intValue()); + List hostPort = Arrays.asList(nodeToHost.get(ni.get_node()), ni + .get_port_iterator().next().intValue()); exec2hostPort.put(entry.getKey(), hostPort); } } List> nodeInfos = - StatsUtil.extractNodeInfosFromHbForComp(exec2hostPort, info.taskToComponent, false, componentId); + StatsUtil.extractNodeInfosFromHbForComp(exec2hostPort, info.taskToComponent, false, + componentId); List ret = new ArrayList<>(); for (Map ni : nodeInfos) { String niHost = (String) ni.get("host"); @@ -4060,7 +4478,8 @@ public List getComponentPendingProfileActions(String id, String String expectedHost = req.get_nodeInfo().get_node(); int expectedPort = req.get_nodeInfo().get_port_iterator().next().intValue(); ProfileAction expectedAction = req.get_action(); - if (niHost.equals(expectedHost) && niPort == expectedPort && action == expectedAction) { + if (niHost.equals(expectedHost) && niPort == expectedPort + && action == expectedAction) { long time = req.get_time_stamp(); if (time > reqTime) { reqTime = time; @@ -4072,7 +4491,8 @@ public List getComponentPendingProfileActions(String id, String ret.add(newestMatch); } } - LOG.info("Latest profile actions for topology {} component {} {}", id, componentId, ret); + LOG.info("Latest profile actions for topology {} component {} {}", id, componentId, + ret); return ret; } catch (Exception e) { LOG.warn("Get comp actions topology exception. (topology id='{}')", id, e); @@ -4106,24 +4526,28 @@ public void uploadNewCredentials(String topoName, Credentials credentials) if (p != null) { caller = principalToLocal.toLocal(p); } - // caller being null means that security is disabled (which why are we uploading credentials with security disabled??? + // caller being null means that security is disabled (which why are we uploading + // credentials with security disabled??? if (caller == null) { - LOG.warn("Please check you settings. Credentials are being uploaded to {} with security disabled.", topoId); + LOG.warn("Please check you settings. Credentials are being uploaded to {} with " + + "security disabled.", topoId); } else if (!realPrincipal.equals(caller) && !realUser.equals(caller)) { throw new AuthorizationException(topoId + " is expected to be owned by " + caller + " but is actually owned by " + realPrincipal); } - // topoOwner is just the owner the client expects, so it can only reject a mismatch, never stand in for the caller. + // topoOwner is just the owner the client expects, so it can only reject a mismatch, + // never stand in for the caller. if (credentials.is_set_topoOwner()) { String expectedOwner = credentials.get_topoOwner(); if (!expectedOwner.equals(realPrincipal) && !expectedOwner.equals(realUser)) { - throw new AuthorizationException(topoId + " is expected to be owned by " + expectedOwner + throw new AuthorizationException(topoId + " is expected to be owned by " + + expectedOwner + " but is actually owned by " + realPrincipal); } } synchronized (credUpdateLock) { - //Merge the old credentials so creds nimbus created are not lost. + // Merge the old credentials so creds nimbus created are not lost. // And in case the user forgot to upload something important this time. Credentials origCreds = state.credentials(topoId, null); if (origCreds != null) { @@ -4179,11 +4603,13 @@ public String beginUpdateBlob(String key) throws AuthorizationException, KeyNotF @SuppressWarnings("deprecation") @Override - public void uploadBlobChunk(String session, ByteBuffer chunk) throws AuthorizationException, TException { + public void uploadBlobChunk(String session, + ByteBuffer chunk) throws AuthorizationException, TException { try { OutputStream os = blobUploaders.get(session); if (os == null) { - throw new RuntimeException("Blob for session " + session + " does not exist (or timed out)"); + throw new RuntimeException("Blob for session " + session + + " does not exist (or timed out)"); } byte[] array = chunk.array(); int remaining = chunk.remaining(); @@ -4206,7 +4632,8 @@ public void finishBlobUpload(String session) throws AuthorizationException, TExc try { OutputStream os = blobUploaders.get(session); if (os == null) { - throw new RuntimeException("Blob for session " + session + " does not exist (or timed out)"); + throw new RuntimeException("Blob for session " + session + + " does not exist (or timed out)"); } os.close(); LOG.info("Finished uploading blob for session {}. Closing session.", session); @@ -4227,7 +4654,8 @@ public void cancelBlobUpload(String session) throws AuthorizationException, TExc try { AtomicOutputStream os = (AtomicOutputStream) blobUploaders.get(session); if (os == null) { - throw new RuntimeException("Blob for session " + session + " does not exist (or timed out)"); + throw new RuntimeException("Blob for session " + session + + " does not exist (or timed out)"); } os.cancel(); LOG.info("Canceled uploading blob for session {}. Closing session.", session); @@ -4299,7 +4727,8 @@ public ByteBuffer downloadBlobChunk(String session) throws AuthorizationExceptio try { BufferInputStream is = blobDownloaders.get(session); if (is == null) { - throw new RuntimeException("Blob for session " + session + " does not exist (or timed out)"); + throw new RuntimeException("Blob for session " + session + + " does not exist (or timed out)"); } byte[] ret = is.read(); if (ret.length == 0) { @@ -4325,14 +4754,16 @@ public void deleteBlob(String key) throws AuthorizationException, KeyNotFoundExc String topoName = ConfigUtils.getIdFromBlobKey(key); if (topoName != null) { if (isTopologyActiveOrActivating(stormClusterState, topoName)) { - String message = "Attempting to delete blob " + key + " from under active topology " + topoName; + String message = "Attempting to delete blob " + key + + " from under active topology " + topoName; LOG.warn(message); throw new WrappedIllegalStateException(message); } } String topoId = topologyUsingThisBlob(stormClusterState, topoCache, key); if (topoId != null) { - String message = "Attempting to delete active blob " + key + " used by topology " + topoId; + String message = "Attempting to delete active blob " + key + " used by topology " + + topoId; LOG.warn(message); throw new WrappedIllegalStateException(message); } @@ -4353,7 +4784,7 @@ public ListBlobsResult listBlobs(String session) throws TException { try { checkAuthorization(null, null, "listBlobs"); Iterator keyIt; - //Create a new session id if the user gave an empty session string. + // Create a new session id if the user gave an empty session string. // This is the use case when the user wishes to list blobs // starting from the beginning. if (session == null || session.isEmpty()) { @@ -4364,7 +4795,8 @@ public ListBlobsResult listBlobs(String session) throws TException { } if (keyIt == null) { - throw new RuntimeException("Blob list for session " + session + " does not exist (or timed out)"); + throw new RuntimeException("Blob list for session " + session + + " does not exist (or timed out)"); } if (!keyIt.hasNext()) { @@ -4378,7 +4810,8 @@ public ListBlobsResult listBlobs(String session) throws TException { ArrayList listChunk = new ArrayList<>(); while (listChunk.size() < 100 && keyIt.hasNext()) { String key = keyIt.next(); - //Only list the blobs whose metadata the caller may read, the same check getBlobMeta does. + // Only list the blobs whose metadata the caller may read, the same check + // getBlobMeta does. try { blobStore.getBlobMeta(key, who); listChunk.add(key); @@ -4435,8 +4868,10 @@ public void createStateInZookeeper(String key) throws TException { BlobStore store = blobStore; NimbusInfo ni = nimbusHostPortInfo; if (store instanceof LocalFsBlobStore) { - //A non-leader only registers its copy of a key the leader created. If zookeeper does not know the key - //any more it was deleted while the copy was downloaded, and registering it would bring it back. + // A non-leader only registers its copy of a key the leader created. If zookeeper + // does not know the key + // any more it was deleted while the copy was downloaded, and registering it would + // bring it back. state.setupBlob(key, ni, getVersionForKey(key, ni, zkClient, isLeader())); } LOG.debug("Created state in zookeeper {} {} {}", state, store, ni); @@ -4459,7 +4894,8 @@ public String beginFileUpload() throws AuthorizationException, TException { assertIsLeader(); checkAuthorization(null, null, "fileUpload"); String fileloc = getInbox() + "/stormjar-" + Utils.uuid() + ".jar"; - uploaders.put(fileloc, new TimedWritableByteChannel(Channels.newChannel(new FileOutputStream(fileloc)), fileUploadDuration)); + uploaders.put(fileloc, new TimedWritableByteChannel(Channels + .newChannel(new FileOutputStream(fileloc)), fileUploadDuration)); LOG.info("Uploading file from client to {}", fileloc); return fileloc; } catch (Exception e) { @@ -4473,7 +4909,8 @@ public String beginFileUpload() throws AuthorizationException, TException { @SuppressWarnings("deprecation") @Override - public void uploadChunk(String location, ByteBuffer chunk) throws AuthorizationException, TException { + public void uploadChunk(String location, + ByteBuffer chunk) throws AuthorizationException, TException { try { uploadChunkCalls.mark(); checkAuthorization(null, null, "fileUpload"); @@ -4590,7 +5027,8 @@ public TopologyInfo getTopologyInfoByName(String name) throws TException { private TopologyInfo getTopologyInfoByNameImpl(String name, GetInfoOptions options) throws NotAliveException, AuthorizationException, Exception { IStormClusterState state = stormClusterState; - String id = state.getTopoId(name).orElseThrow(() -> new WrappedNotAliveException(name + " is not alive")); + String id = state.getTopoId(name).orElseThrow(() -> new WrappedNotAliveException(name + + " is not alive")); return getTopologyInfoWithOptsImpl(id, options); } @@ -4625,7 +5063,8 @@ public TopologyInfo getTopologyInfoWithOpts(String topoId, GetInfoOptions option } } - private TopologyInfo getTopologyInfoWithOptsImpl(String topoId, GetInfoOptions options) throws NotAliveException, + private TopologyInfo getTopologyInfoWithOptsImpl(String topoId, + GetInfoOptions options) throws NotAliveException, AuthorizationException, InvalidTopologyException, Exception { CommonTopoInfo common = getCommonTopoInfo(topoId, "getTopologyInfo"); if (common.base == null) { @@ -4659,20 +5098,25 @@ private TopologyInfo getTopologyInfoWithOptsImpl(String topoId, GetInfoOptions o List summaries = new ArrayList<>(); if (common.assignment != null) { - for (Entry, NodeInfo> entry : common.assignment.get_executor_node_port().entrySet()) { + for (Entry, NodeInfo> entry : common.assignment.get_executor_node_port() + .entrySet()) { NodeInfo ni = entry.getValue(); ExecutorInfo execInfo = toExecInfo(entry.getKey()); Map nodeToHost = common.assignment.get_node_host(); - Map heartbeat = common.beats.get(ClientStatsUtil.convertExecutor(entry.getKey())); + Map heartbeat = common.beats.get(ClientStatsUtil + .convertExecutor(entry.getKey())); if (heartbeat == null) { heartbeat = Collections.emptyMap(); } ExecutorSummary summ = new ExecutorSummary(execInfo, - common.taskToComponent.get(execInfo.get_task_start()), - nodeToHost.get(ni.get_node()), ni.get_port_iterator().next().intValue(), - (Integer) heartbeat.getOrDefault("uptime", 0)); - - //heartbeats "stats" + common.taskToComponent.get(execInfo + .get_task_start()), + nodeToHost.get(ni.get_node()), ni + .get_port_iterator().next().intValue(), + (Integer) heartbeat + .getOrDefault("uptime", 0)); + + // heartbeats "stats" Map ex = (Map) heartbeat.get("stats"); if (ex != null) { ExecutorStats stats = StatsUtil.thriftifyExecutorStats(ex); @@ -4681,7 +5125,8 @@ private TopologyInfo getTopologyInfoWithOptsImpl(String topoId, GetInfoOptions o summaries.add(summ); } } - TopologyInfo topoInfo = new TopologyInfo(topoId, common.topoName, Time.deltaSecs(common.launchTimeSecs), + TopologyInfo topoInfo = new TopologyInfo(topoId, common.topoName, Time + .deltaSecs(common.launchTimeSecs), summaries, extractStatusStr(common.base), errors); if (common.topology.is_set_storm_version()) { topoInfo.set_storm_version(common.topology.get_storm_version()); @@ -4706,7 +5151,8 @@ private TopologyInfo getTopologyInfoWithOptsImpl(String topoId, GetInfoOptions o if (common.base.is_set_component_debug()) { topoInfo.set_component_debug(common.base.get_component_debug()); } - topoInfo.set_replication_count(getBlobReplicationCount(ConfigUtils.masterStormCodeKey(topoId))); + topoInfo.set_replication_count(getBlobReplicationCount(ConfigUtils + .masterStormCodeKey(topoId))); return topoInfo; } @@ -4727,7 +5173,8 @@ public TopologyPageInfo getTopologyPageInfo(String topoId, String window, boolea throw new WrappedNotAliveException(topoId); } String owner = base.get_owner(); - Map workerToResources = getWorkerResourcesForTopology(topoId); + Map workerToResources = + getWorkerResourcesForTopology(topoId); List workerSummaries = null; Map, List> exec2NodePort = new HashMap<>(); if (assignment != null) { @@ -4735,7 +5182,8 @@ public TopologyPageInfo getTopologyPageInfo(String topoId, String window, boolea Map nodeToHost = assignment.get_node_host(); for (Entry, NodeInfo> entry : execToNodeInfo.entrySet()) { NodeInfo ni = entry.getValue(); - List nodePort = Arrays.asList(ni.get_node(), ni.get_port_iterator().next()); + List nodePort = Arrays.asList(ni.get_node(), ni.get_port_iterator() + .next()); exec2NodePort.put(entry.getKey(), nodePort); } @@ -4747,7 +5195,7 @@ public TopologyPageInfo getTopologyPageInfo(String topoId, String window, boolea nodeToHost, workerToResources, includeSys, - true, //this is the topology page, so we know the user is authorized + true, // this is the topology page, so we know the user is authorized null, owner); } @@ -4794,26 +5242,40 @@ public TopologyPageInfo getTopologyPageInfo(String topoId, String window, boolea topoPageInfo.set_assigned_memonheap(resources.getAssignedMemOnHeap()); topoPageInfo.set_assigned_memoffheap(resources.getAssignedMemOffHeap()); topoPageInfo.set_assigned_cpu(resources.getAssignedCpu()); - topoPageInfo.set_requested_shared_off_heap_memory(resources.getRequestedSharedMemOffHeap()); - topoPageInfo.set_requested_regular_off_heap_memory(resources.getRequestedNonSharedMemOffHeap()); - topoPageInfo.set_requested_shared_on_heap_memory(resources.getRequestedSharedMemOnHeap()); - topoPageInfo.set_requested_regular_on_heap_memory(resources.getRequestedNonSharedMemOnHeap()); - topoPageInfo.set_assigned_shared_off_heap_memory(resources.getAssignedSharedMemOffHeap()); - topoPageInfo.set_assigned_regular_off_heap_memory(resources.getAssignedNonSharedMemOffHeap()); - topoPageInfo.set_assigned_shared_on_heap_memory(resources.getAssignedSharedMemOnHeap()); - topoPageInfo.set_assigned_regular_on_heap_memory(resources.getAssignedNonSharedMemOnHeap()); - topoPageInfo.set_assigned_generic_resources(resources.getAssignedGenericResources()); - topoPageInfo.set_requested_generic_resources(resources.getRequestedGenericResources()); + topoPageInfo.set_requested_shared_off_heap_memory(resources + .getRequestedSharedMemOffHeap()); + topoPageInfo.set_requested_regular_off_heap_memory(resources + .getRequestedNonSharedMemOffHeap()); + topoPageInfo.set_requested_shared_on_heap_memory(resources + .getRequestedSharedMemOnHeap()); + topoPageInfo.set_requested_regular_on_heap_memory(resources + .getRequestedNonSharedMemOnHeap()); + topoPageInfo.set_assigned_shared_off_heap_memory(resources + .getAssignedSharedMemOffHeap()); + topoPageInfo.set_assigned_regular_off_heap_memory(resources + .getAssignedNonSharedMemOffHeap()); + topoPageInfo.set_assigned_shared_on_heap_memory(resources + .getAssignedSharedMemOnHeap()); + topoPageInfo.set_assigned_regular_on_heap_memory(resources + .getAssignedNonSharedMemOnHeap()); + topoPageInfo.set_assigned_generic_resources(resources + .getAssignedGenericResources()); + topoPageInfo.set_requested_generic_resources(resources + .getRequestedGenericResources()); } int launchTimeSecs = common.launchTimeSecs; topoPageInfo.set_name(topoName); topoPageInfo.set_status(extractStatusStr(base)); topoPageInfo.set_uptime_secs(Time.deltaSecs(launchTimeSecs)); - // topoConf is the daemon conf merged with the topology conf, so it carries Nimbus secrets - // (the ZooKeeper digest payload, Thrift/Netty TLS store passwords) on top of the topology's own. - // getTopologyPageInfo is a topology read-only operation, so mask before it leaves Nimbus. + // topoConf is the daemon conf merged with the topology conf, so it carries Nimbus + // secrets + // (the ZooKeeper digest payload, Thrift/Netty TLS store passwords) on top of the + // topology's own. + // getTopologyPageInfo is a topology read-only operation, so mask before it leaves + // Nimbus. topoPageInfo.set_topology_conf(JSONValue.toJSONString(maskCredentialsForApi(topoConf))); - topoPageInfo.set_replication_count(getBlobReplicationCount(ConfigUtils.masterStormCodeKey(topoId))); + topoPageInfo.set_replication_count(getBlobReplicationCount(ConfigUtils + .masterStormCodeKey(topoId))); if (base.is_set_component_debug()) { DebugOptions debug = base.get_component_debug().get(topoId); if (debug != null) { @@ -4831,7 +5293,8 @@ public TopologyPageInfo getTopologyPageInfo(String topoId, String window, boolea } /** - * If aggStats are not populated, compute common and component(spout) agg and create placeholder stat. + * If aggStats are not populated, compute common and component(spout) agg and create placeholder + * stat. * This allow the topology page to show component spec even the topo is not scheduled. * Otherwise, just fetch data from current topoPageInfo. * @@ -4839,8 +5302,10 @@ public TopologyPageInfo getTopologyPageInfo(String topoId, String window, boolea * @param topology storm topology used to get spout names * @param topoConf storm topology config */ - private void addSpoutAggStats(TopologyPageInfo topoPageInfo, StormTopology topology, Map topoConf) { - Map spoutResources = ResourceUtils.getSpoutsResources(topology, topoConf); + private void addSpoutAggStats(TopologyPageInfo topoPageInfo, StormTopology topology, Map topoConf) { + Map spoutResources = ResourceUtils + .getSpoutsResources(topology, topoConf); // if agg stats were not populated yet, create placeholder if (topoPageInfo.get_id_to_spout_agg_stats().isEmpty()) { @@ -4854,7 +5319,8 @@ private void addSpoutAggStats(TopologyPageInfo topoPageInfo, StormTopology topol // common aggregate CommonAggregateStats commonStats = getPlaceholderCommonAggregateStats(spoutSpec); - commonStats.set_resources_map(spoutResources.getOrDefault(spoutName, new NormalizedResourceRequest()) + commonStats.set_resources_map(spoutResources.getOrDefault(spoutName, + new NormalizedResourceRequest()) .toNormalizedMap()); placeholderComponentStats.set_common_stats(commonStats); @@ -4867,7 +5333,8 @@ private void addSpoutAggStats(TopologyPageInfo topoPageInfo, StormTopology topol topoPageInfo.get_id_to_spout_agg_stats().put(spoutName, placeholderComponentStats); } } else { - for (Entry entry : topoPageInfo.get_id_to_spout_agg_stats().entrySet()) { + for (Entry entry : topoPageInfo + .get_id_to_spout_agg_stats().entrySet()) { CommonAggregateStats commonStats = entry.getValue().get_common_stats(); setResourcesDefaultIfNotSet(spoutResources, entry.getKey(), topoConf); commonStats.set_resources_map(spoutResources.get(entry.getKey()).toNormalizedMap()); @@ -4876,7 +5343,8 @@ private void addSpoutAggStats(TopologyPageInfo topoPageInfo, StormTopology topol } /** - * If aggStats are not populated, compute common and component(bolt) agg and create placeholder stat. + * If aggStats are not populated, compute common and component(bolt) agg and create placeholder + * stat. * This allow the topology page to show component spec even the topo is not scheduled. * Otherwise, just fetch data from current topoPageInfo. * @@ -4887,14 +5355,16 @@ private void addSpoutAggStats(TopologyPageInfo topoPageInfo, StormTopology topol */ private void addBoltAggStats(TopologyPageInfo topoPageInfo, StormTopology topology, Map topoConf, boolean includeSys) { - Map boltResources = ResourceUtils.getBoltsResources(topology, topoConf); + Map boltResources = ResourceUtils + .getBoltsResources(topology, topoConf); // if agg stats were not populated yet, create placeholder if (topoPageInfo.get_id_to_bolt_agg_stats().isEmpty()) { for (Entry entry : topology.get_bolts().entrySet()) { String boltName = entry.getKey(); Bolt bolt = entry.getValue(); - if ((!includeSys && Utils.isSystemId(boltName)) || boltName.equals(Constants.SYSTEM_COMPONENT_ID)) { + if ((!includeSys && Utils.isSystemId(boltName)) || boltName + .equals(Constants.SYSTEM_COMPONENT_ID)) { continue; } @@ -4904,7 +5374,8 @@ private void addBoltAggStats(TopologyPageInfo topoPageInfo, StormTopology topolo // common aggregate CommonAggregateStats commonStats = getPlaceholderCommonAggregateStats(bolt); - commonStats.set_resources_map(boltResources.getOrDefault(boltName, new NormalizedResourceRequest()) + commonStats.set_resources_map(boltResources.getOrDefault(boltName, + new NormalizedResourceRequest()) .toNormalizedMap()); placeholderComponentStats.set_common_stats(commonStats); @@ -4921,7 +5392,8 @@ private void addBoltAggStats(TopologyPageInfo topoPageInfo, StormTopology topolo topoPageInfo.get_id_to_bolt_agg_stats().put(boltName, placeholderComponentStats); } } else { - for (Entry entry : topoPageInfo.get_id_to_bolt_agg_stats().entrySet()) { + for (Entry entry : topoPageInfo + .get_id_to_bolt_agg_stats().entrySet()) { CommonAggregateStats commonStats = entry.getValue().get_common_stats(); setResourcesDefaultIfNotSet(boltResources, entry.getKey(), topoConf); commonStats.set_resources_map(boltResources.get(entry.getKey()).toNormalizedMap()); @@ -4943,7 +5415,8 @@ private CommonAggregateStats getPlaceholderCommonAggregateStats(Object component // get num_tasks Map jsonMap = StormCommon.componentConf(component); - int numTasks = ObjectReader.getInt(jsonMap.getOrDefault(Config.TOPOLOGY_TASKS, numExecutors)); + int numTasks = ObjectReader.getInt(jsonMap.getOrDefault(Config.TOPOLOGY_TASKS, + numExecutors)); commonStats.set_num_executors(numExecutors); commonStats.set_num_tasks(numTasks); @@ -4998,20 +5471,24 @@ public SupervisorPageInfo getSupervisorPageInfo(String superId, String host, boo Map, List> exec2NodePort = new HashMap<>(); Map nodeToHost; if (assignment != null) { - Map, NodeInfo> execToNodeInfo = assignment.get_executor_node_port(); + Map, NodeInfo> execToNodeInfo = assignment + .get_executor_node_port(); for (Entry, NodeInfo> entry : execToNodeInfo.entrySet()) { NodeInfo ni = entry.getValue(); - List nodePort = Arrays.asList(ni.get_node(), ni.get_port_iterator().next()); + List nodePort = Arrays.asList(ni.get_node(), ni + .get_port_iterator().next()); exec2NodePort.put(entry.getKey(), nodePort); } nodeToHost = assignment.get_node_host(); } else { nodeToHost = Collections.emptyMap(); } - Map workerResources = getWorkerResourcesForTopology(topoId); + Map workerResources = + getWorkerResourcesForTopology(topoId); boolean isAllowed = userTopologies.contains(topoId); String owner = (common.base == null) ? null : common.base.get_owner(); - for (WorkerSummary workerSummary : StatsUtil.aggWorkerStats(topoId, topoName, taskToComp, beats, + for (WorkerSummary workerSummary : StatsUtil.aggWorkerStats(topoId, topoName, + taskToComp, beats, exec2NodePort, nodeToHost, workerResources, includeSys, isAllowed, sid, owner)) { pageInfo.add_to_worker_summaries(workerSummary); @@ -5029,7 +5506,8 @@ public SupervisorPageInfo getSupervisorPageInfo(String superId, String host, boo } @Override - public ComponentPageInfo getComponentPageInfo(String topoId, String componentId, String window, boolean includeSys) + public ComponentPageInfo getComponentPageInfo(String topoId, String componentId, String window, + boolean includeSys) throws NotAliveException, AuthorizationException, TException { try { getComponentPageInfoCalls.mark(); @@ -5049,8 +5527,10 @@ public ComponentPageInfo getComponentPageInfo(String topoId, String componentId, nodeToHost = assignment.get_node_host(); for (Entry, NodeInfo> entry : execToNodeInfo.entrySet()) { NodeInfo ni = entry.getValue(); - List nodePort = Arrays.asList(ni.get_node(), ni.get_port_iterator().next()); - List hostPort = Arrays.asList(nodeToHost.get(ni.get_node()), ni.get_port_iterator().next()); + List nodePort = Arrays.asList(ni.get_node(), ni.get_port_iterator() + .next()); + List hostPort = Arrays.asList(nodeToHost.get(ni.get_node()), ni + .get_port_iterator().next()); exec2NodePort.put(entry.getKey(), nodePort); exec2HostPort.put(entry.getKey(), hostPort); } @@ -5059,16 +5539,19 @@ public ComponentPageInfo getComponentPageInfo(String topoId, String componentId, } String sanitizedComponentId = URLDecoder.decode(componentId, StandardCharsets.UTF_8); - ComponentPageInfo compPageInfo = StatsUtil.aggCompExecsStats(exec2HostPort, info.taskToComponent, info.beats, window, + ComponentPageInfo compPageInfo = StatsUtil.aggCompExecsStats(exec2HostPort, + info.taskToComponent, info.beats, window, includeSys, topoId, topology, sanitizedComponentId); if (compPageInfo.get_component_type() == ComponentType.SPOUT) { - NormalizedResourceRequest spoutResources = ResourceUtils.getSpoutResources(topology, topoConf, sanitizedComponentId); + NormalizedResourceRequest spoutResources = ResourceUtils.getSpoutResources(topology, + topoConf, sanitizedComponentId); if (spoutResources == null) { spoutResources = new NormalizedResourceRequest(topoConf, sanitizedComponentId); } compPageInfo.set_resources_map(spoutResources.toNormalizedMap()); - } else { //bolt - NormalizedResourceRequest boltResources = ResourceUtils.getBoltResources(topology, topoConf, sanitizedComponentId); + } else { // bolt + NormalizedResourceRequest boltResources = ResourceUtils.getBoltResources(topology, + topoConf, sanitizedComponentId); if (boltResources == null) { boltResources = new NormalizedResourceRequest(topoConf, sanitizedComponentId); } @@ -5089,7 +5572,8 @@ public ComponentPageInfo getComponentPageInfo(String topoId, String componentId, List tasks = compToTasks.get(StormCommon.EVENTLOGGER_COMPONENT_ID); tasks.sort(null); // Find the task the events from this component route to. - int taskIndex = TupleUtils.chooseTaskIndex(Collections.singletonList(sanitizedComponentId), tasks.size()); + int taskIndex = TupleUtils.chooseTaskIndex(Collections + .singletonList(sanitizedComponentId), tasks.size()); int taskId = tasks.get(taskIndex); String host = null; Integer port = null; @@ -5177,8 +5661,10 @@ public StormTopology getUserTopology(String id) throws NotAliveException, Author public TopologyHistoryInfo getTopologyHistory(String user) throws AuthorizationException, TException { try { checkAuthorization(null, null, "getTopologyHistory"); - List adminUsers = (List) conf.getOrDefault(Config.NIMBUS_ADMINS, Collections.emptyList()); - List adminGroups = (List) conf.getOrDefault(Config.NIMBUS_ADMINS_GROUPS, Collections.emptyList()); + List adminUsers = (List) conf.getOrDefault(Config.NIMBUS_ADMINS, + Collections.emptyList()); + List adminGroups = (List) conf.getOrDefault(Config.NIMBUS_ADMINS_GROUPS, + Collections.emptyList()); String historyUser = topologyHistoryUser(user, adminUsers, adminGroups); IStormClusterState state = stormClusterState; List assignedIds = state.assignments(null); @@ -5244,7 +5730,8 @@ public TopologySummary getTopologySummaryByName(String name) getTopologySummaryByNameCalls.mark(); checkAuthorization(name, null, "getTopologySummaryByName"); IStormClusterState state = stormClusterState; - String topoId = state.getTopoId(name).orElseThrow(() -> new WrappedNotAliveException(name + " is not alive")); + String topoId = state.getTopoId(name) + .orElseThrow(() -> new WrappedNotAliveException(name + " is not alive")); return getTopologySummaryImpl(topoId, state.topologyBases().get(topoId)); } catch (Exception e) { LOG.warn("Get TopologySummaryByName info exception.", e); @@ -5316,9 +5803,10 @@ public List getOwnerResourceSummaries(String owner) throws Map topoIdToBases = state.topologyBases(); Map clusterSchedulerConfig = scheduler.config(); - //put [owner-> StormBase-list] mapping to ownerToBasesMap - //if this owner (the input parameter) is null, add all the owners with stormbase and guarantees - //else, add only this owner (the input paramter) to the map + // put [owner-> StormBase-list] mapping to ownerToBasesMap + // if this owner (the input parameter) is null, add all the owners with stormbase and + // guarantees + // else, add only this owner (the input paramter) to the map Map> ownerToBasesMap = new HashMap<>(); if (owner == null) { @@ -5333,15 +5821,16 @@ public List getOwnerResourceSummaries(String owner) throws ownerToBasesMap.get(baseOwner).add(base); } } - //in addition, add all the owners with guarantees - List ownersWithGuarantees = new ArrayList<>(clusterSchedulerConfig.keySet()); + // in addition, add all the owners with guarantees + List ownersWithGuarantees = new ArrayList<>(clusterSchedulerConfig + .keySet()); for (String ownerWithGuarantees : ownersWithGuarantees) { if (!ownerToBasesMap.containsKey(ownerWithGuarantees)) { ownerToBasesMap.put(ownerWithGuarantees, new ArrayList<>()); } } } else { - //only put this owner to the map + // only put this owner to the map List stormbases = new ArrayList<>(); for (StormBase base : topoIdToBases.values()) { if (owner.equals(base.get_owner())) { @@ -5353,7 +5842,7 @@ public List getOwnerResourceSummaries(String owner) throws List ret = new ArrayList<>(); - //for each owner, get resources, configs, and aggregate + // for each owner, get resources, configs, and aggregate for (Entry> ownerToBasesEntry : ownerToBasesMap.entrySet()) { String theOwner = ownerToBasesEntry.getKey(); TopologyResources totalResourcesAggregate = new TopologyResources(); @@ -5368,10 +5857,14 @@ public List getOwnerResourceSummaries(String owner) throws TopologyResources resources = getResourcesForTopology(topoId, base); totalResourcesAggregate = totalResourcesAggregate.add(resources); Assignment ownerAssignment = topoIdToAssignments.get(topoId); - if (ownerAssignment != null && ownerAssignment.get_executor_node_port() != null) { - totalExecutors += ownerAssignment.get_executor_node_port().keySet().size(); - totalWorkers += new HashSet(ownerAssignment.get_executor_node_port().values()).size(); - for (List executorId : ownerAssignment.get_executor_node_port().keySet()) { + if (ownerAssignment != null && ownerAssignment + .get_executor_node_port() != null) { + totalExecutors += ownerAssignment.get_executor_node_port().keySet() + .size(); + totalWorkers += new HashSet(ownerAssignment.get_executor_node_port() + .values()).size(); + for (List executorId : ownerAssignment.get_executor_node_port() + .keySet()) { totalTasks += StormCommon.executorIdToTasks(executorId).size(); } } @@ -5392,26 +5885,38 @@ public List getOwnerResourceSummaries(String owner) throws ownerResourceSummary.set_total_tasks(totalTasks); ownerResourceSummary.set_memory_usage(assignedTotalMemory); ownerResourceSummary.set_cpu_usage(totalResourcesAggregate.getAssignedCpu()); - ownerResourceSummary.set_requested_on_heap_memory(totalResourcesAggregate.getRequestedMemOnHeap()); - ownerResourceSummary.set_requested_off_heap_memory(totalResourcesAggregate.getRequestedMemOffHeap()); + ownerResourceSummary.set_requested_on_heap_memory(totalResourcesAggregate + .getRequestedMemOnHeap()); + ownerResourceSummary.set_requested_off_heap_memory(totalResourcesAggregate + .getRequestedMemOffHeap()); ownerResourceSummary.set_requested_total_memory(requestedTotalMemory); ownerResourceSummary.set_requested_cpu(totalResourcesAggregate.getRequestedCpu()); - ownerResourceSummary.set_assigned_on_heap_memory(totalResourcesAggregate.getAssignedMemOnHeap()); - ownerResourceSummary.set_assigned_off_heap_memory(totalResourcesAggregate.getAssignedMemOffHeap()); + ownerResourceSummary.set_assigned_on_heap_memory(totalResourcesAggregate + .getAssignedMemOnHeap()); + ownerResourceSummary.set_assigned_off_heap_memory(totalResourcesAggregate + .getAssignedMemOffHeap()); if (clusterSchedulerConfig.containsKey(theOwner)) { if (underlyingScheduler instanceof ResourceAwareScheduler) { - Map schedulerConfig = (Map) clusterSchedulerConfig.get(theOwner); + Map schedulerConfig = (Map) clusterSchedulerConfig + .get(theOwner); if (schedulerConfig != null) { - ownerResourceSummary.set_memory_guarantee((double) schedulerConfig.getOrDefault("memory", 0)); - ownerResourceSummary.set_cpu_guarantee((double) schedulerConfig.getOrDefault("cpu", 0)); - ownerResourceSummary.set_memory_guarantee_remaining(ownerResourceSummary.get_memory_guarantee() + ownerResourceSummary.set_memory_guarantee((double) schedulerConfig + .getOrDefault("memory", 0)); + ownerResourceSummary.set_cpu_guarantee((double) schedulerConfig + .getOrDefault("cpu", 0)); + ownerResourceSummary.set_memory_guarantee_remaining(ownerResourceSummary + .get_memory_guarantee() - ownerResourceSummary.get_memory_usage()); - ownerResourceSummary.set_cpu_guarantee_remaining(ownerResourceSummary.get_cpu_guarantee() - - ownerResourceSummary.get_cpu_usage()); + ownerResourceSummary.set_cpu_guarantee_remaining(ownerResourceSummary + .get_cpu_guarantee() + - ownerResourceSummary + .get_cpu_usage()); } } else if (underlyingScheduler instanceof MultitenantScheduler) { - ownerResourceSummary.set_isolated_node_guarantee((int) clusterSchedulerConfig.getOrDefault(theOwner, 0)); + ownerResourceSummary + .set_isolated_node_guarantee((int) clusterSchedulerConfig + .getOrDefault(theOwner, 0)); } } @@ -5435,7 +5940,8 @@ public SupervisorAssignments getSupervisorAssignments(String nodeId) throws Auth try { if (isLeader() && isAssignmentsRecovered()) { SupervisorAssignments supervisorAssignments = new SupervisorAssignments(); - supervisorAssignments.set_storm_assignment(assignmentsForNodeId(stormClusterState.assignmentsInfo(), nodeId)); + supervisorAssignments.set_storm_assignment(assignmentsForNodeId(stormClusterState + .assignmentsInfo(), nodeId)); return supervisorAssignments; } } catch (Exception e) { @@ -5465,7 +5971,8 @@ public void sendSupervisorWorkerHeartbeats(SupervisorWorkerHeartbeats heartbeats if (e instanceof TException) { throw (TException) e; } - // When this master is not leader and get heartbeats report from supervisor/node, just ignore it. + // When this master is not leader and get heartbeats report from supervisor/node, just + // ignore it. } } @@ -5539,9 +6046,12 @@ public void processWorkerMetrics(WorkerMetrics metrics) throws TException { for (WorkerMetricPoint m : metrics.get_metricList().get_metrics()) { try { - Metric metric = new Metric(m.get_metricName(), m.get_timestamp(), metrics.get_topologyId(), - m.get_metricValue(), m.get_componentId(), m.get_executorId(), metrics.get_hostname(), - m.get_streamId(), metrics.get_port(), AggLevel.AGG_LEVEL_NONE); + Metric metric = new Metric(m.get_metricName(), m.get_timestamp(), metrics + .get_topologyId(), + m.get_metricValue(), m.get_componentId(), m + .get_executorId(), metrics.get_hostname(), + m.get_streamId(), metrics + .get_port(), AggLevel.AGG_LEVEL_NONE); this.metricsStore.insert(metric); } catch (Exception e) { LOG.error("Failed to save metric", e); @@ -5593,14 +6103,14 @@ public Map apply(Map t) { } } - //Daemon common methods + // Daemon common methods @VisibleForTesting public static class StandaloneINimbus implements INimbus { @Override public void prepare(Map topoConf, String schedulerLocalDir) { - //NOOP + // NOOP } @SuppressWarnings("unchecked") @@ -5618,8 +6128,9 @@ public Collection allSlotsAvailableForScheduling(Collection> newSlotsByTopologyId) { - //NOOP + public void assignSlots(Topologies topologies, Map> newSlotsByTopologyId) { + // NOOP } @Override @@ -5671,77 +6182,108 @@ private class ClusterSummaryMetricSet implements Runnable { private final ClusterSummaryMetrics clusterSummaryMetrics = new ClusterSummaryMetrics(); private final Function registerHistogram = (name) -> { - //This histogram reflects the data distribution across only one ClusterSummary, i.e., - // data distribution across all entities of a type (e.g., data from all nimbus/topologies) at one moment. - // Hence we use half of the CACHING_WINDOW time to ensure it retains only data from the most recent update - final Histogram histogram = new Histogram(new SlidingTimeWindowReservoir(CACHING_WINDOW / 2, TimeUnit.SECONDS)); + // This histogram reflects the data distribution across only one ClusterSummary, i.e., + // data distribution across all entities of a type (e.g., data from all + // nimbus/topologies) at one moment. + // Hence we use half of the CACHING_WINDOW time to ensure it retains only data from the + // most recent update + final Histogram histogram = + new Histogram(new SlidingTimeWindowReservoir(CACHING_WINDOW / 2, + TimeUnit.SECONDS)); clusterSummaryMetrics.put(name, histogram); return histogram; }; private volatile boolean active = false; - //NImbus metrics distribution + // NImbus metrics distribution private final Histogram nimbusUptime = registerHistogram.apply("nimbuses:uptime-secs"); - //Supervisor metrics distribution - private final Histogram supervisorsUptime = registerHistogram.apply("supervisors:uptime-secs"); - private final Histogram supervisorsNumWorkers = registerHistogram.apply("supervisors:num-workers"); - private final Histogram supervisorsNumUsedWorkers = registerHistogram.apply("supervisors:num-used-workers"); - private final Histogram supervisorsUsedMem = registerHistogram.apply("supervisors:used-mem"); - private final Histogram supervisorsUsedCpu = registerHistogram.apply("supervisors:used-cpu"); - private final Histogram supervisorsFragmentedMem = registerHistogram.apply("supervisors:fragmented-mem"); - private final Histogram supervisorsFragmentedCpu = registerHistogram.apply("supervisors:fragmented-cpu"); - - //Topology metrics distribution - private final Histogram topologiesNumTasks = registerHistogram.apply("topologies:num-tasks"); - private final Histogram topologiesNumExecutors = registerHistogram.apply("topologies:num-executors"); - private final Histogram topologiesNumWorker = registerHistogram.apply("topologies:num-workers"); - private final Histogram topologiesUptime = registerHistogram.apply("topologies:uptime-secs"); - private final Histogram topologiesReplicationCount = registerHistogram.apply("topologies:replication-count"); - private final Histogram topologiesRequestedMemOnHeap = registerHistogram.apply("topologies:requested-mem-on-heap"); - private final Histogram topologiesRequestedMemOffHeap = registerHistogram.apply("topologies:requested-mem-off-heap"); - private final Histogram topologiesRequestedCpu = registerHistogram.apply("topologies:requested-cpu"); - private final Histogram topologiesAssignedMemOnHeap = registerHistogram.apply("topologies:assigned-mem-on-heap"); - private final Histogram topologiesAssignedMemOffHeap = registerHistogram.apply("topologies:assigned-mem-off-heap"); - private final Histogram topologiesAssignedCpu = registerHistogram.apply("topologies:assigned-cpu"); + // Supervisor metrics distribution + private final Histogram supervisorsUptime = registerHistogram + .apply("supervisors:uptime-secs"); + private final Histogram supervisorsNumWorkers = registerHistogram + .apply("supervisors:num-workers"); + private final Histogram supervisorsNumUsedWorkers = registerHistogram + .apply("supervisors:num-used-workers"); + private final Histogram supervisorsUsedMem = registerHistogram + .apply("supervisors:used-mem"); + private final Histogram supervisorsUsedCpu = registerHistogram + .apply("supervisors:used-cpu"); + private final Histogram supervisorsFragmentedMem = registerHistogram + .apply("supervisors:fragmented-mem"); + private final Histogram supervisorsFragmentedCpu = registerHistogram + .apply("supervisors:fragmented-cpu"); + + // Topology metrics distribution + private final Histogram topologiesNumTasks = registerHistogram + .apply("topologies:num-tasks"); + private final Histogram topologiesNumExecutors = registerHistogram + .apply("topologies:num-executors"); + private final Histogram topologiesNumWorker = registerHistogram + .apply("topologies:num-workers"); + private final Histogram topologiesUptime = registerHistogram + .apply("topologies:uptime-secs"); + private final Histogram topologiesReplicationCount = registerHistogram + .apply("topologies:replication-count"); + private final Histogram topologiesRequestedMemOnHeap = registerHistogram + .apply("topologies:requested-mem-on-heap"); + private final Histogram topologiesRequestedMemOffHeap = registerHistogram + .apply("topologies:requested-mem-off-heap"); + private final Histogram topologiesRequestedCpu = registerHistogram + .apply("topologies:requested-cpu"); + private final Histogram topologiesAssignedMemOnHeap = registerHistogram + .apply("topologies:assigned-mem-on-heap"); + private final Histogram topologiesAssignedMemOffHeap = registerHistogram + .apply("topologies:assigned-mem-off-heap"); + private final Histogram topologiesAssignedCpu = registerHistogram + .apply("topologies:assigned-cpu"); private final StormMetricsRegistry metricsRegistry; /** * Constructor to put all items in ClusterSummary in MetricSet as a metric. * All metrics are derived from a cached ClusterSummary object, - * expired {@link ClusterSummaryMetricSet#CACHING_WINDOW} seconds after first query in a while from reporters. - * In case of {@link com.codahale.metrics.ScheduledReporter}, CACHING_WINDOW should be set shorter than + * expired {@link ClusterSummaryMetricSet#CACHING_WINDOW} seconds after first query in a + * while from reporters. + * In case of {@link com.codahale.metrics.ScheduledReporter}, CACHING_WINDOW should be set + * shorter than * reporting interval to avoid outdated reporting. */ ClusterSummaryMetricSet(StormMetricsRegistry metricsRegistry) { this.metricsRegistry = metricsRegistry; - //Break the code if out of sync to thrift protocol + // Break the code if out of sync to thrift protocol if (ClusterSummary._Fields.values().length != 3 - || ClusterSummary._Fields.findByName("supervisors") != ClusterSummary._Fields.SUPERVISORS - || ClusterSummary._Fields.findByName("topologies") != ClusterSummary._Fields.TOPOLOGIES - || ClusterSummary._Fields.findByName("nimbuses") != ClusterSummary._Fields.NIMBUSES) { + || ClusterSummary._Fields + .findByName("supervisors") != ClusterSummary._Fields.SUPERVISORS + || ClusterSummary._Fields + .findByName("topologies") != ClusterSummary._Fields.TOPOLOGIES + || ClusterSummary._Fields + .findByName("nimbuses") != ClusterSummary._Fields.NIMBUSES) { throw new AssertionError("Out of sync with thrift protocol"); } - final CachedGauge cachedSummary = new CachedGauge(CACHING_WINDOW, TimeUnit.SECONDS) { - @Override + final CachedGauge cachedSummary = + new CachedGauge(CACHING_WINDOW, TimeUnit.SECONDS) { + @Override protected ClusterSummary loadValue() { - try { - ClusterSummary newSummary = getClusterInfoImpl(); - LOG.debug("The new summary is {}", newSummary); - /* - * Update histograms based on the new summary. Most common implementation of Reporter reports Gauges before - * Histograms. Because DerivativeGauge will trigger cache refresh upon reporter's query, histogram will also be - * updated before query - */ - updateHistogram(newSummary); - return newSummary; - } catch (Exception e) { - LOG.warn("Get cluster info exception.", e); - throw new RuntimeException(e); + try { + ClusterSummary newSummary = getClusterInfoImpl(); + LOG.debug("The new summary is {}", newSummary); + /* + * Update histograms based on the new summary. Most common + * implementation of + * Reporter reports Gauges before + * Histograms. Because DerivativeGauge will trigger cache refresh upon + * reporter's query, histogram will also be + * updated before query + */ + updateHistogram(newSummary); + return newSummary; + } catch (Exception e) { + LOG.warn("Get cluster info exception.", e); + throw new RuntimeException(e); + } } - } - }; + }; clusterSummaryMetrics.put("cluster:num-nimbus-leaders", new DerivativeGauge(cachedSummary) { @@ -5796,8 +6338,9 @@ protected Integer transform(ClusterSummary clusterSummary) { @Override protected Double transform(ClusterSummary clusterSummary) { return clusterSummary.get_supervisors().stream() - //Filtered negative value - .mapToDouble(supervisorSummary -> Math.max(supervisorSummary.get_fragmented_mem(), 0)) + // Filtered negative value + .mapToDouble(supervisorSummary -> Math.max(supervisorSummary + .get_fragmented_mem(), 0)) .sum(); } }); @@ -5806,8 +6349,9 @@ protected Double transform(ClusterSummary clusterSummary) { @Override protected Double transform(ClusterSummary clusterSummary) { return clusterSummary.get_supervisors().stream() - //Filtered negative value - .mapToDouble(supervisorSummary -> Math.max(supervisorSummary.get_fragmented_cpu(), 0)) + // Filtered negative value + .mapToDouble(supervisorSummary -> Math.max(supervisorSummary + .get_fragmented_cpu(), 0)) .sum(); } }); @@ -5833,7 +6377,8 @@ private void updateHistogram(ClusterSummary newSummary) { topologiesUptime.update(summary.get_uptime_secs()); topologiesReplicationCount.update(summary.get_replication_count()); topologiesRequestedMemOnHeap.update(Math.round(summary.get_requested_memonheap())); - topologiesRequestedMemOffHeap.update(Math.round(summary.get_requested_memoffheap())); + topologiesRequestedMemOffHeap.update(Math.round(summary + .get_requested_memoffheap())); topologiesRequestedCpu.update(Math.round(summary.get_requested_cpu())); topologiesAssignedMemOnHeap.update(Math.round(summary.get_assigned_memonheap())); topologiesAssignedMemOffHeap.update(Math.round(summary.get_assigned_memoffheap())); diff --git a/storm-server/src/main/java/org/apache/storm/daemon/nimbus/TimedWritableByteChannel.java b/storm-server/src/main/java/org/apache/storm/daemon/nimbus/TimedWritableByteChannel.java index e179477fb0d..fa8833cfb17 100644 --- a/storm-server/src/main/java/org/apache/storm/daemon/nimbus/TimedWritableByteChannel.java +++ b/storm-server/src/main/java/org/apache/storm/daemon/nimbus/TimedWritableByteChannel.java @@ -1,15 +1,19 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. - * See the NOTICE file distributed with this work for additional information regarding copyright ownership. + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. + * See the NOTICE file distributed with this work for additional information regarding copyright + * ownership. * The ASF licenses this file to you under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. You may obtain a copy of the License at + * you may not use this file except in compliance with the License. You may obtain a copy of the + * License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, + *

      Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and limitations under the License. + * See the License for the specific language governing permissions and limitations under the + * License. */ package org.apache.storm.daemon.nimbus; @@ -18,7 +22,6 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.nio.channels.WritableByteChannel; - import org.apache.storm.metric.timed.TimedResource; public class TimedWritableByteChannel extends TimedResource implements WritableByteChannel { @@ -42,9 +45,9 @@ public void close() throws IOException { try { super.close(); } catch (Exception e) { - //WritableByteChannel is a Channel which implements Closeable. + // WritableByteChannel is a Channel which implements Closeable. // Hence although declared AutoCloseable super#close here should only throws IOException - //We rethrow to conform the signature + // We rethrow to conform the signature throw (IOException) e; } } diff --git a/storm-server/src/main/java/org/apache/storm/daemon/nimbus/TopoCache.java b/storm-server/src/main/java/org/apache/storm/daemon/nimbus/TopoCache.java index 505d77678ad..71af90ee5d0 100644 --- a/storm-server/src/main/java/org/apache/storm/daemon/nimbus/TopoCache.java +++ b/storm-server/src/main/java/org/apache/storm/daemon/nimbus/TopoCache.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

      - * http://www.apache.org/licenses/LICENSE-2.0 - *

      - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

      http://www.apache.org/licenses/LICENSE-2.0 + * + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -42,8 +48,10 @@ public class TopoCache { public static final Logger LOG = LoggerFactory.getLogger(TopoCache.class); private final BlobStore store; private final BlobStoreAclHandler aclHandler; - private final ConcurrentHashMap> topos = new ConcurrentHashMap<>(); - private final ConcurrentHashMap>> confs = new ConcurrentHashMap<>(); + private final ConcurrentHashMap> topos = + new ConcurrentHashMap<>(); + private final ConcurrentHashMap>> confs = + new ConcurrentHashMap<>(); public TopoCache(BlobStore store, Map conf) { this.store = store; @@ -52,6 +60,7 @@ public TopoCache(BlobStore store, Map conf) { /** * Read a topology. + * * @param topoId the id of the topology to read * @param who who to read it as * @return the deserialized topology. @@ -64,7 +73,7 @@ public StormTopology readTopology(final String topoId, final Subject who) final String key = ConfigUtils.masterStormCodeKey(topoId); WithAcl cached = topos.get(topoId); if (cached == null) { - //We need to read a new one + // We need to read a new one StormTopology topo = Utils.deserialize(store.readBlob(key, who), StormTopology.class); ReadableBlobMeta meta = store.getBlobMeta(key, who); cached = new WithAcl<>(meta.get_settable().get_acl(), topo); @@ -73,7 +82,7 @@ public StormTopology readTopology(final String topoId, final Subject who) cached = previous; } } else { - //Check if the user is allowed to read this + // Check if the user is allowed to read this aclHandler.hasPermissions(cached.acl, READ, who, key); } return cached.data; @@ -81,12 +90,14 @@ public StormTopology readTopology(final String topoId, final Subject who) /** * Delete a topology when we are done. + * * @param topoId the id of the topology * @param who who is deleting it * @throws AuthorizationException if who is not allowed to delete the blob * @throws KeyNotFoundException if the blob could not be found */ - public void deleteTopology(final String topoId, final Subject who) throws AuthorizationException, KeyNotFoundException { + public void deleteTopology(final String topoId, + final Subject who) throws AuthorizationException, KeyNotFoundException { final String key = ConfigUtils.masterStormCodeKey(topoId); store.deleteBlob(key, who); topos.remove(topoId); @@ -94,6 +105,7 @@ public void deleteTopology(final String topoId, final Subject who) throws Author /** * Add a new topology. + * * @param topoId the id of the topology * @param who who is doing it * @param topo the topology itself @@ -112,6 +124,7 @@ public void addTopology(final String topoId, final Subject who, final StormTopol /** * Update an existing topology . + * * @param topoId the id of the topology * @param who who is doing it * @param topo the new topology to save @@ -135,6 +148,7 @@ public void updateTopology(final String topoId, final Subject who, final StormTo /** * Read a topology conf. + * * @param topoId the id of the topology to read the conf for * @param who who to read it as * @return the deserialized config. @@ -147,7 +161,7 @@ public Map readTopoConf(final String topoId, final Subject who) final String key = ConfigUtils.masterStormConfKey(topoId); WithAcl> cached = confs.get(topoId); if (cached == null) { - //We need to read a new one + // We need to read a new one Map topoConf = Utils.fromCompressedJsonConf(store.readBlob(key, who)); ReadableBlobMeta meta = store.getBlobMeta(key, who); cached = new WithAcl<>(meta.get_settable().get_acl(), topoConf); @@ -156,7 +170,7 @@ public Map readTopoConf(final String topoId, final Subject who) cached = previous; } } else { - //Check if the user is allowed to read this + // Check if the user is allowed to read this aclHandler.hasPermissions(cached.acl, READ, who, key); } return cached.data; @@ -164,12 +178,14 @@ public Map readTopoConf(final String topoId, final Subject who) /** * Delete a topology conf when we are done. + * * @param topoId the id of the topology * @param who who is deleting it * @throws AuthorizationException if who is not allowed to delete the topo conf * @throws KeyNotFoundException if the topo conf is not found in the blob store */ - public void deleteTopoConf(final String topoId, final Subject who) throws AuthorizationException, KeyNotFoundException { + public void deleteTopoConf(final String topoId, + final Subject who) throws AuthorizationException, KeyNotFoundException { final String key = ConfigUtils.masterStormConfKey(topoId); store.deleteBlob(key, who); confs.remove(topoId); @@ -177,6 +193,7 @@ public void deleteTopoConf(final String topoId, final Subject who) throws Author /** * Add a new topology config. + * * @param topoId the id of the topology * @param who who is doing it * @param topoConf the topology conf itself @@ -184,7 +201,8 @@ public void deleteTopoConf(final String topoId, final Subject who) throws Author * @throws KeyAlreadyExistsException if the toplogy conf already exists in the blob store * @throws IOException on any error interacting with the blob store. */ - public void addTopoConf(final String topoId, final Subject who, final Map topoConf) + public void addTopoConf(final String topoId, final Subject who, final Map topoConf) throws AuthorizationException, KeyAlreadyExistsException, IOException { final String key = ConfigUtils.masterStormConfKey(topoId); final List acl = BlobStoreAclHandler.DEFAULT; @@ -195,6 +213,7 @@ public void addTopoConf(final String topoId, final Subject who, final Map topoConf) + public void updateTopoConf(final String topoId, final Subject who, final Map topoConf) throws AuthorizationException, KeyNotFoundException, IOException { final String key = ConfigUtils.masterStormConfKey(topoId); store.updateBlob(key, Utils.toCompressedJsonConf(topoConf), who); @@ -217,7 +237,8 @@ public void updateTopoConf(final String topoId, final Subject who, final Maphttp://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-server/src/main/java/org/apache/storm/daemon/nimbus/TopologyResources.java b/storm-server/src/main/java/org/apache/storm/daemon/nimbus/TopologyResources.java index 0daa7c70c0a..e33d4f6306d 100644 --- a/storm-server/src/main/java/org/apache/storm/daemon/nimbus/TopologyResources.java +++ b/storm-server/src/main/java/org/apache/storm/daemon/nimbus/TopologyResources.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

      - * http://www.apache.org/licenses/LICENSE-2.0 - *

      - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

      http://www.apache.org/licenses/LICENSE-2.0 + * + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -15,7 +21,6 @@ import java.util.Collection; import java.util.HashMap; import java.util.Map; - import org.apache.storm.generated.Assignment; import org.apache.storm.generated.NodeInfo; import org.apache.storm.generated.WorkerResources; @@ -83,33 +88,13 @@ private TopologyResources(TopologyDetails td, Collection worker } if (nodeIdToSharedOffHeapNode != null) { - double sharedOff = nodeIdToSharedOffHeapNode.values().stream().reduce(0.0, (sum, val) -> sum + val); + double sharedOff = nodeIdToSharedOffHeapNode.values().stream().reduce(0.0, (sum, + val) -> sum + val); assignedSharedMemOffHeap += sharedOff; assignedMemOffHeap += sharedOff; } } - private Map computeAssignedGenericResources(Collection workers) { - Map genericResources = new HashMap<>(); - for (WorkerResources worker : workers) { - genericResources = NormalizedResourceRequest.addResourceMap(genericResources, worker.get_resources()); - } - NormalizedResourceRequest.removeNonGenericResources(genericResources); - return genericResources; - } - - public TopologyResources(TopologyDetails td, SchedulerAssignment assignment) { - this(td, getWorkerResources(assignment), getNodeIdToSharedOffHeapNode(assignment)); - } - - public TopologyResources(TopologyDetails td, Assignment assignment) { - this(td, getWorkerResources(assignment), getNodeIdToSharedOffHeapNode(assignment)); - } - - public TopologyResources() { - this(0, 0, 0, 0, 0, 0, 0, new HashMap<>(), 0, 0, 0, 0, 0, 0, 0, new HashMap<>()); - } - protected TopologyResources( double requestedMemOnHeap, double requestedMemOffHeap, @@ -145,6 +130,28 @@ protected TopologyResources( this.assignedGenericResources = assignedGenericResources; } + public TopologyResources() { + this(0, 0, 0, 0, 0, 0, 0, new HashMap<>(), 0, 0, 0, 0, 0, 0, 0, new HashMap<>()); + } + + public TopologyResources(TopologyDetails td, Assignment assignment) { + this(td, getWorkerResources(assignment), getNodeIdToSharedOffHeapNode(assignment)); + } + + public TopologyResources(TopologyDetails td, SchedulerAssignment assignment) { + this(td, getWorkerResources(assignment), getNodeIdToSharedOffHeapNode(assignment)); + } + + private Map computeAssignedGenericResources(Collection workers) { + Map genericResources = new HashMap<>(); + for (WorkerResources worker : workers) { + genericResources = NormalizedResourceRequest.addResourceMap(genericResources, worker + .get_resources()); + } + NormalizedResourceRequest.removeNonGenericResources(genericResources); + return genericResources; + } + private static Collection getWorkerResources(SchedulerAssignment assignment) { Collection ret = null; if (assignment != null) { @@ -277,6 +284,7 @@ public Map getRequestedGenericResources() { /** * Add the values in other to this and return a combined resources object. + * * @param other the other resources to add to this * @return the combined resources with the sum of the values in each. */ @@ -289,7 +297,8 @@ public TopologyResources add(TopologyResources other) { requestedNonSharedMemOnHeap + other.requestedNonSharedMemOnHeap, requestedNonSharedMemOffHeap + other.requestedNonSharedMemOffHeap, requestedCpu + other.requestedCpu, - NormalizedResourceRequest.addResourceMap(requestedGenericResources, other.requestedGenericResources), + NormalizedResourceRequest.addResourceMap(requestedGenericResources, + other.requestedGenericResources), assignedMemOnHeap + other.assignedMemOnHeap, assignedMemOffHeap + other.assignedMemOffHeap, assignedSharedMemOnHeap + other.assignedSharedMemOnHeap, @@ -297,6 +306,7 @@ public TopologyResources add(TopologyResources other) { assignedNonSharedMemOnHeap + other.assignedNonSharedMemOnHeap, assignedNonSharedMemOffHeap + other.assignedNonSharedMemOffHeap, assignedCpu + other.assignedCpu, - NormalizedResourceRequest.addResourceMap(assignedGenericResources, other.assignedGenericResources)); + NormalizedResourceRequest.addResourceMap(assignedGenericResources, + other.assignedGenericResources)); } } diff --git a/storm-server/src/main/java/org/apache/storm/daemon/nimbus/TopologyStateTransition.java b/storm-server/src/main/java/org/apache/storm/daemon/nimbus/TopologyStateTransition.java index 5308df697f6..cfca7c34a5f 100644 --- a/storm-server/src/main/java/org/apache/storm/daemon/nimbus/TopologyStateTransition.java +++ b/storm-server/src/main/java/org/apache/storm/daemon/nimbus/TopologyStateTransition.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -19,5 +25,6 @@ */ interface TopologyStateTransition { - StormBase transition(Object argument, Nimbus nimbus, String topoId, StormBase base) throws Exception; + StormBase transition(Object argument, Nimbus nimbus, String topoId, + StormBase base) throws Exception; } diff --git a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/BasicContainer.java b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/BasicContainer.java index 35ce6c56f7e..e8073fa46f4 100644 --- a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/BasicContainer.java +++ b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/BasicContainer.java @@ -31,8 +31,8 @@ import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; -import java.util.Map; import java.util.Map.Entry; +import java.util.Map; import java.util.NavigableMap; import org.apache.commons.lang3.StringUtils; import org.apache.storm.Config; @@ -60,7 +60,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; - /** * A container that runs processes on the local box. */ @@ -88,18 +87,22 @@ public class BasicContainer extends Container { * @param supervisorId the ID of the supervisor this is a part of. * @param supervisorPort the thrift server port of the supervisor this is a part of. * @param port the port the container is on. Should be <= 0 if only a partial recovery - * @param assignment the assignment for this container. Should be null if only a partial recovery. - * @param resourceIsolationManager used to isolate resources for a container can be null if no isolation is used. + * @param assignment the assignment for this container. Should be null if only a partial + * recovery. + * @param resourceIsolationManager used to isolate resources for a container can be null if no + * isolation is used. * @param localState the local state of the supervisor. May be null if partial recovery * @param workerId the id of the worker to use. Must not be null if doing a partial recovery. * @param metricsRegistry The metrics registry. * @param containerMemoryTracker The shared memory tracker for the supervisor's containers */ - public BasicContainer(ContainerType type, Map conf, String supervisorId, int supervisorPort, + public BasicContainer(ContainerType type, Map conf, String supervisorId, + int supervisorPort, int port, LocalAssignment assignment, ResourceIsolationInterface resourceIsolationManager, LocalState localState, String workerId, StormMetricsRegistry metricsRegistry, ContainerMemoryTracker containerMemoryTracker) throws IOException { - this(type, conf, supervisorId, supervisorPort, port, assignment, resourceIsolationManager, localState, + this(type, conf, supervisorId, supervisorPort, port, assignment, resourceIsolationManager, + localState, workerId, metricsRegistry, containerMemoryTracker, null, null, null); } @@ -111,20 +114,24 @@ public BasicContainer(ContainerType type, Map conf, String super * @param supervisorId the ID of the supervisor this is a part of. * @param supervisorPort the thrift server port of the supervisor this is a part of. * @param port the port the container is on. Should be <= 0 if only a partial recovery - * @param assignment the assignment for this container. Should be null if only a partial recovery. - * @param resourceIsolationManager used to isolate resources for a container can be null if no isolation is used. + * @param assignment the assignment for this container. Should be null if only a partial + * recovery. + * @param resourceIsolationManager used to isolate resources for a container can be null if no + * isolation is used. * @param localState the local state of the supervisor. May be null if partial recovery * @param workerId the id of the worker to use. Must not be null if doing a partial recovery. * @param metricsRegistry The metrics registry. * @param containerMemoryTracker The shared memory tracker for the supervisor's containers * @param ops file system operations (mostly for testing) if null a new one is made - * @param topoConf the config of the topology (mostly for testing) if null and not a partial recovery the real conf is + * @param topoConf the config of the topology (mostly for testing) if null and not a partial + * recovery the real conf is * read. * @param profileCmd the command to use when profiling (used for testing) * @throws IOException on any error * @throws ContainerRecoveryException if the Container could not be recovered. */ - BasicContainer(ContainerType type, Map conf, String supervisorId, int supervisorPort, int port, + BasicContainer(ContainerType type, Map conf, String supervisorId, + int supervisorPort, int port, LocalAssignment assignment, ResourceIsolationInterface resourceIsolationManager, LocalState localState, String workerId, StormMetricsRegistry metricsRegistry, ContainerMemoryTracker containerMemoryTracker, Map topoConf, AdvancedFSOps ops, String profileCmd) throws IOException { @@ -145,7 +152,8 @@ public BasicContainer(ContainerType type, Map conf, String super } } if (wid == null) { - throw new ContainerRecoveryException("Could not find worker id for " + port + " " + assignment); + throw new ContainerRecoveryException("Could not find worker id for " + port + + " " + assignment); } LOG.info("Recovered Worker {}", wid); this.workerId = wid; @@ -155,11 +163,14 @@ public BasicContainer(ContainerType type, Map conf, String super } if (resourceIsolationManager instanceof OciContainerManager) { - //When we use OciContainerManager, we will only use the profiler configured in worker-launcher.cfg due to security reasons + // When we use OciContainerManager, we will only use the profiler configured in + // worker-launcher.cfg due to security reasons LOG.debug("Supervisor is using {} as the {}." - + "The profiler set at worker.profiler.script.path in worker-launcher.cfg is the only profiler to be used. " + + "The profiler set at worker.profiler.script.path in worker-launcher.cfg is " + + "the only profiler to be used. " + "Please make sure it is configured properly", - resourceIsolationManager.getClass().getName(), ResourceIsolationInterface.class.getName()); + resourceIsolationManager.getClass().getName(), ResourceIsolationInterface.class + .getName()); this.profileCmd = ""; } else { if (profileCmd == null) { @@ -170,14 +181,19 @@ public BasicContainer(ContainerType type, Map conf, String super } hardMemoryLimitMultiplier = - ObjectReader.getDouble(conf.get(DaemonConfig.STORM_SUPERVISOR_HARD_MEMORY_LIMIT_MULTIPLIER), 2.0); + ObjectReader.getDouble(conf + .get(DaemonConfig.STORM_SUPERVISOR_HARD_MEMORY_LIMIT_MULTIPLIER), 2.0); hardMemoryLimitOver = - ObjectReader.getInt(conf.get(DaemonConfig.STORM_SUPERVISOR_HARD_LIMIT_MEMORY_OVERAGE_MB), 0); - lowMemoryThresholdMb = ObjectReader.getInt(conf.get(DaemonConfig.STORM_SUPERVISOR_LOW_MEMORY_THRESHOLD_MB), 1024); + ObjectReader.getInt(conf + .get(DaemonConfig.STORM_SUPERVISOR_HARD_LIMIT_MEMORY_OVERAGE_MB), 0); + lowMemoryThresholdMb = ObjectReader.getInt(conf + .get(DaemonConfig.STORM_SUPERVISOR_LOW_MEMORY_THRESHOLD_MB), 1024); mediumMemoryThresholdMb = - ObjectReader.getInt(conf.get(DaemonConfig.STORM_SUPERVISOR_MEDIUM_MEMORY_THRESHOLD_MB), 1536); + ObjectReader.getInt(conf.get(DaemonConfig.STORM_SUPERVISOR_MEDIUM_MEMORY_THRESHOLD_MB), + 1536); mediumMemoryGracePeriodMs = - ObjectReader.getInt(conf.get(DaemonConfig.STORM_SUPERVISOR_MEDIUM_MEMORY_GRACE_PERIOD_MS), 20_000); + ObjectReader.getInt(conf + .get(DaemonConfig.STORM_SUPERVISOR_MEDIUM_MEMORY_GRACE_PERIOD_MS), 20_000); if (assignment != null) { WorkerResources resources = assignment.get_resources(); @@ -186,7 +202,8 @@ public BasicContainer(ContainerType type, Map conf, String super } private static void removeWorkersOn(Map workerToPort, int port) { - for (Iterator> i = workerToPort.entrySet().iterator(); i.hasNext(); ) { + for (Iterator> i = workerToPort.entrySet().iterator(); i + .hasNext(); ) { Entry found = i.next(); if (port == found.getValue()) { LOG.warn("Deleting worker {} from state", found.getKey()); @@ -195,24 +212,28 @@ private static void removeWorkersOn(Map workerToPort, int port) } } - public static List getDependencyLocationsFor(final Map conf, final String topologyId, final AdvancedFSOps ops, + public static List getDependencyLocationsFor(final Map conf, + final String topologyId, final AdvancedFSOps ops, String stormRoot) throws IOException { return TOPO_META_CACHE.get(conf, topologyId, ops, stormRoot).getDepLocs(); } - public static String getStormVersionFor(final Map conf, final String topologyId, final AdvancedFSOps ops, + public static String getStormVersionFor(final Map conf, final String topologyId, + final AdvancedFSOps ops, String stormRoot) throws IOException { return TOPO_META_CACHE.get(conf, topologyId, ops, stormRoot).getStormVersion(); } /** - * Create a new worker ID for this process and store in in this object and in the local state. Never call this if a worker is currently + * Create a new worker ID for this process and store in in this object and in the local state. + * Never call this if a worker is currently * up and running. We will lose track of the process. */ protected void createNewWorkerId() { type.assertFull(); if (workerId != null) { - String err = "Incorrect usage of createNewWorkerId(), current workerId is " + workerId + ", expecting null"; + String err = "Incorrect usage of createNewWorkerId(), current workerId is " + workerId + + ", expecting null"; LOG.error(err); throw new AssertionError(err); } @@ -249,7 +270,7 @@ public void cleanUpForRestart() throws IOException { @Override public void relaunch() throws IOException { type.assertFull(); - //We are launching it now... + // We are launching it now... type = ContainerType.LAUNCH; createNewWorkerId(); setup(); @@ -262,7 +283,8 @@ public boolean didMainProcessExit() { } @Override - public boolean runProfiling(ProfileRequest request, boolean stop) throws IOException, InterruptedException { + public boolean runProfiling(ProfileRequest request, + boolean stop) throws IOException, InterruptedException { type.assertFull(); String targetDir = ConfigUtils.workerArtifactsRoot(conf, topologyId, port); @@ -277,14 +299,16 @@ public boolean runProfiling(ProfileRequest request, boolean stop) throws IOExcep String workerPid = ops.slurpString(new File(str)).trim(); ProfileAction profileAction = request.get_action(); - String logPrefix = "ProfilerAction process " + topologyId + ":" + port + " PROFILER_ACTION: " + profileAction + String logPrefix = "ProfilerAction process " + topologyId + ":" + port + + " PROFILER_ACTION: " + profileAction + " "; List command = mkProfileCommand(profileAction, stop, workerPid, targetDir); File targetFile = new File(targetDir); if (command.size() > 0) { - return resourceIsolationManager.runProfilingCommand(getWorkerUser(), workerId, command, env, logPrefix, targetFile); + return resourceIsolationManager.runProfilingCommand(getWorkerUser(), workerId, command, + env, logPrefix, targetFile); } LOG.warn("PROFILING REQUEST NOT SUPPORTED {} IGNORED...", request); return true; @@ -299,7 +323,8 @@ public boolean runProfiling(ProfileRequest request, boolean stop) throws IOExcep * @param targetDir the current working directory of the worker process * @return the command to run for profiling. */ - private List mkProfileCommand(ProfileAction action, boolean stop, String workerPid, String targetDir) { + private List mkProfileCommand(ProfileAction action, boolean stop, String workerPid, + String targetDir) { switch (action) { case JMAP_DUMP: return jmapDumpCmd(workerPid, targetDir); @@ -344,7 +369,8 @@ private List jprofileJvmRestart(String pid) { } /** - * Compute the java.library.path that should be used for the worker. This helps it to load JNI libraries that are packaged in the uber + * Compute the java.library.path that should be used for the worker. This helps it to load JNI + * libraries that are packaged in the uber * jar. * * @param stormRoot the root directory of the worker process @@ -362,7 +388,8 @@ protected String javaLibraryPath(String stormRoot, Map conf) { } /** - * Returns a path with a wildcard as the final element, so that the JVM will expand that to all JARs in the directory. + * Returns a path with a wildcard as the final element, so that the JVM will expand that to all + * JARs in the directory. * * @param dir the directory to which a wildcard will be appended * @return the path with wildcard ("*") suffix @@ -389,7 +416,8 @@ protected List frameworkClasspath(SimpleVersion topoVersion) { pathElements.add(extcp); pathElements.add(topoConfDir); - NavigableMap> classpaths = Utils.getConfiguredClasspathVersions(conf, pathElements); + NavigableMap> classpaths = Utils + .getConfiguredClasspathVersions(conf, pathElements); return Utils.getCompatibleVersion(classpaths, topoVersion, "classpath", pathElements); } @@ -397,24 +425,27 @@ protected List frameworkClasspath(SimpleVersion topoVersion) { protected String getWorkerMain(SimpleVersion topoVersion) { String defaultWorkerGuess = "org.apache.storm.daemon.worker.Worker"; if (topoVersion.getMajor() == 0) { - //Prior to the org.apache change + // Prior to the org.apache change defaultWorkerGuess = "backtype.storm.daemon.worker"; } else if (topoVersion.getMajor() == 1) { - //Have not moved to a java worker yet + // Have not moved to a java worker yet defaultWorkerGuess = "org.apache.storm.daemon.worker"; } NavigableMap mains = Utils.getConfiguredWorkerMainVersions(conf); - return Utils.getCompatibleVersion(mains, topoVersion, "worker main class", defaultWorkerGuess); + return Utils.getCompatibleVersion(mains, topoVersion, "worker main class", + defaultWorkerGuess); } protected String getWorkerLogWriter(SimpleVersion topoVersion) { String defaultGuess = "org.apache.storm.LogWriter"; if (topoVersion.getMajor() == 0) { - //Prior to the org.apache change + // Prior to the org.apache change defaultGuess = "backtype.storm.LogWriter"; } - NavigableMap mains = Utils.getConfiguredWorkerLogWriterVersions(conf); - return Utils.getCompatibleVersion(mains, topoVersion, "worker log writer class", defaultGuess); + NavigableMap mains = Utils + .getConfiguredWorkerLogWriterVersions(conf); + return Utils.getCompatibleVersion(mains, topoVersion, "worker log writer class", + defaultGuess); } @SuppressWarnings("unchecked") @@ -435,7 +466,8 @@ private List asStringList(Object o) { * @param topoVersion the version of the storm framework to use * @return the full classpath */ - protected String getWorkerClassPath(String stormJar, List dependencyLocations, SimpleVersion topoVersion) { + protected String getWorkerClassPath(String stormJar, List dependencyLocations, + SimpleVersion topoVersion) { List workercp = new ArrayList<>(); workercp.addAll(asStringList(topoConf.get(Config.TOPOLOGY_CLASSPATH_BEGINNING))); workercp.addAll(frameworkClasspath(topoVersion)); @@ -519,10 +551,13 @@ private String getWorkerLoggingConfigFile() { * * @throws IOException on any error. */ - private List getClassPathParams(final String stormRoot, final SimpleVersion topoVersion) throws IOException { + private List getClassPathParams(final String stormRoot, + final SimpleVersion topoVersion) throws IOException { final String stormJar = ConfigUtils.supervisorStormJarPath(stormRoot); - final List dependencyLocations = getDependencyLocationsFor(conf, topologyId, ops, stormRoot); - final String workerClassPath = getWorkerClassPath(stormJar, dependencyLocations, topoVersion); + final List dependencyLocations = getDependencyLocationsFor(conf, topologyId, ops, + stormRoot); + final String workerClassPath = getWorkerClassPath(stormJar, dependencyLocations, + topoVersion); List classPathParams = new ArrayList<>(); classPathParams.add("-cp"); @@ -531,7 +566,8 @@ private List getClassPathParams(final String stormRoot, final SimpleVers } /** - * Get a set of java properties that are common to both the log writer and the worker processes. These are mostly system properties that + * Get a set of java properties that are common to both the log writer and the worker processes. + * These are mostly system properties that * are used by logging. * * @return a list of command line options @@ -541,7 +577,8 @@ private List getCommonParams() { String stormLogDir = ConfigUtils.getLogDir(); List commonParams = new ArrayList<>(); - commonParams.add("-Dlogging.sensitivity=" + OR((String) topoConf.get(Config.TOPOLOGY_LOGGING_SENSITIVITY), "S3")); + commonParams.add("-Dlogging.sensitivity=" + OR((String) topoConf + .get(Config.TOPOLOGY_LOGGING_SENSITIVITY), "S3")); commonParams.add("-Dlogfile.name=worker.log"); commonParams.add("-Dstorm.home=" + OR(stormHome, "")); commonParams.add("-Dworkers.artifacts=" + workersArtifacts); @@ -549,7 +586,9 @@ private List getCommonParams() { commonParams.add("-Dworker.id=" + workerId); commonParams.add("-Dworker.port=" + port); commonParams.add("-Dstorm.log.dir=" + stormLogDir); - commonParams.add("-DLog4jContextSelector=org.apache.logging.log4j.core.selector.BasicContextSelector"); + commonParams + .add("-DLog4jContextSelector=org.apache.logging.log4j.core.selector.BasicContextSe" + + "lector"); commonParams.add("-Dstorm.local.dir=" + conf.get(Config.STORM_LOCAL_DIR)); if (memoryLimitMb > 0) { commonParams.add("-Dworker.memory_limit_mb=" + memoryLimitMb); @@ -572,7 +611,8 @@ private int getMemOnHeap(WorkerResources resources) { private int getMemOffHeap(WorkerResources resources) { int memOffheap = 0; - if (resources != null && resources.is_set_mem_off_heap() && resources.get_mem_off_heap() > 0) { + if (resources != null && resources.is_set_mem_off_heap() && resources + .get_mem_off_heap() > 0) { memOffheap = (int) Math.ceil(resources.get_mem_off_heap()); } return memOffheap; @@ -609,15 +649,19 @@ protected String javaCmd(String cmd) { * * @throws IOException on any error. */ - private List mkLaunchCommand(final int memOnheap, final int memOffheap, final String stormRoot, + private List mkLaunchCommand(final int memOnheap, final int memOffheap, + final String stormRoot, final String jlp, final String numaId) throws IOException { final String javaCmd = javaCmd("java"); - final String stormOptions = ConfigUtils.concatIfNotNull(System.getProperty("storm.options")); - final String topoConfFile = ConfigUtils.concatIfNotNull(System.getProperty("storm.conf.file")); + final String stormOptions = ConfigUtils.concatIfNotNull(System + .getProperty("storm.options")); + final String topoConfFile = ConfigUtils.concatIfNotNull(System + .getProperty("storm.conf.file")); final String workerTmpDir = ConfigUtils.workerTmpRoot(conf, workerId); String topoVersionString = getStormVersionFor(conf, topologyId, ops, stormRoot); if (topoVersionString == null) { - topoVersionString = (String) conf.getOrDefault(Config.SUPERVISOR_WORKER_DEFAULT_VERSION, VersionInfo.getVersion()); + topoVersionString = (String) conf.getOrDefault(Config.SUPERVISOR_WORKER_DEFAULT_VERSION, + VersionInfo.getVersion()); } final SimpleVersion topoVersion = new SimpleVersion(topoVersionString); @@ -627,28 +671,32 @@ private List mkLaunchCommand(final int memOnheap, final int memOffheap, String log4jConfigurationFile = getWorkerLoggingConfigFile(); String workerLog4jConfig = log4jConfigurationFile; if (topoConf.get(Config.TOPOLOGY_LOGGING_CONFIG_FILE) != null) { - workerLog4jConfig = workerLog4jConfig + "," + topoConf.get(Config.TOPOLOGY_LOGGING_CONFIG_FILE); + workerLog4jConfig = workerLog4jConfig + "," + topoConf + .get(Config.TOPOLOGY_LOGGING_CONFIG_FILE); } List commandList = new ArrayList<>(); String logWriter = getWorkerLogWriter(topoVersion); if (logWriter != null) { - //Log Writer Command... + // Log Writer Command... commandList.add(javaCmd); commandList.addAll(classPathParams); - commandList.addAll(substituteChildopts(topoConf.get(Config.TOPOLOGY_WORKER_LOGWRITER_CHILDOPTS))); + commandList.addAll(substituteChildopts(topoConf + .get(Config.TOPOLOGY_WORKER_LOGWRITER_CHILDOPTS))); commandList.addAll(commonParams); commandList.add("-Dlog4j.configurationFile=" + log4jConfigurationFile); - commandList.add(logWriter); //The LogWriter in turn launches the actual worker. + commandList.add(logWriter); // The LogWriter in turn launches the actual worker. } - //Worker Command... + // Worker Command... commandList.add(javaCmd); commandList.add("-server"); commandList.addAll(commonParams); commandList.add("-Dlog4j.configurationFile=" + workerLog4jConfig); - commandList.addAll(substituteChildopts(conf.get(Config.WORKER_CHILDOPTS), memOnheap, memOffheap)); - commandList.addAll(substituteChildopts(topoConf.get(Config.TOPOLOGY_WORKER_CHILDOPTS), memOnheap, memOffheap)); + commandList.addAll(substituteChildopts(conf.get(Config.WORKER_CHILDOPTS), memOnheap, + memOffheap)); + commandList.addAll(substituteChildopts(topoConf.get(Config.TOPOLOGY_WORKER_CHILDOPTS), + memOnheap, memOffheap)); commandList.addAll(substituteChildopts(Utils.OR( topoConf.get(Config.TOPOLOGY_WORKER_GC_CHILDOPTS), conf.get(Config.WORKER_GC_CHILDOPTS)), memOnheap, memOffheap)); @@ -694,8 +742,9 @@ public boolean isMemoryLimitViolated(LocalAssignment withUpdatedLimits) throws I String typeOfCheck; if (withUpdatedLimits.is_set_total_node_shared()) { - //We need to do enforcement on a topology level, not a single worker level... - // Because in for cgroups each page in shared memory goes to the worker that touched it + // We need to do enforcement on a topology level, not a single worker level... + // Because in for cgroups each page in shared memory goes to the worker that touched + // it // first. We may need to make this more plugable in the future and let the resource // isolation manager tell us what to do usageMb = getTotalTopologyMemoryUsed(); @@ -709,24 +758,27 @@ public boolean isMemoryLimitViolated(LocalAssignment withUpdatedLimits) throws I typeOfCheck = "WORKER " + workerId; } LOG.debug( - "Enforcing memory usage for {} with usage of {} out of {} total and a hard limit of {}", + "Enforcing memory usage for {} with usage of {} out of {} total and a hard limit " + + "of {}", typeOfCheck, usageMb, memoryLimitMb, hardMemoryLimitOver); if (usageMb <= 0) { - //Looks like usage might not be supported + // Looks like usage might not be supported return false; } - long hardLimitMb = Math.max((long) (memoryLimitMb * hardMemoryLimitMultiplier), memoryLimitMb + hardMemoryLimitOver); + long hardLimitMb = Math.max((long) (memoryLimitMb * hardMemoryLimitMultiplier), + memoryLimitMb + hardMemoryLimitOver); if (usageMb > hardLimitMb) { LOG.warn( "{} is using {} MB > adjusted hard limit {} MB", typeOfCheck, usageMb, hardLimitMb); return true; } if (usageMb > memoryLimitMb) { - //For others using too much it is really a question of how much memory is free in the system + // For others using too much it is really a question of how much memory is free in + // the system // to be use. If we cannot calculate it assume that it is bad long systemFreeMemoryMb = 0; try { @@ -735,10 +787,11 @@ public boolean isMemoryLimitViolated(LocalAssignment withUpdatedLimits) throws I LOG.warn("Error trying to calculate free memory on the system {}", e); } LOG.debug("SYSTEM MEMORY FREE {} MB", systemFreeMemoryMb); - //If the system is low on memory we cannot be kind and need to shoot something + // If the system is low on memory we cannot be kind and need to shoot something if (systemFreeMemoryMb <= lowMemoryThresholdMb) { LOG.warn( - "{} is using {} MB > memory limit {} MB and system is low on memory {} free", + "{} is using {} MB > memory limit {} MB and system is low on memory {} " + + "free", typeOfCheck, usageMb, memoryLimitMb, @@ -746,7 +799,7 @@ public boolean isMemoryLimitViolated(LocalAssignment withUpdatedLimits) throws I return true; } - //If the system still has some free memory give them a grace period to + // If the system still has some free memory give them a grace period to // drop back down. if (systemFreeMemoryMb < mediumMemoryThresholdMb) { if (memoryLimitExceededStart < 0) { @@ -764,8 +817,9 @@ public boolean isMemoryLimitViolated(LocalAssignment withUpdatedLimits) throws I } } } else { - //Otherwise don't bother them - LOG.debug("{} is using {} MB > memory limit {} MB", typeOfCheck, usageMb, memoryLimitMb); + // Otherwise don't bother them + LOG.debug("{} is using {} MB > memory limit {} MB", typeOfCheck, usageMb, + memoryLimitMb); memoryLimitExceededStart = -1; } } else { @@ -780,7 +834,8 @@ public long getMemoryUsageMb() { try { long ret = 0; if (resourceIsolationManager.isResourceManaged()) { - long usageBytes = resourceIsolationManager.getMemoryUsage(getWorkerUser(), workerId, port); + long usageBytes = resourceIsolationManager.getMemoryUsage(getWorkerUser(), workerId, + port); if (usageBytes >= 0) { ret = usageBytes / 1024 / 1024; } @@ -817,10 +872,12 @@ public void launch() throws IOException { type.assertFull(); String numaId = SupervisorUtils.getNumaIdForPort(port, conf); if (numaId == null) { - LOG.info("Launching worker with assignment {} for this supervisor {} on port {} with id {}", + LOG.info("Launching worker with assignment {} for this supervisor {} on port {} with " + + "id {}", assignment, supervisorId, port, workerId); } else { - LOG.info("Launching worker with assignment {} for this supervisor {} on port {} with id {} bound to numa zone {}", + LOG.info("Launching worker with assignment {} for this supervisor {} on port {} with " + + "id {} bound to numa zone {}", assignment, supervisorId, port, workerId, numaId); } exitedEarly = false; @@ -834,7 +891,8 @@ public void launch() throws IOException { Map topEnvironment = new HashMap(); @SuppressWarnings("unchecked") - Map environment = (Map) topoConf.get(Config.TOPOLOGY_ENVIRONMENT); + Map environment = (Map) topoConf + .get(Config.TOPOLOGY_ENVIRONMENT); if (environment != null) { topEnvironment.putAll(environment); } @@ -848,8 +906,9 @@ public void launch() throws IOException { if (resourceIsolationManager.isResourceManaged()) { final int cpu = (int) Math.ceil(resources.get_cpu()); - //Save the memory limit so we can enforce it less strictly - resourceIsolationManager.reserveResourcesForWorker(workerId, (int) memoryLimitMb, cpu, numaId); + // Save the memory limit so we can enforce it less strictly + resourceIsolationManager.reserveResourcesForWorker(workerId, (int) memoryLimitMb, cpu, + numaId); } List commandList = mkLaunchCommand(memOnHeap, memOffHeap, stormRoot, jlp, numaId); @@ -860,7 +919,8 @@ public void launch() throws IOException { String logPrefix = "Worker Process " + workerId; ProcessExitCallback processExitCallback = new ProcessExitCallback(logPrefix); - resourceIsolationManager.launchWorkerProcess(getWorkerUser(), topologyId, topoConf, port, workerId, + resourceIsolationManager.launchWorkerProcess(getWorkerUser(), topologyId, topoConf, port, + workerId, commandList, topEnvironment, logPrefix, processExitCallback, new File(workerDir)); } @@ -873,7 +933,8 @@ private static class TopologyMetaData { private List depLocs = null; private String stormVersion = null; - TopologyMetaData(final Map conf, final String topologyId, final AdvancedFSOps ops, final String stormRoot) { + TopologyMetaData(final Map conf, final String topologyId, + final AdvancedFSOps ops, final String stormRoot) { this.conf = conf; this.topologyId = topologyId; this.ops = ops; @@ -888,11 +949,13 @@ public String toString() { data = depLocs; stormVersion = this.stormVersion; } - return "META for " + topologyId + " DEP_LOCS => " + data + " STORM_VERSION => " + stormVersion; + return "META for " + topologyId + " DEP_LOCS => " + data + " STORM_VERSION => " + + stormVersion; } private synchronized void readData() throws IOException { - final StormTopology stormTopology = ConfigUtils.readSupervisorTopology(conf, topologyId, ops); + final StormTopology stormTopology = ConfigUtils.readSupervisorTopology(conf, topologyId, + ops); final List dependencyLocations = new ArrayList<>(); if (stormTopology.get_dependency_jars() != null) { for (String dependency : stormTopology.get_dependency_jars()) { @@ -926,22 +989,25 @@ public synchronized String getStormVersion() throws IOException { } static class TopoMetaLruCache { - public final int maxSize = 100; //We could make this configurable in the future... + public final int maxSize = 100; // We could make this configurable in the future... @SuppressWarnings("serial") - private LinkedHashMap cache = new LinkedHashMap() { - @Override + private LinkedHashMap cache = + new LinkedHashMap() { + @Override protected boolean removeEldestEntry(Map.Entry eldest) { - return (size() > maxSize); - } - }; + return (size() > maxSize); + } + }; - public synchronized TopologyMetaData get(final Map conf, final String topologyId, final AdvancedFSOps ops, + public synchronized TopologyMetaData get(final Map conf, + final String topologyId, final AdvancedFSOps ops, String stormRoot) { - //Only go off of the topology id for now. + // Only go off of the topology id for now. TopologyMetaData dl = cache.get(topologyId); if (dl == null) { - cache.putIfAbsent(topologyId, new TopologyMetaData(conf, topologyId, ops, stormRoot)); + cache.putIfAbsent(topologyId, new TopologyMetaData(conf, topologyId, ops, + stormRoot)); dl = cache.get(topologyId); } return dl; diff --git a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/BasicContainerLauncher.java b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/BasicContainerLauncher.java index 51dceb4c384..ffb7200503b 100644 --- a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/BasicContainerLauncher.java +++ b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/BasicContainerLauncher.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -43,8 +49,10 @@ public BasicContainerLauncher(Map conf, String supervisorId, int } @Override - public Container launchContainer(int port, LocalAssignment assignment, LocalState state) throws IOException { - Container container = new BasicContainer(ContainerType.LAUNCH, conf, supervisorId, supervisorPort, port, + public Container launchContainer(int port, LocalAssignment assignment, + LocalState state) throws IOException { + Container container = new BasicContainer(ContainerType.LAUNCH, conf, supervisorId, + supervisorPort, port, assignment, resourceIsolationManager, state, null, metricsRegistry, containerMemoryTracker); container.setup(); @@ -53,14 +61,17 @@ public Container launchContainer(int port, LocalAssignment assignment, LocalStat } @Override - public Container recoverContainer(int port, LocalAssignment assignment, LocalState state) throws IOException { - return new BasicContainer(ContainerType.RECOVER_FULL, conf, supervisorId, supervisorPort, port, assignment, + public Container recoverContainer(int port, LocalAssignment assignment, + LocalState state) throws IOException { + return new BasicContainer(ContainerType.RECOVER_FULL, conf, supervisorId, supervisorPort, + port, assignment, resourceIsolationManager, state, null, metricsRegistry, containerMemoryTracker); } @Override public Killable recoverContainer(String workerId, LocalState localState) throws IOException { - return new BasicContainer(ContainerType.RECOVER_PARTIAL, conf, supervisorId, supervisorPort, -1, null, + return new BasicContainer(ContainerType.RECOVER_PARTIAL, conf, supervisorId, supervisorPort, + -1, null, resourceIsolationManager, localState, workerId, metricsRegistry, containerMemoryTracker); } } diff --git a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/Container.java b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/Container.java index d45f1a639d1..a0f53a0a01a 100644 --- a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/Container.java +++ b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/Container.java @@ -32,7 +32,6 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; - import org.apache.commons.lang3.StringUtils; import org.apache.storm.Config; import org.apache.storm.DaemonConfig; @@ -74,12 +73,12 @@ public abstract class Container implements Killable { private final Timer shutdownDuration; private final Timer cleanupDuration; protected final Map conf; - protected final Map topoConf; //Not set if RECOVER_PARTIAL - protected final String topologyId; //Not set if RECOVER_PARTIAL + protected final Map topoConf; // Not set if RECOVER_PARTIAL + protected final String topologyId; // Not set if RECOVER_PARTIAL protected final String supervisorId; protected final int supervisorPort; - protected final int port; //Not set if RECOVER_PARTIAL - protected final LocalAssignment assignment; //Not set if RECOVER_PARTIAL + protected final int port; // Not set if RECOVER_PARTIAL + protected final LocalAssignment assignment; // Not set if RECOVER_PARTIAL protected final AdvancedFSOps ops; protected final ResourceIsolationInterface resourceIsolationManager; protected final boolean symlinksDisabled; @@ -98,17 +97,21 @@ public abstract class Container implements Killable { * @param conf the supervisor config * @param supervisorId the ID of the supervisor this is a part of. * @param supervisorPort the thrift server port of the supervisor this is a part of. - * @param port the port the container is on. Should be <= 0 if only a partial recovery @param assignment + * @param port the port the container is on. Should be <= 0 if only a partial recovery @param + * assignment * the assignment for this container. Should be null if only a partial recovery. - * @param resourceIsolationManager used to isolate resources for a container can be null if no isolation is used. + * @param resourceIsolationManager used to isolate resources for a container can be null if no + * isolation is used. * @param workerId the id of the worker to use. Must not be null if doing a partial recovery. - * @param topoConf the config of the topology (mostly for testing) if null and not a partial recovery the real conf is read. + * @param topoConf the config of the topology (mostly for testing) if null and not a partial + * recovery the real conf is read. * @param ops file system operations (mostly for testing) if null a new one is made * @param metricsRegistry The metrics registry. * @param containerMemoryTracker The shared memory tracker for the supervisor's containers * @throws IOException on any error. */ - protected Container(ContainerType type, Map conf, String supervisorId, int supervisorPort, + protected Container(ContainerType type, Map conf, String supervisorId, + int supervisorPort, int port, LocalAssignment assignment, ResourceIsolationInterface resourceIsolationManager, String workerId, Map topoConf, AdvancedFSOps ops, StormMetricsRegistry metricsRegistry, ContainerMemoryTracker containerMemoryTracker) throws IOException { @@ -140,32 +143,39 @@ protected Container(ContainerType type, Map conf, String supervi runAsUser = ObjectReader.getBoolean(conf.get(Config.SUPERVISOR_RUN_WORKER_AS_USER), false); if (runAsUser && Utils.isOnWindows()) { - throw new UnsupportedOperationException("ERROR: Windows doesn't support running workers as different users yet"); + throw new UnsupportedOperationException("ERROR: Windows doesn't support running " + + "workers as different users yet"); } if (this.type.isOnlyKillable()) { if (this.assignment != null) { - throw new IOException("With ContainerType==OnlyKillable, expecting LocalAssignment member variable to be null"); + throw new IOException("With ContainerType==OnlyKillable, expecting " + + "LocalAssignment member variable to be null"); } if (this.port > 0) { - throw new IOException("With ContainerType==OnlyKillable, expecting port member variable <=0 but found " + this.port); + throw new IOException("With ContainerType==OnlyKillable, expecting port member " + + "variable <=0 but found " + this.port); } if (this.workerId == null) { - throw new IOException("With ContainerType==OnlyKillable, expecting WorkerId member variable to be assigned"); + throw new IOException("With ContainerType==OnlyKillable, expecting WorkerId " + + "member variable to be assigned"); } topologyId = null; this.topoConf = null; } else { if (this.assignment == null) { - throw new IOException("With ContainerType!=OnlyKillable, expecting LocalAssignment member variable to be assigned"); + throw new IOException("With ContainerType!=OnlyKillable, expecting " + + "LocalAssignment member variable to be assigned"); } if (this.port <= 0) { - throw new IOException("With ContainerType!=OnlyKillable, expecting port member variable >0 but found " + this.port); + throw new IOException("With ContainerType!=OnlyKillable, expecting port member " + + "variable >0 but found " + this.port); } topologyId = assignment.get_topology_id(); if (!this.ops.doRequiredTopoFilesExist(this.conf, topologyId)) { LOG.info( - "Missing topology storm code, so can't launch worker with assignment {} for this supervisor {} on port {} with id {}", + "Missing topology storm code, so can't launch worker with assignment {} for " + + "this supervisor {} on port {} with id {}", this.assignment, this.supervisorId, this.port, this.workerId); throw new ContainerRecoveryException("Missing required topology files..."); @@ -173,16 +183,20 @@ protected Container(ContainerType type, Map conf, String supervi if (topoConf == null) { this.topoConf = readTopoConf(); } else { - //For testing... + // For testing... this.topoConf = topoConf; } } - this.numCleanupExceptions = metricsRegistry.registerMeter("supervisor:num-cleanup-exceptions"); + this.numCleanupExceptions = metricsRegistry + .registerMeter("supervisor:num-cleanup-exceptions"); this.numKillExceptions = metricsRegistry.registerMeter("supervisor:num-kill-exceptions"); - this.numForceKillExceptions = metricsRegistry.registerMeter("supervisor:num-force-kill-exceptions"); + this.numForceKillExceptions = metricsRegistry + .registerMeter("supervisor:num-force-kill-exceptions"); this.numForceKill = metricsRegistry.registerMeter("supervisor:num-workers-force-kill"); - this.shutdownDuration = metricsRegistry.registerTimer("supervisor:worker-shutdown-duration-ns"); - this.cleanupDuration = metricsRegistry.registerTimer("supervisor:worker-per-call-clean-up-duration-ns"); + this.shutdownDuration = metricsRegistry + .registerTimer("supervisor:worker-shutdown-duration-ns"); + this.cleanupDuration = metricsRegistry + .registerTimer("supervisor:worker-per-call-clean-up-duration-ns"); this.containerMemoryTracker = containerMemoryTracker; } @@ -263,14 +277,15 @@ public void cleanUp() throws IOException { containerMemoryTracker.remove(port); cleanUpForRestart(); } catch (IOException e) { - //This may or may not be reported depending on when process exits + // This may or may not be reported depending on when process exits numCleanupExceptions.mark(); throw e; } } /** - * Setup the container to run. By default this creates the needed directories/links in the local file system PREREQUISITE: All needed + * Setup the container to run. By default this creates the needed directories/links in the local + * file system PREREQUISITE: All needed * blobs and topology, jars/configs have been downloaded and placed in the appropriate locations * * @throws IOException on any error @@ -278,7 +293,8 @@ public void cleanUp() throws IOException { protected void setup() throws IOException { type.assertFull(); if (!ops.doRequiredTopoFilesExist(conf, topologyId)) { - LOG.info("Missing topology storm code, so can't launch worker with assignment {} for this supervisor {} on port {} with id {}", + LOG.info("Missing topology storm code, so can't launch worker with assignment {} for " + + "this supervisor {} on port {} with id {}", assignment, supervisorId, port, workerId); throw new IllegalStateException("Not all needed files are here!!!!"); @@ -337,8 +353,10 @@ protected void writeLogMetadata(String user) throws IOException { data.put(DaemonConfig.LOGS_USERS, logsUsers.toArray()); if (topoConf.get(Config.TOPOLOGY_WORKER_TIMEOUT_SECS) != null) { - int topoTimeout = ObjectReader.getInt(topoConf.get(Config.TOPOLOGY_WORKER_TIMEOUT_SECS)); - int defaultWorkerTimeout = ObjectReader.getInt(conf.get(Config.SUPERVISOR_WORKER_TIMEOUT_SECS)); + int topoTimeout = ObjectReader.getInt(topoConf + .get(Config.TOPOLOGY_WORKER_TIMEOUT_SECS)); + int defaultWorkerTimeout = ObjectReader.getInt(conf + .get(Config.SUPERVISOR_WORKER_TIMEOUT_SECS)); topoTimeout = Math.max(topoTimeout, defaultWorkerTimeout); data.put(Config.TOPOLOGY_WORKER_TIMEOUT_SECS, topoTimeout); } @@ -362,14 +380,17 @@ protected void createArtifactsLink() throws IOException { File workerDir = new File(ConfigUtils.workerRoot(conf, workerId)); File topoDir = new File(ConfigUtils.workerArtifactsRoot(conf, topologyId, port)); if (ops.fileExists(workerDir)) { - LOG.debug("Creating symlinks for worker-id: {} topology-id: {} to its port artifacts directory", workerId, topologyId); - ops.createSymlink(new File(ConfigUtils.workerArtifactsSymlink(conf, workerId)), topoDir); + LOG.debug("Creating symlinks for worker-id: {} topology-id: {} to its port " + + "artifacts directory", workerId, topologyId); + ops.createSymlink(new File(ConfigUtils.workerArtifactsSymlink(conf, workerId)), + topoDir); } } } /** - * Create symlinks for each of the blobs from the container's directory to corresponding links in the storm dist directory. + * Create symlinks for each of the blobs from the container's directory to corresponding links + * in the storm dist directory. * * @throws IOException on any error. */ @@ -379,7 +400,8 @@ protected void createBlobstoreLinks() throws IOException { String workerRoot = ConfigUtils.workerRoot(conf, workerId); @SuppressWarnings("unchecked") - Map> blobstoreMap = (Map>) topoConf.get(Config.TOPOLOGY_BLOBSTORE_MAP); + Map> blobstoreMap = (Map>) topoConf + .get(Config.TOPOLOGY_BLOBSTORE_MAP); List blobFileNames = new ArrayList<>(); if (blobstoreMap != null) { for (Map.Entry> entry : blobstoreMap.entrySet()) { @@ -402,17 +424,22 @@ protected void createBlobstoreLinks() throws IOException { resourceFileNames.addAll(blobFileNames); if (!symlinksDisabled) { - LOG.info("Creating symlinks for worker-id: {} storm-id: {} for files({}): {}", workerId, topologyId, resourceFileNames.size(), + LOG.info("Creating symlinks for worker-id: {} storm-id: {} for files({}): {}", workerId, + topologyId, resourceFileNames.size(), resourceFileNames); if (targetResourcesDir.exists()) { - ops.createSymlink(new File(workerRoot, ServerConfigUtils.RESOURCES_SUBDIR), targetResourcesDir); + ops.createSymlink(new File(workerRoot, ServerConfigUtils.RESOURCES_SUBDIR), + targetResourcesDir); } else { - LOG.info("Topology jar for worker-id: {} storm-id: {} does not contain re sources directory {}.", workerId, topologyId, + LOG.info("Topology jar for worker-id: {} storm-id: {} does not contain re sources " + + "directory {}.", workerId, topologyId, targetResourcesDir.toString()); } for (String fileName : blobFileNames) { - // the localname may come from the topology conf, it must not point outside of the worker/dist dirs - ops.createSymlink(ServerUtils.resolveTopologyConfSuppliedName(new File(workerRoot), fileName), + // the localname may come from the topology conf, it must not point outside of the + // worker/dist dirs + ops.createSymlink(ServerUtils.resolveTopologyConfSuppliedName(new File(workerRoot), + fileName), ServerUtils.resolveTopologyConfSuppliedName(new File(stormRoot), fileName)); } } else if (blobFileNames.size() > 0) { @@ -422,6 +449,7 @@ protected void createBlobstoreLinks() throws IOException { /** * Get the user of the worker. + * * @return the user that some operations should be done as. * @throws IOException on any error */ @@ -473,7 +501,8 @@ protected void deleteSavedWorkerUser() throws IOException { } /** - * Clean up the container partly preparing for restart. By default delete all of the temp directories we are going to get a new + * Clean up the container partly preparing for restart. By default delete all of the temp + * directories we are going to get a new * worker_id anyways. POST CONDITION: the workerId will be set to null * * @throws IOException on any error @@ -482,14 +511,15 @@ public void cleanUpForRestart() throws IOException { LOG.info("Cleaning up {}:{}", supervisorId, workerId); String user = getWorkerUser(); - //clean up for resource isolation if enabled + // clean up for resource isolation if enabled if (resourceIsolationManager != null) { resourceIsolationManager.cleanup(user, workerId, port); } - //Always make sure to clean up everything else before worker directory - //is removed since that is what is going to trigger the retry for cleanup - ops.deleteIfExists(new File(ConfigUtils.workerHeartbeatsRoot(conf, workerId)), user, workerId); + // Always make sure to clean up everything else before worker directory + // is removed since that is what is going to trigger the retry for cleanup + ops.deleteIfExists(new File(ConfigUtils.workerHeartbeatsRoot(conf, workerId)), user, + workerId); ops.deleteIfExists(new File(ConfigUtils.workerPidsRoot(conf, workerId)), user, workerId); ops.deleteIfExists(new File(ConfigUtils.workerTmpRoot(conf, workerId)), user, workerId); ops.deleteIfExists(new File(ConfigUtils.workerRoot(conf, workerId)), user, workerId); @@ -498,7 +528,8 @@ public void cleanUpForRestart() throws IOException { } /** - * Check if the container is over its memory limit AND needs to be killed. This does not necessarily mean that it just went over the + * Check if the container is over its memory limit AND needs to be killed. This does not + * necessarily mean that it just went over the * limit. * * @throws IOException on any error @@ -576,7 +607,8 @@ public long getMemoryReservationMb() { public abstract void relaunch() throws IOException; /** - * Return true if the main process exited, else false. This is just best effort return false if unknown. + * Return true if the main process exited, else false. This is just best effort return false if + * unknown. */ public abstract boolean didMainProcessExit(); @@ -590,7 +622,8 @@ public long getMemoryReservationMb() { * @throws IOException on any error * @throws InterruptedException if running the command is interrupted. */ - public abstract boolean runProfiling(ProfileRequest request, boolean stop) throws IOException, InterruptedException; + public abstract boolean runProfiling(ProfileRequest request, + boolean stop) throws IOException, InterruptedException; /** * Get the id of the container or null if there is no worker id right now. diff --git a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/ContainerLauncher.java b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/ContainerLauncher.java index 11600286302..6ab1e96bc40 100644 --- a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/ContainerLauncher.java +++ b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/ContainerLauncher.java @@ -1,12 +1,17 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -34,60 +39,72 @@ public abstract class ContainerLauncher { private static final Logger LOG = LoggerFactory.getLogger(ContainerLauncher.class); protected ContainerLauncher() { - //Empty + // Empty } /** * Factory to create the right container launcher * for the config and the environment. + * * @param conf the config * @param supervisorId the ID of the supervisor * @param supervisorPort the parent supervisor thrift server port * @param sharedContext Used in local mode to let workers talk together without netty * @param metricsRegistry The metrics registry. * @param containerMemoryTracker The shared memory tracker for the supervisor's containers - * @param localSupervisor The local supervisor Thrift interface. Only used for local clusters, distributed clusters use Thrift directly. + * @param localSupervisor The local supervisor Thrift interface. Only used for local clusters, + * distributed clusters use Thrift directly. * @return the proper container launcher * @throws IOException on any error */ - public static ContainerLauncher make(Map conf, String supervisorId, int supervisorPort, + public static ContainerLauncher make(Map conf, String supervisorId, + int supervisorPort, IContext sharedContext, StormMetricsRegistry metricsRegistry, ContainerMemoryTracker containerMemoryTracker, org.apache.storm.generated.Supervisor.Iface localSupervisor) throws IOException { if (ConfigUtils.isLocalMode(conf)) { - return new LocalContainerLauncher(conf, supervisorId, supervisorPort, sharedContext, metricsRegistry, containerMemoryTracker, + return new LocalContainerLauncher(conf, supervisorId, supervisorPort, sharedContext, + metricsRegistry, containerMemoryTracker, localSupervisor); } ResourceIsolationInterface resourceIsolationManager; - if (ObjectReader.getBoolean(conf.get(DaemonConfig.STORM_RESOURCE_ISOLATION_PLUGIN_ENABLE), false)) { - resourceIsolationManager = ReflectionUtils.newInstance((String) conf.get(DaemonConfig.STORM_RESOURCE_ISOLATION_PLUGIN)); - LOG.info("Using resource isolation plugin {}: {}", conf.get(DaemonConfig.STORM_RESOURCE_ISOLATION_PLUGIN), + if (ObjectReader.getBoolean(conf.get(DaemonConfig.STORM_RESOURCE_ISOLATION_PLUGIN_ENABLE), + false)) { + resourceIsolationManager = ReflectionUtils.newInstance((String) conf + .get(DaemonConfig.STORM_RESOURCE_ISOLATION_PLUGIN)); + LOG.info("Using resource isolation plugin {}: {}", conf + .get(DaemonConfig.STORM_RESOURCE_ISOLATION_PLUGIN), resourceIsolationManager); } else { resourceIsolationManager = new DefaultResourceIsolationManager(); - LOG.info("{} is false. Using default resource isolation plugin: {}", DaemonConfig.STORM_RESOURCE_ISOLATION_PLUGIN_ENABLE, + LOG.info("{} is false. Using default resource isolation plugin: {}", + DaemonConfig.STORM_RESOURCE_ISOLATION_PLUGIN_ENABLE, resourceIsolationManager); } resourceIsolationManager.prepare(conf); - return new BasicContainerLauncher(conf, supervisorId, supervisorPort, resourceIsolationManager, metricsRegistry, + return new BasicContainerLauncher(conf, supervisorId, supervisorPort, + resourceIsolationManager, metricsRegistry, containerMemoryTracker); } /** * Launch a container in a given slot. + * * @param port the port to run this on * @param assignment what to launch * @param state the current state of the supervisor * @return The container that can be used to manager the processes. * @throws IOException on any error */ - public abstract Container launchContainer(int port, LocalAssignment assignment, LocalState state) throws IOException; + public abstract Container launchContainer(int port, LocalAssignment assignment, + LocalState state) throws IOException; /** * Recover a container for a running process. + * * @param port the port the assignment is running on * @param assignment the assignment that was launched * @param state the current state of the supervisor @@ -95,7 +112,8 @@ public static ContainerLauncher make(Map conf, String supervisor * @throws IOException on any error * @throws ContainerRecoveryException if the Container could not be recovered */ - public abstract Container recoverContainer(int port, LocalAssignment assignment, LocalState state) throws IOException, + public abstract Container recoverContainer(int port, LocalAssignment assignment, + LocalState state) throws IOException, ContainerRecoveryException; /** @@ -104,11 +122,13 @@ public abstract Container recoverContainer(int port, LocalAssignment assignment, * and so is returning a Killable. Even if a Container is returned * do not case the result to Container because only the Killable APIs * are guaranteed to work. + * * @param workerId the id of the worker to use * @param localState the state of the running supervisor * @return a Killable that can be used to kill the underlying container. * @throws IOException on any error * @throws ContainerRecoveryException if the Container could not be recovered */ - public abstract Killable recoverContainer(String workerId, LocalState localState) throws IOException, ContainerRecoveryException; + public abstract Killable recoverContainer(String workerId, + LocalState localState) throws IOException, ContainerRecoveryException; } diff --git a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/ContainerMemoryTracker.java b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/ContainerMemoryTracker.java index e79794e3754..2fd75b8ef83 100644 --- a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/ContainerMemoryTracker.java +++ b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/ContainerMemoryTracker.java @@ -23,13 +23,15 @@ public class ContainerMemoryTracker { private final ConcurrentHashMap usedMemory = new ConcurrentHashMap<>(); - private final ConcurrentHashMap reservedMemory = new ConcurrentHashMap<>(); + private final ConcurrentHashMap reservedMemory = + new ConcurrentHashMap<>(); public ContainerMemoryTracker(StormMetricsRegistry metricsRegistry) { metricsRegistry.registerGauge( "supervisor:current-used-memory-mb", () -> { - Long val = usedMemory.values().stream().mapToLong((topoAndMem) -> topoAndMem.memory).sum(); + Long val = usedMemory.values().stream().mapToLong((topoAndMem) -> topoAndMem.memory) + .sum(); int ret = val.intValue(); if (val > Integer.MAX_VALUE) { // Would only happen at 2 PB so we are OK for now ret = Integer.MAX_VALUE; @@ -39,7 +41,8 @@ public ContainerMemoryTracker(StormMetricsRegistry metricsRegistry) { metricsRegistry.registerGauge( "supervisor:current-reserved-memory-mb", () -> { - Long val = reservedMemory.values().stream().mapToLong((topoAndMem) -> topoAndMem.memory).sum(); + Long val = reservedMemory.values().stream() + .mapToLong((topoAndMem) -> topoAndMem.memory).sum(); int ret = val.intValue(); if (val > Integer.MAX_VALUE) { // Would only happen at 2 PB so we are OK for now ret = Integer.MAX_VALUE; @@ -117,7 +120,8 @@ public void remove(int port) { } /** - * Assigns the given topology id to the given port, and sets the used memory for that port and topology id. + * Assigns the given topology id to the given port, and sets the used memory for that port and + * topology id. * * @param port The worker port * @param topologyId The topology id diff --git a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/ContainerRecoveryException.java b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/ContainerRecoveryException.java index 491ffdcce41..74be5a6c468 100644 --- a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/ContainerRecoveryException.java +++ b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/ContainerRecoveryException.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/DefaultUncaughtExceptionHandler.java b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/DefaultUncaughtExceptionHandler.java index 7b7ff1bce32..ba7ab05e354 100644 --- a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/DefaultUncaughtExceptionHandler.java +++ b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/DefaultUncaughtExceptionHandler.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -17,7 +23,8 @@ import org.slf4j.LoggerFactory; public class DefaultUncaughtExceptionHandler implements Thread.UncaughtExceptionHandler { - private static final Logger LOG = LoggerFactory.getLogger(DefaultUncaughtExceptionHandler.class); + private static final Logger LOG = LoggerFactory + .getLogger(DefaultUncaughtExceptionHandler.class); @Override public void uncaughtException(Thread t, Throwable e) { diff --git a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/EventManagerPushCallback.java b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/EventManagerPushCallback.java index 09a7bfbfe3e..457a4a65a01 100644 --- a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/EventManagerPushCallback.java +++ b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/EventManagerPushCallback.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/Killable.java b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/Killable.java index 78fc2debf45..a65395fb10a 100644 --- a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/Killable.java +++ b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/Killable.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -19,6 +25,7 @@ public interface Killable { /** * Kill the processes in this container nicely. * kill -15 equivalent + * * @throws IOException on any error */ void kill() throws IOException; @@ -26,12 +33,14 @@ public interface Killable { /** * Kill the processes in this container violently. * kill -9 equivalent + * * @throws IOException on any error */ void forceKill() throws IOException; /** * Check whether all processes are dead. + * * @return true if all of the processes are dead, else false * @throws IOException on any error */ @@ -40,6 +49,7 @@ public interface Killable { /** * Clean up the container. It is not coming back. * by default do the same thing as when restarting. + * * @throws IOException on any error */ void cleanUp() throws IOException; diff --git a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/LocalContainer.java b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/LocalContainer.java index b74bbf95889..cc918e0c82d 100644 --- a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/LocalContainer.java +++ b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/LocalContainer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -30,11 +36,13 @@ public class LocalContainer extends Container { private final org.apache.storm.generated.Supervisor.Iface localSupervisor; private volatile boolean isAlive = false; - public LocalContainer(Map conf, String supervisorId, int supervisorPort, int port, + public LocalContainer(Map conf, String supervisorId, int supervisorPort, + int port, LocalAssignment assignment, IContext sharedContext, StormMetricsRegistry metricsRegistry, ContainerMemoryTracker containerMemoryTracker, org.apache.storm.generated.Supervisor.Iface localSupervisor) throws IOException { - super(ContainerType.LAUNCH, conf, supervisorId, supervisorPort, port, assignment, null, null, null, null, metricsRegistry, + super(ContainerType.LAUNCH, conf, supervisorId, supervisorPort, port, assignment, null, + null, null, null, metricsRegistry, containerMemoryTracker); this.sharedContext = sharedContext; workerId = Utils.uuid(); @@ -43,7 +51,7 @@ public LocalContainer(Map conf, String supervisorId, int supervi @Override protected void createArtifactsLink() { - //NOOP no need to create links in local mode + // NOOP no need to create links in local mode } @Override @@ -53,7 +61,8 @@ protected void createBlobstoreLinks() { @Override public void launch() throws IOException { - Worker worker = new Worker(conf, sharedContext, topologyId, supervisorId, supervisorPort, port, workerId, + Worker worker = new Worker(conf, sharedContext, topologyId, supervisorId, supervisorPort, + port, workerId, () -> { return () -> localSupervisor; }); @@ -71,7 +80,7 @@ public void launch() throws IOException { public void kill() throws IOException { ProcessSimulator.killProcess(workerId); isAlive = false; - //Make sure the worker is down before we try to shoot any child processes + // Make sure the worker is down before we try to shoot any child processes super.kill(); } @@ -87,12 +96,13 @@ public void relaunch() throws IOException { @Override public boolean didMainProcessExit() { - //In local mode the main process should never exit on it's own + // In local mode the main process should never exit on it's own return false; } @Override - public boolean runProfiling(ProfileRequest request, boolean stop) throws IOException, InterruptedException { + public boolean runProfiling(ProfileRequest request, + boolean stop) throws IOException, InterruptedException { throw new RuntimeException("Profiling requests are not supported in local mode"); } } diff --git a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/LocalContainerLauncher.java b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/LocalContainerLauncher.java index 5434c7772be..1c5e972c5c4 100644 --- a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/LocalContainerLauncher.java +++ b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/LocalContainerLauncher.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -45,7 +51,8 @@ public LocalContainerLauncher(Map conf, String supervisorId, int } @Override - public Container launchContainer(int port, LocalAssignment assignment, LocalState state) throws IOException { + public Container launchContainer(int port, LocalAssignment assignment, + LocalState state) throws IOException { LocalContainer ret = new LocalContainer(conf, supervisorId, supervisorPort, port, assignment, sharedContext, metricsRegistry, containerMemoryTracker, localSupervisor); ret.setup(); @@ -54,14 +61,15 @@ public Container launchContainer(int port, LocalAssignment assignment, LocalStat } @Override - public Container recoverContainer(int port, LocalAssignment assignment, LocalState state) throws IOException { - //We are in the same process we cannot recover anything + public Container recoverContainer(int port, LocalAssignment assignment, + LocalState state) throws IOException { + // We are in the same process we cannot recover anything throw new ContainerRecoveryException("Local Mode Recovery is not supported"); } @Override public Killable recoverContainer(String workerId, LocalState localState) throws IOException { - //We are in the same process we cannot recover anything + // We are in the same process we cannot recover anything throw new ContainerRecoveryException("Local Mode Recovery is not supported"); } } diff --git a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/OnlyLatestExecutor.java b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/OnlyLatestExecutor.java index 81f38185ec0..1a253b83bde 100644 --- a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/OnlyLatestExecutor.java +++ b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/OnlyLatestExecutor.java @@ -25,7 +25,8 @@ import org.slf4j.LoggerFactory; /** - * This allows you to submit a Runnable with a key. If the previous submission for that key has not yet run, it will be replaced with the + * This allows you to submit a Runnable with a key. If the previous submission for that key has not + * yet run, it will be replaced with the * latest one. */ public class OnlyLatestExecutor { @@ -47,7 +48,7 @@ public OnlyLatestExecutor(Executor exec) { public void execute(final K key, Runnable r) { Runnable old = latest.put(key, r); if (old == null) { - //It was not there before so we need to run it. + // It was not there before so we need to run it. exec.execute(() -> { Runnable run = latest.remove(key); if (run != null) { diff --git a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/ReadClusterState.java b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/ReadClusterState.java index 0186230d075..f7d989be82d 100644 --- a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/ReadClusterState.java +++ b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/ReadClusterState.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -17,8 +23,8 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Map.Entry; +import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; @@ -44,17 +50,21 @@ public class ReadClusterState implements Runnable, AutoCloseable { private static final Logger LOG = LoggerFactory.getLogger(ReadClusterState.class); - private static final long ERROR_MILLIS = 60_000; //1 min. This really means something is wrong. Even on a very slow node + private static final long ERROR_MILLIS = + 60_000; // 1 min. This really means something is wrong. Even on a very slow node public static final UniFunc DEFAULT_ON_ERROR_TIMEOUT = (slot) -> { - throw new IllegalStateException("It took over " + ERROR_MILLIS + "ms to shut down slot " + slot); + throw new IllegalStateException("It took over " + ERROR_MILLIS + "ms to shut down slot " + + slot); }; public static final UniFunc THREAD_DUMP_ON_ERROR = (slot) -> { LOG.warn("Shutdown of slot {} appears to be stuck\n{}", slot, Utils.threadDump()); DEFAULT_ON_ERROR_TIMEOUT.call(slot); }; - private static final long WARN_MILLIS = 1_000; //Initial timeout 1 second. Workers commit suicide after this + private static final long WARN_MILLIS = + 1_000; // Initial timeout 1 second. Workers commit suicide after this public static final BiConsumer DEFAULT_ON_WARN_TIMEOUT = - (slot, elapsedTimeMs) -> LOG.warn("It has taken {}ms so far and {} is still not shut down.", elapsedTimeMs, slot); + (slot, elapsedTimeMs) -> LOG.warn("It has taken {}ms so far and {} is still not shut down.", + elapsedTimeMs, slot); private final Map superConf; private final IStormClusterState stormClusterState; private final Map slots = new HashMap<>(); @@ -85,14 +95,16 @@ public ReadClusterState(Supervisor supervisor) throws Exception { this.slotMetrics = supervisor.getSlotMetrics(); this.launcher = ContainerLauncher.make(superConf, assignmentId, supervisorPort, - supervisor.getSharedContext(), supervisor.getMetricsRegistry(), supervisor.getContainerMemoryTracker(), + supervisor.getSharedContext(), supervisor.getMetricsRegistry(), supervisor + .getContainerMemoryTracker(), supervisor.getSupervisorThriftInterface()); this.metricsProcessor = null; try { this.metricsProcessor = MetricStoreConfig.configureMetricProcessor(superConf); } catch (Exception e) { - // the metrics processor is not critical to the operation of the cluster, allow Supervisor to come up + // the metrics processor is not critical to the operation of the cluster, allow + // Supervisor to come up LOG.error("Failed to initialize metric processor", e); } @@ -103,10 +115,12 @@ public ReadClusterState(Supervisor supervisor) throws Exception { } try { - Collection detachedRunningWorkers = SupervisorUtils.supervisorWorkerIds(superConf); + Collection detachedRunningWorkers = SupervisorUtils + .supervisorWorkerIds(superConf); for (Slot slot : slots.values()) { String workerId = slot.getWorkerId(); - // We ignore workers that are still bound to a slot, which is monitored by a supervisor + // We ignore workers that are still bound to a slot, which is monitored by a + // supervisor if (workerId != null) { detachedRunningWorkers.remove(workerId); } @@ -137,10 +151,11 @@ public synchronized void run() { Map allAssignments = readAssignments(assignmentsSnapshot); if (allAssignments == null) { - //Something odd happened try again later + // Something odd happened try again later return; } - Map> topoIdToProfilerActions = getProfileActions(stormClusterState, stormIds); + Map> topoIdToProfilerActions = + getProfileActions(stormClusterState, stormIds); HashSet assignedPorts = new HashSet<>(); LOG.debug("Synchronizing supervisor"); @@ -199,20 +214,23 @@ protected Map> getProfileActions(IStormClusterState Exception { Map> ret = new HashMap>(); for (String stormId : stormIds) { - List profileRequests = stormClusterState.getTopologyProfileRequests(stormId); + List profileRequests = stormClusterState + .getTopologyProfileRequests(stormId); ret.put(stormId, profileRequests); } return ret; } - protected Map readAssignments(Map assignmentsSnapshot) { + protected Map readAssignments(Map assignmentsSnapshot) { try { Map portLocalAssignment = new HashMap<>(); for (Map.Entry assignEntry : assignmentsSnapshot.entrySet()) { String topoId = assignEntry.getKey(); Assignment assignment = assignEntry.getValue(); - Map portTasks = readMyExecutors(topoId, assignmentId, assignment); + Map portTasks = readMyExecutors(topoId, assignmentId, + assignment); for (Map.Entry entry : portTasks.entrySet()) { @@ -223,7 +241,8 @@ protected Map readAssignments(Map if (!portLocalAssignment.containsKey(port)) { portLocalAssignment.put(port, la); } else { - throw new RuntimeException("Should not have multiple topologies assigned to one port " + throw new RuntimeException("Should not have multiple topologies assigned " + + "to one port " + port + " " + la + " " + portLocalAssignment); } } @@ -241,12 +260,15 @@ protected Map readAssignments(Map } } - protected Map readMyExecutors(String topoId, String assignmentId, Assignment assignment) { + protected Map readMyExecutors(String topoId, String assignmentId, + Assignment assignment) { Map portTasks = new HashMap<>(); Map slotsResources = new HashMap<>(); - Map nodeInfoWorkerResourcesMap = assignment.get_worker_resources(); + Map nodeInfoWorkerResourcesMap = assignment + .get_worker_resources(); if (nodeInfoWorkerResourcesMap != null) { - for (Map.Entry entry : nodeInfoWorkerResourcesMap.entrySet()) { + for (Map.Entry entry : nodeInfoWorkerResourcesMap + .entrySet()) { if (entry.getKey().get_node().startsWith(assignmentId)) { Set ports = entry.getKey().get_port(); for (Long port : ports) { @@ -286,7 +308,8 @@ protected Map readMyExecutors(String topoId, String as } List executorInfoList = localAssignment.get_executors(); executorInfoList.add(new ExecutorInfo(entry.getKey().get(0).intValue(), - entry.getKey().get(entry.getKey().size() - 1).intValue())); + entry.getKey().get(entry.getKey() + .size() - 1).intValue())); } } } @@ -294,7 +317,8 @@ protected Map readMyExecutors(String topoId, String as return portTasks; } - public synchronized void shutdownAllWorkers(BiConsumer onWarnTimeout, UniFunc onErrorTimeout) { + public synchronized void shutdownAllWorkers(BiConsumer onWarnTimeout, + UniFunc onErrorTimeout) { for (Slot slot : slots.values()) { LOG.info("Setting {} assignment to null", slot); slot.setNewAssignment(null); diff --git a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/Slot.java b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/Slot.java index 27b8d7cc612..fef4cb0d961 100644 --- a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/Slot.java +++ b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/Slot.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -66,7 +72,8 @@ public String toString() { private static final long ONE_SEC_IN_NANO = TimeUnit.NANOSECONDS.convert(1, TimeUnit.SECONDS); private final AtomicReference newAssignment = new AtomicReference<>(); - private final AtomicReference> profiling = new AtomicReference<>(new HashSet<>()); + private final AtomicReference> profiling = + new AtomicReference<>(new HashSet<>()); private final BlockingQueue changingBlobs = new LinkedBlockingQueue<>(); private final StaticState staticState; @@ -91,9 +98,11 @@ public Slot(AsyncLocalizer localizer, Map conf, this.clusterState = clusterState; this.staticState = new StaticState(localizer, ObjectReader.getInt(conf.get(Config.SUPERVISOR_WORKER_TIMEOUT_SECS)) * 1000, - ObjectReader.getInt(conf.get(DaemonConfig.SUPERVISOR_WORKER_START_TIMEOUT_SECS)) * 1000, + ObjectReader.getInt(conf + .get(DaemonConfig.SUPERVISOR_WORKER_START_TIMEOUT_SECS)) * 1000, ObjectReader.getInt(conf.get(Config.SUPERVISOR_WORKER_SHUTDOWN_SLEEP_SECS)) * 1000, - ObjectReader.getInt(conf.get(DaemonConfig.SUPERVISOR_MONITOR_FREQUENCY_SECS)) * 1000, + ObjectReader.getInt(conf + .get(DaemonConfig.SUPERVISOR_MONITOR_FREQUENCY_SECS)) * 1000, containerLauncher, host, port, @@ -111,20 +120,25 @@ public Slot(AsyncLocalizer localizer, Map conf, currentAssignment = assignments.get(port); if (currentAssignment != null) { try { - // For now we do not make a transaction when removing a topology assignment from local, an overdue + // For now we do not make a transaction when removing a topology assignment from + // local, an overdue // assignment may be left on local disk. // So we should check if the local disk assignment is valid when initializing: - // if topology files does not exist, the worker[possibly alive] will be reassigned if it is timed-out; - // if topology files exist but the topology id is invalid, just let Supervisor make a sync; + // if topology files does not exist, the worker[possibly alive] will be + // reassigned if it is timed-out; + // if topology files exist but the topology id is invalid, just let Supervisor + // make a sync; // if topology files exist and topology files is valid, recover the container. - if (ClientSupervisorUtils.doRequiredTopoFilesExist(conf, currentAssignment.get_topology_id())) { - container = containerLauncher.recoverContainer(port, currentAssignment, localState); + if (ClientSupervisorUtils.doRequiredTopoFilesExist(conf, currentAssignment + .get_topology_id())) { + container = containerLauncher.recoverContainer(port, currentAssignment, + localState); } else { // Make the assignment null to let slot clean up the disk assignment. currentAssignment = null; } } catch (ContainerRecoveryException e) { - //We could not recover container will be null. + // We could not recover container will be null. currentAssignment = null; } @@ -133,31 +147,37 @@ public Slot(AsyncLocalizer localizer, Map conf, } setNewAssignment(newAssignment); - //if the current assignment is already running, new assignment will never be promoted to currAssignment, - // because Timer is not being compared in #equals or #equivalent, meaning newAssignment always equals to currAssignment. + // if the current assignment is already running, new assignment will never be promoted to + // currAssignment, + // because Timer is not being compared in #equals or #equivalent, meaning newAssignment + // always equals to currAssignment. // Therefore the timer in newAssignment won't be invoked - this.dynamicState = new DynamicState(currentAssignment, container, this.newAssignment.get(), slotMetrics); + this.dynamicState = new DynamicState(currentAssignment, container, this.newAssignment.get(), + slotMetrics); if (MachineState.RUNNING == dynamicState.state) { - //We are running so we should recover the blobs. + // We are running so we should recover the blobs. staticState.localizer.recoverRunningTopology(currentAssignment, port, this); saveNewAssignment(currentAssignment); } - LOG.info("SLOT {}:{} Starting in state {} - assignment {}", staticState.host, staticState.port, dynamicState.state, + LOG.info("SLOT {}:{} Starting in state {} - assignment {}", staticState.host, + staticState.port, dynamicState.state, dynamicState.currentAssignment); } - //In some cases the new LocalAssignment may be equivalent to the old, but + // In some cases the new LocalAssignment may be equivalent to the old, but // It is not equal. In those cases we want to update the current assignment to // be the same as the new assignment - //PRECONDITION: The new and current assignments must be equivalent + // PRECONDITION: The new and current assignments must be equivalent private static DynamicState updateAssignmentIfNeeded(DynamicState dynamicState) { - if (!EquivalenceUtils.areLocalAssignmentsEquivalent(dynamicState.newAssignment, dynamicState.currentAssignment)) { + if (!EquivalenceUtils.areLocalAssignmentsEquivalent(dynamicState.newAssignment, + dynamicState.currentAssignment)) { throw new IllegalArgumentException("new and current assignments must be equivalent"); } if (dynamicState.newAssignment != null && !dynamicState.newAssignment.equals(dynamicState.currentAssignment)) { dynamicState = - dynamicState.withCurrentAssignment(dynamicState.container, dynamicState.newAssignment); + dynamicState.withCurrentAssignment(dynamicState.container, + dynamicState.newAssignment); } return dynamicState; } @@ -175,7 +195,8 @@ static boolean forSameTopology(LocalAssignment a, LocalAssignment b) { return false; } - static DynamicState stateMachineStep(DynamicState dynamicState, StaticState staticState) throws Exception { + static DynamicState stateMachineStep(DynamicState dynamicState, + StaticState staticState) throws Exception { LOG.debug("STATE {}", dynamicState.state); switch (dynamicState.state) { case EMPTY: @@ -195,13 +216,16 @@ static DynamicState stateMachineStep(DynamicState dynamicState, StaticState stat case WAITING_FOR_BLOB_UPDATE: return handleWaitingForBlobUpdate(dynamicState, staticState); default: - throw new IllegalStateException("Code not ready to handle a state of " + dynamicState.state); + throw new IllegalStateException("Code not ready to handle a state of " + + dynamicState.state); } } /** - * Prepare for a new assignment by downloading new required blobs, or going to empty if there is nothing to download. + * Prepare for a new assignment by downloading new required blobs, or going to empty if there is + * nothing to download. * PRECONDITION: The slot should be empty + * * @param dynamicState current state * @param staticState static data * @return the next state @@ -216,24 +240,27 @@ private static DynamicState prepareForNewAssignmentNoWorkersRunning(DynamicState throw new IOException("dynamicState.currentAssignment expected to be null"); } - //We're either going to empty, or starting fresh blob download. Either way, the changing blob notifications are outdated. + // We're either going to empty, or starting fresh blob download. Either way, the changing + // blob notifications are outdated. dynamicState = drainAllChangingBlobs(dynamicState); if (dynamicState.newAssignment == null) { return dynamicState.withState(MachineState.EMPTY); } - Future pendingDownload = staticState.localizer.requestDownloadTopologyBlobs(dynamicState.newAssignment, + Future pendingDownload = staticState.localizer + .requestDownloadTopologyBlobs(dynamicState.newAssignment, staticState.port, staticState.changingCallback); return dynamicState.withPendingLocalization(dynamicState.newAssignment, pendingDownload) .withState(MachineState.WAITING_FOR_BLOB_LOCALIZATION); } - private static DynamicState killContainerFor(KillReason reason, DynamicState dynamicState, StaticState staticState) + private static DynamicState killContainerFor(KillReason reason, DynamicState dynamicState, + StaticState staticState) throws Exception { if (dynamicState.container == null) { throw new Exception("dynamicState.container is null"); } - //Skip special case if `storm kill_workers` is already invoked + // Skip special case if `storm kill_workers` is already invoked Boolean isDead = dynamicState.container.areAllProcessesDead(); if (!isDead) { if (reason == KillReason.ASSIGNMENT_CHANGED || reason == KillReason.BLOB_CHANGED) { @@ -263,10 +290,12 @@ private static DynamicState killContainerFor(KillReason reason, DynamicState dyn case MEMORY_VIOLATION: case HB_TIMEOUT: case HB_NULL: - //any stop profile actions that hadn't timed out yet, we should restart after the worker is running again. + // any stop profile actions that hadn't timed out yet, we should restart after the + // worker is running again. HashSet mod = new HashSet<>(dynamicState.profileActions); mod.addAll(dynamicState.pendingStopProfileActions); - next = dynamicState.withState(MachineState.KILL_AND_RELAUNCH).withProfileActions(mod, Collections.emptySet()); + next = dynamicState.withState(MachineState.KILL_AND_RELAUNCH) + .withProfileActions(mod, Collections.emptySet()); break; default: @@ -282,12 +311,14 @@ private static DynamicState killContainerFor(KillReason reason, DynamicState dyn /** * Clean up a container. * PRECONDITION: All of the processes have died. + * * @param dynamicState current state * @param staticState static data * @param nextState the next MachineState to go to. * @return the next state. */ - private static DynamicState cleanupCurrentContainer(DynamicState dynamicState, StaticState staticState, MachineState nextState) throws + private static DynamicState cleanupCurrentContainer(DynamicState dynamicState, + StaticState staticState, MachineState nextState) throws Exception { if (dynamicState.container == null) { throw new Exception("dynamicState.container is null"); @@ -337,7 +368,8 @@ private static DynamicState drainAllChangingBlobs(DynamicState dynamicState) { } /** - * Informs the async localizer for all of blobs that the worker acknowledged the change of blobs. + * Informs the async localizer for all of blobs that the worker acknowledged the change of + * blobs. * Worker has stop as of now. * *

      PRECONDITION: container is null @@ -346,13 +378,16 @@ private static DynamicState drainAllChangingBlobs(DynamicState dynamicState) { * @param dynamicState the current state * @return the futures for the current assignment. */ - private static DynamicState informChangedBlobs(DynamicState dynamicState, LocalAssignment assignment) { + private static DynamicState informChangedBlobs(DynamicState dynamicState, + LocalAssignment assignment) { if (dynamicState.container != null) { throw new AssertionError("dynamicState.container is not null"); } - boolean allMatch = dynamicState.changingBlobs.stream().allMatch((cr) -> forSameTopology(cr.assignment, assignment)); + boolean allMatch = dynamicState.changingBlobs.stream() + .allMatch((cr) -> forSameTopology(cr.assignment, assignment)); if (!allMatch) { - throw new IllegalArgumentException("dynamicState.changingBlobs assignments for same topology do not match"); + throw new IllegalArgumentException("dynamicState.changingBlobs assignments for same " + + "topology do not match"); } Set> futures = new HashSet<>(dynamicState.changingBlobs.size()); @@ -380,7 +415,8 @@ private static DynamicState informChangedBlobs(DynamicState dynamicState, LocalA * @param assignment the assignment to look for * @return the updated dynamicState */ - private static DynamicState filterChangingBlobsFor(DynamicState dynamicState, final LocalAssignment assignment) { + private static DynamicState filterChangingBlobsFor(DynamicState dynamicState, + final LocalAssignment assignment) { if (dynamicState.changingBlobs.isEmpty()) { return dynamicState; } @@ -398,15 +434,18 @@ private static DynamicState filterChangingBlobsFor(DynamicState dynamicState, fi /** * State Transitions for WAITING_FOR_BLOB_LOCALIZATION state, when the slot is waiting for - * blobs of the pending assignment to be completely downloaded, before the container is launched/relaunched. + * blobs of the pending assignment to be completely downloaded, before the container is + * launched/relaunched. * PRECONDITION: neither pendingLocalization nor pendingDownload is null. * PRECONDITION: The slot should be empty + * * @param dynamicState current state * @param staticState static data * @return the next state * @throws Exception on any error */ - private static DynamicState handleWaitingForBlobLocalization(DynamicState dynamicState, StaticState staticState) throws Exception { + private static DynamicState handleWaitingForBlobLocalization(DynamicState dynamicState, + StaticState staticState) throws Exception { if (dynamicState.pendingLocalization == null) { throw new Exception("dynamicState.pendingLocalization is null"); } @@ -420,24 +459,27 @@ private static DynamicState handleWaitingForBlobLocalization(DynamicState dynami throw new Exception("dynamicState.currentAssignment is not null"); } - //Ignore changes to scheduling while downloading the topology blobs + // Ignore changes to scheduling while downloading the topology blobs // We don't support canceling the download through the future yet, // because pending blobs may be shared by multiple workers and cancel it // may lead to race condition // To keep everything in sync, just wait for all workers try { - //Release things that don't need to wait for us to finish downloading. + // Release things that don't need to wait for us to finish downloading. dynamicState = filterChangingBlobsFor(dynamicState, dynamicState.pendingLocalization); if (!dynamicState.changingBlobs.isEmpty()) { - //Unblock downloading by accepting the futures. + // Unblock downloading by accepting the futures. dynamicState = informChangedBlobs(dynamicState, dynamicState.pendingLocalization); } - if (!EquivalenceUtils.areLocalAssignmentsEquivalent(dynamicState.newAssignment, dynamicState.pendingLocalization)) { - //Scheduling changed + if (!EquivalenceUtils.areLocalAssignmentsEquivalent(dynamicState.newAssignment, + dynamicState.pendingLocalization)) { + // Scheduling changed dynamicState.cancelPendingBlobs(); - staticState.localizer.releaseSlotFor(dynamicState.pendingLocalization, staticState.port); - // Switch to the new assignment even if localization hasn't completed, or go to empty state + staticState.localizer.releaseSlotFor(dynamicState.pendingLocalization, + staticState.port); + // Switch to the new assignment even if localization hasn't completed, or go to + // empty state // if no new assignment. return prepareForNewAssignmentNoWorkersRunning(dynamicState .withPendingLocalization(null, null), @@ -446,22 +488,25 @@ private static DynamicState handleWaitingForBlobLocalization(DynamicState dynami // Wait until time out dynamicState.pendingDownload.get(1000, TimeUnit.MILLISECONDS); - //Downloading of all blobs finished. This is the precondition for all codes below. + // Downloading of all blobs finished. This is the precondition for all codes below. if (!dynamicState.pendingChangingBlobs.isEmpty()) { - LOG.info("There are pending changes, waiting for them to finish before launching container..."); - //We cannot launch the container yet the resources may still be updating + LOG.info("There are pending changes, waiting for them to finish before launching " + + "container..."); + // We cannot launch the container yet the resources may still be updating return dynamicState.withState(MachineState.WAITING_FOR_BLOB_UPDATE) .withPendingLocalization(null, null); } staticState.slotMetrics.numWorkersLaunched.mark(); Container c = - staticState.containerLauncher.launchContainer(staticState.port, dynamicState.pendingLocalization, staticState.localState); + staticState.containerLauncher.launchContainer(staticState.port, + dynamicState.pendingLocalization, staticState.localState); return dynamicState - .withCurrentAssignment(c, dynamicState.pendingLocalization).withState(MachineState.WAITING_FOR_WORKER_START) + .withCurrentAssignment(c, dynamicState.pendingLocalization) + .withState(MachineState.WAITING_FOR_WORKER_START) .withPendingLocalization(null, null); } catch (TimeoutException e) { - //We waited for 1 second loop around and try again.... + // We waited for 1 second loop around and try again.... return dynamicState; } catch (ExecutionException e) { if (e.getCause() instanceof AuthorizationException) { @@ -474,11 +519,12 @@ private static DynamicState handleWaitingForBlobLocalization(DynamicState dynami // release the reference on all blobs associated with this worker. dynamicState.cancelPendingBlobs(); - staticState.localizer.releaseSlotFor(dynamicState.pendingLocalization, staticState.port); + staticState.localizer.releaseSlotFor(dynamicState.pendingLocalization, + staticState.port); // we wait for 3 seconds Time.sleepSecs(3); - //Try again, or go to empty if assignment has been nulled + // Try again, or go to empty if assignment has been nulled return prepareForNewAssignmentNoWorkersRunning(dynamicState .withPendingLocalization(null, null), staticState); @@ -497,7 +543,8 @@ private static DynamicState handleWaitingForBlobLocalization(DynamicState dynami * @return the next state * @throws Exception on any error */ - private static DynamicState handleWaitingForBlobUpdate(DynamicState dynamicState, StaticState staticState) + private static DynamicState handleWaitingForBlobUpdate(DynamicState dynamicState, + StaticState staticState) throws Exception { if (dynamicState.container != null) { throw new Exception("dynamicState.container is not null"); @@ -515,27 +562,32 @@ private static DynamicState handleWaitingForBlobUpdate(DynamicState dynamicState throw new Exception("dynamicState.pendingLocalization is not null"); } - if (!EquivalenceUtils.areLocalAssignmentsEquivalent(dynamicState.newAssignment, dynamicState.currentAssignment)) { - //We were rescheduled while waiting for the resources to be updated, + if (!EquivalenceUtils.areLocalAssignmentsEquivalent(dynamicState.newAssignment, + dynamicState.currentAssignment)) { + // We were rescheduled while waiting for the resources to be updated, // but the container is already not running. LOG.info("SLOT {}: Assignment Changed from {} to {}", staticState.port, dynamicState.currentAssignment, dynamicState.newAssignment); dynamicState.cancelPendingBlobs(); if (dynamicState.currentAssignment != null) { - staticState.localizer.releaseSlotFor(dynamicState.currentAssignment, staticState.port); + staticState.localizer.releaseSlotFor(dynamicState.currentAssignment, + staticState.port); } staticState.localizer.releaseSlotFor(dynamicState.pendingChangingBlobsAssignment, staticState.port); - return prepareForNewAssignmentNoWorkersRunning(dynamicState.withCurrentAssignment(null, null), + return prepareForNewAssignmentNoWorkersRunning(dynamicState.withCurrentAssignment(null, + null), staticState); } - dynamicState = filterChangingBlobsFor(dynamicState, dynamicState.pendingChangingBlobsAssignment); + dynamicState = filterChangingBlobsFor(dynamicState, + dynamicState.pendingChangingBlobsAssignment); if (!dynamicState.changingBlobs.isEmpty()) { - dynamicState = informChangedBlobs(dynamicState, dynamicState.pendingChangingBlobsAssignment); + dynamicState = informChangedBlobs(dynamicState, + dynamicState.pendingChangingBlobsAssignment); } - //We only have a set amount of time we can wait for before looping around again + // We only have a set amount of time we can wait for before looping around again long start = Time.nanoTime(); try { for (Future pending : dynamicState.pendingChangingBlobs) { @@ -546,11 +598,13 @@ private static DynamicState handleWaitingForBlobUpdate(DynamicState dynamicState } pending.get(timeLeft, TimeUnit.NANOSECONDS); } - //All done we can launch the worker now - Container c = staticState.containerLauncher.launchContainer(staticState.port, dynamicState.pendingChangingBlobsAssignment, + // All done we can launch the worker now + Container c = staticState.containerLauncher.launchContainer(staticState.port, + dynamicState.pendingChangingBlobsAssignment, staticState.localState); return dynamicState - .withCurrentAssignment(c, dynamicState.pendingChangingBlobsAssignment).withState(MachineState.WAITING_FOR_WORKER_START) + .withCurrentAssignment(c, dynamicState.pendingChangingBlobsAssignment) + .withState(MachineState.WAITING_FOR_WORKER_START) .withPendingChangingBlobs(Collections.emptySet(), null); } catch (TimeoutException ex) { return dynamicState; @@ -561,12 +615,14 @@ private static DynamicState handleWaitingForBlobUpdate(DynamicState dynamicState * State Transitions for KILL state. * PRECONDITION: container.kill() was called * PRECONDITION: container != null && currentAssignment != null + * * @param dynamicState current state * @param staticState static data * @return the next state * @throws Exception on any error */ - private static DynamicState handleKill(DynamicState dynamicState, StaticState staticState) throws Exception { + private static DynamicState handleKill(DynamicState dynamicState, + StaticState staticState) throws Exception { if (dynamicState.container == null) { throw new Exception("dynamicState.container is null"); } @@ -599,12 +655,14 @@ private static DynamicState handleKill(DynamicState dynamicState, StaticState st * State Transitions for KILL_AND_RELAUNCH state. * PRECONDITION: container.kill() was called * PRECONDITION: container != null && currentAssignment != null + * * @param dynamicState current state * @param staticState static data * @return the next state * @throws Exception on any error */ - private static DynamicState handleKillAndRelaunch(DynamicState dynamicState, StaticState staticState) throws Exception { + private static DynamicState handleKillAndRelaunch(DynamicState dynamicState, + StaticState staticState) throws Exception { if (dynamicState.container == null) { throw new Exception("dynamicState.container is null"); } @@ -625,17 +683,21 @@ private static DynamicState handleKillAndRelaunch(DynamicState dynamicState, Sta } if (dynamicState.container.areAllProcessesDead()) { - if (EquivalenceUtils.areLocalAssignmentsEquivalent(dynamicState.newAssignment, dynamicState.currentAssignment)) { + if (EquivalenceUtils.areLocalAssignmentsEquivalent(dynamicState.newAssignment, + dynamicState.currentAssignment)) { dynamicState.container.cleanUpForRestart(); dynamicState.container.relaunch(); return dynamicState.withState(MachineState.WAITING_FOR_WORKER_START); } - //Scheduling changed after we killed all of the processes - return prepareForNewAssignmentNoWorkersRunning(cleanupCurrentContainer(dynamicState, staticState, null), staticState); + // Scheduling changed after we killed all of the processes + return prepareForNewAssignmentNoWorkersRunning(cleanupCurrentContainer(dynamicState, + staticState, null), staticState); } - //The child processes typically exit in < 1 sec. If 2 mins later they are still around something is wrong + // The child processes typically exit in < 1 sec. If 2 mins later they are still around + // something is wrong if ((Time.currentTimeMillis() - dynamicState.startTime) > 120_000) { - throw new RuntimeException("Not all processes in " + dynamicState.container + " exited after 120 seconds"); + throw new RuntimeException("Not all processes in " + dynamicState.container + + " exited after 120 seconds"); } dynamicState.container.forceKill(); Time.sleep(staticState.killSleepMs); @@ -652,7 +714,8 @@ private static DynamicState handleKillAndRelaunch(DynamicState dynamicState, Sta * @return the next state * @throws Exception on any error */ - private static DynamicState handleKillBlobUpdate(DynamicState dynamicState, StaticState staticState) throws Exception { + private static DynamicState handleKillBlobUpdate(DynamicState dynamicState, + StaticState staticState) throws Exception { if (dynamicState.container == null) { throw new Exception("dynamicState.container is null"); } @@ -672,22 +735,27 @@ private static DynamicState handleKillBlobUpdate(DynamicState dynamicState, Stat throw new Exception("dynamicState.pendingDownload is not null"); } - //Release things that don't need to wait for us + // Release things that don't need to wait for us dynamicState = filterChangingBlobsFor(dynamicState, dynamicState.currentAssignment); if (dynamicState.container.areAllProcessesDead()) { - if (EquivalenceUtils.areLocalAssignmentsEquivalent(dynamicState.newAssignment, dynamicState.currentAssignment)) { + if (EquivalenceUtils.areLocalAssignmentsEquivalent(dynamicState.newAssignment, + dynamicState.currentAssignment)) { dynamicState.container.cleanUp(); - dynamicState = dynamicState.withCurrentAssignment(null, dynamicState.currentAssignment); + dynamicState = dynamicState.withCurrentAssignment(null, + dynamicState.currentAssignment); return informChangedBlobs(dynamicState, dynamicState.currentAssignment) .withState(MachineState.WAITING_FOR_BLOB_UPDATE); } - //Scheduling changed after we killed all of the processes - return prepareForNewAssignmentNoWorkersRunning(cleanupCurrentContainer(dynamicState, staticState, null), staticState); + // Scheduling changed after we killed all of the processes + return prepareForNewAssignmentNoWorkersRunning(cleanupCurrentContainer(dynamicState, + staticState, null), staticState); } - //The child processes typically exit in < 1 sec. If 2 mins later they are still around something is wrong + // The child processes typically exit in < 1 sec. If 2 mins later they are still around + // something is wrong if ((Time.currentTimeMillis() - dynamicState.startTime) > 120_000) { - throw new RuntimeException("Not all processes in " + dynamicState.container + " exited after 120 seconds"); + throw new RuntimeException("Not all processes in " + dynamicState.container + + " exited after 120 seconds"); } dynamicState.container.forceKill(); Time.sleep(staticState.killSleepMs); @@ -697,12 +765,14 @@ private static DynamicState handleKillBlobUpdate(DynamicState dynamicState, Stat /** * State Transitions for WAITING_FOR_WORKER_START state. * PRECONDITION: container != null && currentAssignment != null + * * @param dynamicState current state * @param staticState static data * @return the next state * @throws Exception on any error */ - private static DynamicState handleWaitingForWorkerStart(DynamicState dynamicState, StaticState staticState) throws Exception { + private static DynamicState handleWaitingForWorkerStart(DynamicState dynamicState, + StaticState staticState) throws Exception { if (dynamicState.container == null) { throw new Exception("dynamicState.container is null"); } @@ -731,9 +801,11 @@ private static DynamicState handleWaitingForWorkerStart(DynamicState dynamicStat } } - if (!EquivalenceUtils.areLocalAssignmentsEquivalent(dynamicState.newAssignment, dynamicState.currentAssignment)) { - //We were rescheduled while waiting for the worker to come up - LOG.info("SLOT {}: Assignment Changed from {} to {}", staticState.port, dynamicState.currentAssignment, + if (!EquivalenceUtils.areLocalAssignmentsEquivalent(dynamicState.newAssignment, + dynamicState.currentAssignment)) { + // We were rescheduled while waiting for the worker to come up + LOG.info("SLOT {}: Assignment Changed from {} to {}", staticState.port, + dynamicState.currentAssignment, dynamicState.newAssignment); return killContainerFor(KillReason.ASSIGNMENT_CHANGED, dynamicState, staticState); } @@ -743,14 +815,15 @@ private static DynamicState handleWaitingForWorkerStart(DynamicState dynamicStat long hbFirstTimeoutMs = getFirstHbTimeoutMs(staticState, dynamicState); if (timeDiffms > hbFirstTimeoutMs) { staticState.slotMetrics.numWorkerStartTimedOut.mark(); - LOG.warn("SLOT {}: Container {} failed to launch in {} ms.", staticState.port, dynamicState.container, + LOG.warn("SLOT {}: Container {} failed to launch in {} ms.", staticState.port, + dynamicState.container, hbFirstTimeoutMs); return killContainerFor(KillReason.HB_TIMEOUT, dynamicState, staticState); } dynamicState = filterChangingBlobsFor(dynamicState, dynamicState.currentAssignment); if (!dynamicState.changingBlobs.isEmpty()) { - //Kill the container and restart it + // Kill the container and restart it return killContainerFor(KillReason.BLOB_CHANGED, dynamicState, staticState); } Time.sleep(1000); @@ -760,12 +833,14 @@ private static DynamicState handleWaitingForWorkerStart(DynamicState dynamicStat /** * State Transitions for RUNNING state. * PRECONDITION: container != null && currentAssignment != null + * * @param dynamicState current state * @param staticState static data * @return the next state * @throws Exception on any error */ - private static DynamicState handleRunning(DynamicState dynamicState, StaticState staticState) throws Exception { + private static DynamicState handleRunning(DynamicState dynamicState, + StaticState staticState) throws Exception { if (dynamicState.container == null) { throw new Exception("dynamicState.container is null"); } @@ -785,17 +860,19 @@ private static DynamicState handleRunning(DynamicState dynamicState, StaticState throw new Exception("dynamicState.pendingDownload is not null"); } - if (!EquivalenceUtils.areLocalAssignmentsEquivalent(dynamicState.newAssignment, dynamicState.currentAssignment)) { - LOG.info("SLOT {}: Assignment Changed from {} to {}", staticState.port, dynamicState.currentAssignment, + if (!EquivalenceUtils.areLocalAssignmentsEquivalent(dynamicState.newAssignment, + dynamicState.currentAssignment)) { + LOG.info("SLOT {}: Assignment Changed from {} to {}", staticState.port, + dynamicState.currentAssignment, dynamicState.newAssignment); - //Scheduling changed while running... + // Scheduling changed while running... return killContainerFor(KillReason.ASSIGNMENT_CHANGED, dynamicState, staticState); } dynamicState = updateAssignmentIfNeeded(dynamicState); dynamicState = filterChangingBlobsFor(dynamicState, dynamicState.currentAssignment); if (!dynamicState.changingBlobs.isEmpty()) { - //Kill the container and restart it + // Kill the container and restart it return killContainerFor(KillReason.BLOB_CHANGED, dynamicState, staticState); } @@ -815,7 +892,7 @@ private static DynamicState handleRunning(DynamicState dynamicState, StaticState if (hb == null) { LOG.warn("SLOT {}: HB returned as null for topology: {}", staticState.port, dynamicState.currentAssignment.get_topology_id()); - //This can happen if the supervisor crashed after launching a + // This can happen if the supervisor crashed after launching a // worker that never came up. return killContainerFor(KillReason.HB_NULL, dynamicState, staticState); } @@ -824,24 +901,27 @@ private static DynamicState handleRunning(DynamicState dynamicState, StaticState long hbTimeoutMs = getHbTimeoutMs(staticState, dynamicState); if (timeDiffMs > hbTimeoutMs) { LOG.warn("SLOT {}: HB is too old {} > {} for topology: {}", - staticState.port, timeDiffMs, hbTimeoutMs, dynamicState.currentAssignment.get_topology_id()); + staticState.port, timeDiffMs, hbTimeoutMs, dynamicState.currentAssignment + .get_topology_id()); return killContainerFor(KillReason.HB_TIMEOUT, dynamicState, staticState); } - //The worker is up and running check for profiling requests + // The worker is up and running check for profiling requests if (!dynamicState.profileActions.isEmpty()) { HashSet mod = new HashSet<>(dynamicState.profileActions); - HashSet modPending = new HashSet<>(dynamicState.pendingStopProfileActions); + HashSet modPending = + new HashSet<>(dynamicState.pendingStopProfileActions); Iterator iter = mod.iterator(); while (iter.hasNext()) { TopoProfileAction action = iter.next(); if (!action.topoId.equals(dynamicState.currentAssignment.get_topology_id())) { iter.remove(); LOG.warn("Dropping {} wrong topology is running", action); - //Not for this topology so skip it + // Not for this topology so skip it } else { if (modPending.contains(action)) { - boolean isTimeForStop = Time.currentTimeMillis() > action.request.get_time_stamp(); + boolean isTimeForStop = Time.currentTimeMillis() > action.request + .get_time_stamp(); if (isTimeForStop) { if (dynamicState.container.runProfiling(action.request, true)) { LOG.debug("Stopped {} action finished", action); @@ -854,7 +934,7 @@ private static DynamicState handleRunning(DynamicState dynamicState, StaticState LOG.debug("Still pending {} now: {}", action, Time.currentTimeMillis()); } } else { - //J_PROFILE_START is not used. When you see a J_PROFILE_STOP + // J_PROFILE_START is not used. When you see a J_PROFILE_STOP // start profiling and save it away to stop when timeout happens if (action.request.get_action() == ProfileAction.JPROFILE_STOP) { if (dynamicState.container.runProfiling(action.request, false)) { @@ -877,13 +957,15 @@ private static DynamicState handleRunning(DynamicState dynamicState, StaticState dynamicState = dynamicState.withProfileActions(mod, modPending); } - dynamicState.container.processMetrics(staticState.metricsExec, staticState.metricsProcessor); + dynamicState.container.processMetrics(staticState.metricsExec, + staticState.metricsProcessor); Time.sleep(staticState.monitorFreqMs); return dynamicState; } - static DynamicState handleEmpty(DynamicState dynamicState, StaticState staticState) throws InterruptedException, IOException { + static DynamicState handleEmpty(DynamicState dynamicState, + StaticState staticState) throws InterruptedException, IOException { if (dynamicState.container != null) { throw new IOException("dynamicState.container is not null"); } @@ -903,18 +985,20 @@ static DynamicState handleEmpty(DynamicState dynamicState, StaticState staticSta throw new IOException("dynamicState.pendingDownload is not null"); } - if (!EquivalenceUtils.areLocalAssignmentsEquivalent(dynamicState.newAssignment, dynamicState.currentAssignment)) { + if (!EquivalenceUtils.areLocalAssignmentsEquivalent(dynamicState.newAssignment, + dynamicState.currentAssignment)) { return prepareForNewAssignmentNoWorkersRunning(dynamicState, staticState); } dynamicState = updateAssignmentIfNeeded(dynamicState); - //Both assignments are null, just wait + // Both assignments are null, just wait if (dynamicState.profileActions != null && !dynamicState.profileActions.isEmpty()) { - //Nothing is scheduled here so throw away all of the profileActions + // Nothing is scheduled here so throw away all of the profileActions LOG.warn("Dropping {} no topology is running", dynamicState.profileActions); - dynamicState = dynamicState.withProfileActions(Collections.emptySet(), Collections.emptySet()); + dynamicState = dynamicState.withProfileActions(Collections.emptySet(), Collections + .emptySet()); } - //Drop the change notifications we are not running anything right now + // Drop the change notifications we are not running anything right now dynamicState = drainAllChangingBlobs(dynamicState); Time.sleep(1000); return dynamicState; @@ -932,7 +1016,8 @@ private static long getHbTimeoutMs(StaticState staticState, DynamicState dynamic Map topoConf = dynamicState.container.topoConf; if (topoConf != null && topoConf.containsKey(Config.TOPOLOGY_WORKER_TIMEOUT_SECS)) { - long topoHbTimeoutMs = ObjectReader.getInt(topoConf.get(Config.TOPOLOGY_WORKER_TIMEOUT_SECS)) * 1000; + long topoHbTimeoutMs = ObjectReader.getInt(topoConf + .get(Config.TOPOLOGY_WORKER_TIMEOUT_SECS)) * 1000; topoHbTimeoutMs = Math.max(topoHbTimeoutMs, hbTimeoutMs); hbTimeoutMs = topoHbTimeoutMs; } @@ -942,7 +1027,8 @@ private static long getHbTimeoutMs(StaticState staticState, DynamicState dynamic /* * Get worker heartbeat timeout when waiting for worker to start. - * If topology specific timeout if set, ensure first heartbeat timeout >= topology specific timeout. + * If topology specific timeout if set, ensure first heartbeat timeout >= topology specific + * timeout. */ private static long getFirstHbTimeoutMs(StaticState staticState, DynamicState dynamicState) { return Math.max(getHbTimeoutMs(staticState, dynamicState), staticState.firstHbTimeoutMs); @@ -950,24 +1036,29 @@ private static long getFirstHbTimeoutMs(StaticState staticState, DynamicState dy /** * Set a new assignment asynchronously. + * * @param newAssignment the new assignment for this slot to run, null to run nothing */ public final void setNewAssignment(LocalAssignment newAssignment) { this.newAssignment.set(newAssignment == null ? null - : new TimerDecoratedAssignment(newAssignment, staticState.slotMetrics.workerLaunchDuration)); + : new TimerDecoratedAssignment(newAssignment, + staticState.slotMetrics.workerLaunchDuration)); } @Override - public void blobChanging(LocalAssignment assignment, int port, LocallyCachedBlob blob, GoodToGo go) { + public void blobChanging(LocalAssignment assignment, int port, LocallyCachedBlob blob, + GoodToGo go) { if (port != staticState.port) { - throw new AssertionError("got a callback that is not for us " + port + " != " + staticState.port); + throw new AssertionError("got a callback that is not for us " + port + " != " + + staticState.port); } - //This is called async so lets assume that it is something we care about + // This is called async so lets assume that it is something we care about try { changingBlobs.put(new BlobChanging(assignment, blob, go.getLatch())); } catch (InterruptedException e) { - throw new RuntimeException("This should not have happened, but it did (the queue is unbounded)", e); + throw new RuntimeException("This should not have happened, but it did (the queue is " + + "unbounded)", e); } } @@ -985,7 +1076,8 @@ public void addProfilerActions(Set actions) { } /** - * get the workerID (nullable) from CURRENT container, if existed, or return null. + * Get the workerID (nullable) from CURRENT container, if existed, or return null. + * * @return workerID */ public String getWorkerId() { @@ -999,7 +1091,8 @@ public String getWorkerId() { private void saveNewAssignment(LocalAssignment assignment) { synchronized (staticState.localState) { - Map assignments = staticState.localState.getLocalAssignmentsMap(); + Map assignments = staticState.localState + .getLocalAssignmentsMap(); if (assignments == null) { assignments = new HashMap<>(); } @@ -1036,12 +1129,12 @@ public void run() { changingBlobs.drainTo(changingResourcesToHandle); Iterator it = changingResourcesToHandle.iterator(); - //Remove/Clean up changed requests that are not for us + // Remove/Clean up changed requests that are not for us while (it.hasNext()) { BlobChanging rc = it.next(); if (!forSameTopology(rc.assignment, dynamicState.currentAssignment) && !forSameTopology(rc.assignment, dynamicState.newAssignment)) { - rc.latch.countDown(); //Ignore the future + rc.latch.countDown(); // Ignore the future it.remove(); } } @@ -1049,33 +1142,41 @@ public void run() { DynamicState nextState = stateMachineStep(dynamicState.withNewAssignment(newAssignment.get()) - .withProfileActions(origProfileActions, dynamicState.pendingStopProfileActions) + .withProfileActions(origProfileActions, + dynamicState.pendingStopProfileActions) .withChangingBlobs(changingResourcesToHandle), staticState); if (LOG.isDebugEnabled() || dynamicState.state != nextState.state) { LOG.info("STATE {} -> {}", dynamicState, nextState); } - //Save the current state for recovery + // Save the current state for recovery if ((nextState.currentAssignment != null - && !nextState.currentAssignment.equals(dynamicState.currentAssignment)) + && !nextState.currentAssignment + .equals(dynamicState.currentAssignment)) || (dynamicState.currentAssignment != null - && !dynamicState.currentAssignment.equals(nextState.currentAssignment))) { - LOG.info("SLOT {}: Changing current assignment from {} to {}", staticState.port, dynamicState.currentAssignment, + && !dynamicState.currentAssignment + .equals(nextState.currentAssignment))) { + LOG.info("SLOT {}: Changing current assignment from {} to {}", staticState.port, + dynamicState.currentAssignment, nextState.currentAssignment); saveNewAssignment(nextState.currentAssignment); } - if (EquivalenceUtils.areLocalAssignmentsEquivalent(nextState.newAssignment, nextState.currentAssignment) + if (EquivalenceUtils.areLocalAssignmentsEquivalent(nextState.newAssignment, + nextState.currentAssignment) && nextState.currentAssignment != null && nextState.currentAssignment.get_owner() == null && nextState.newAssignment != null && nextState.newAssignment.get_owner() != null) { - //This is an odd case for a rolling upgrade where the user on the old assignment may be null, + // This is an odd case for a rolling upgrade where the user on the old + // assignment may be null, // but not on the new one. Although in all other ways they are the same. // If this happens we want to use the assignment with the owner. - LOG.info("Updating assignment to save owner {}", nextState.newAssignment.get_owner()); + LOG.info("Updating assignment to save owner {}", nextState.newAssignment + .get_owner()); saveNewAssignment(nextState.newAssignment); - nextState = nextState.withCurrentAssignment(nextState.container, nextState.newAssignment); + nextState = nextState.withCurrentAssignment(nextState.container, + nextState.newAssignment); } // clean up the profiler actions that are not being processed @@ -1086,7 +1187,8 @@ public void run() { try { clusterState.deleteTopologyProfileRequests(action.topoId, action.request); } catch (Exception e) { - LOG.error("Error trying to remove profiling request, it will be retried", e); + LOG.error("Error trying to remove profiling request, it will be retried", + e); } } Set orig; @@ -1127,23 +1229,28 @@ enum MachineState { */ WAITING_FOR_WORKER_START, /** - * Slot has just killed its worker, and is now waiting for it to die so it can be relaunched in the same container. + * Slot has just killed its worker, and is now waiting for it to die so it can be relaunched + * in the same container. */ KILL_AND_RELAUNCH, /** - * Slot has just killed its worker, and is now waiting for it to die so the container can be deleted. + * Slot has just killed its worker, and is now waiting for it to die so the container can be + * deleted. */ KILL, /** - * Slot has just killed its worker, and is now waiting for it to die so the localizer can update a blob. + * Slot has just killed its worker, and is now waiting for it to die so the localizer can + * update a blob. */ KILL_BLOB_UPDATE, /** - * The slot is empty, and is waiting for blobs to download before the worker can be launched. + * The slot is empty, and is waiting for blobs to download before the worker can be + * launched. */ WAITING_FOR_BLOB_LOCALIZATION, /** - * The slot is empty, and is waiting for blobs to be updated before the worker can be (re)launched. + * The slot is empty, and is waiting for blobs to be updated before the worker can be + * (re)launched. */ WAITING_FOR_BLOB_UPDATE; @@ -1224,9 +1331,12 @@ static class DynamicState { public final Set pendingStopProfileActions; /** - * Blobs that are changed and need to be synced. The localizer notifies the Slot about changing blobs on every state step. - * Blob updates are blocked until the Slot unblocks them, at which point they go in {@link #pendingChangingBlobs}. - * Updates are blocked until the Slot worker is dead, since blobs may otherwise be actively used. + * Blobs that are changed and need to be synced. The localizer notifies the Slot about + * changing blobs on every state step. + * Blob updates are blocked until the Slot unblocks them, at which point they go in {@link + * #pendingChangingBlobs}. + * Updates are blocked until the Slot worker is dead, since blobs may otherwise be actively + * used. */ public final Set changingBlobs; @@ -1243,12 +1353,14 @@ static class DynamicState { public final long startTime; private final SlotMetrics slotMetrics; - DynamicState(final LocalAssignment currentAssignment, Container container, final LocalAssignment newAssignment, + DynamicState(final LocalAssignment currentAssignment, Container container, + final LocalAssignment newAssignment, SlotMetrics slotMetrics) { this.currentAssignment = currentAssignment; this.container = container; if ((currentAssignment == null) ^ (container == null)) { - throw new IllegalArgumentException("Container and current assignment must both be null, or neither can be null"); + throw new IllegalArgumentException("Container and current assignment must both be " + + "null, or neither can be null"); } if (currentAssignment == null) { @@ -1282,7 +1394,8 @@ static class DynamicState { throw new AssertionError("pendingChangingBlobs is null"); } if (pendingChangingBlobs.isEmpty() != (pendingChaningBlobsAssignment == null)) { - throw new AssertionError("pendingChangingBlobs.isEmpty=" + pendingChangingBlobs.isEmpty() + throw new AssertionError("pendingChangingBlobs.isEmpty=" + pendingChangingBlobs + .isEmpty() + " but pendingChaningBlobsAssignment is " + (pendingChaningBlobsAssignment == null ? "null" : "not null")); } @@ -1315,8 +1428,10 @@ public String toString() { } /** - * Set the new assignment for the state. This should never be called from within the state machine. + * Set the new assignment for the state. This should never be called from within the state + * machine. * It is an input from outside. + * * @param newAssignment the new assignment to set * @return the updated DynamicState. */ @@ -1329,7 +1444,8 @@ public DynamicState withNewAssignment(LocalAssignment newAssignment) { this.pendingChangingBlobs, this.pendingChangingBlobsAssignment, this.slotMetrics); } - public DynamicState withPendingLocalization(LocalAssignment pendingLocalization, Future pendingDownload) { + public DynamicState withPendingLocalization(LocalAssignment pendingLocalization, + Future pendingDownload) { return new DynamicState(this.state, this.newAssignment, this.container, this.currentAssignment, pendingLocalization, this.startTime, @@ -1345,20 +1461,23 @@ public DynamicState withPendingLocalization(Future pendingDownload) { /** * Transition to the given state. Notice that it's possible to transition to * the same state. + * * @param state The state to transition into * @return New dynamicState */ public DynamicState withState(final MachineState state) { long newStartTime = Time.currentTimeMillis(); - //We may (though unlikely) lose metering here if state transition is too frequent (less than a millisecond) - slotMetrics.timeSpentInState.get(this.state).update(newStartTime - startTime, TimeUnit.MILLISECONDS); + // We may (though unlikely) lose metering here if state transition is too frequent (less + // than a millisecond) + slotMetrics.timeSpentInState.get(this.state).update(newStartTime - startTime, + TimeUnit.MILLISECONDS); slotMetrics.transitionIntoState.get(state).mark(); LocalAssignment assignment = this.currentAssignment; if (MachineState.RUNNING != this.state && MachineState.RUNNING == state && this.currentAssignment instanceof TimerDecoratedAssignment) { ((TimerDecoratedAssignment) assignment).stopTiming(); - //Timer is discarded after the initial launch of an assignment + // Timer is discarded after the initial launch of an assignment assignment = new LocalAssignment(this.currentAssignment); } @@ -1370,7 +1489,8 @@ public DynamicState withState(final MachineState state) { this.pendingChangingBlobs, this.pendingChangingBlobsAssignment, this.slotMetrics); } - public DynamicState withCurrentAssignment(final Container container, final LocalAssignment currentAssignment) { + public DynamicState withCurrentAssignment(final Container container, + final LocalAssignment currentAssignment) { return new DynamicState(this.state, this.newAssignment, container, currentAssignment, this.pendingLocalization, this.startTime, @@ -1379,7 +1499,8 @@ public DynamicState withCurrentAssignment(final Container container, final Local this.pendingChangingBlobs, this.pendingChangingBlobsAssignment, this.slotMetrics); } - public DynamicState withProfileActions(Set profileActions, Set pendingStopProfileActions) { + public DynamicState withProfileActions(Set profileActions, + Set pendingStopProfileActions) { return new DynamicState(this.state, this.newAssignment, this.container, this.currentAssignment, this.pendingLocalization, this.startTime, @@ -1389,7 +1510,8 @@ public DynamicState withProfileActions(Set profileActions, Se } /** - * Set the blocked changing blobs. This is an input from the outside, and should never be called by the state machine steps. + * Set the blocked changing blobs. This is an input from the outside, and should never be + * called by the state machine steps. */ public DynamicState withChangingBlobs(Set changingBlobs) { if (changingBlobs == this.changingBlobs) { @@ -1464,7 +1586,8 @@ static class BlobChanging { private final LocallyCachedBlob blob; private final GoodToGo.GoodToGoLatch latch; - BlobChanging(LocalAssignment assignment, LocallyCachedBlob blob, GoodToGo.GoodToGoLatch latch) { + BlobChanging(LocalAssignment assignment, LocallyCachedBlob blob, + GoodToGo.GoodToGoLatch latch) { this.assignment = assignment; this.blob = blob; this.latch = latch; diff --git a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/SlotMetrics.java b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/SlotMetrics.java index 8b2f5f13186..df16447d06a 100644 --- a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/SlotMetrics.java +++ b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/SlotMetrics.java @@ -30,19 +30,25 @@ class SlotMetrics { final Map numWorkersKilledFor; final Timer workerLaunchDuration; final Map transitionIntoState; - //This also tracks how many times worker transitioning out of a state + // This also tracks how many times worker transitioning out of a state final Map timeSpentInState; SlotMetrics(StormMetricsRegistry metricsRegistry) { numWorkersLaunched = metricsRegistry.registerMeter("supervisor:num-workers-launched"); - numWorkerStartTimedOut = metricsRegistry.registerMeter("supervisor:num-worker-start-timed-out"); + numWorkerStartTimedOut = metricsRegistry + .registerMeter("supervisor:num-worker-start-timed-out"); numWorkersKilledFor = Collections.unmodifiableMap(EnumUtil.toEnumMap(Slot.KillReason.class, - killReason -> metricsRegistry.registerMeter("supervisor:num-workers-killed-" + killReason.toString()))); + killReason -> metricsRegistry.registerMeter("supervisor:num-workers-killed-" + + killReason.toString()))); workerLaunchDuration = metricsRegistry.registerTimer("supervisor:worker-launch-duration"); - transitionIntoState = Collections.unmodifiableMap(EnumUtil.toEnumMap(Slot.MachineState.class, - machineState -> metricsRegistry.registerMeter("supervisor:num-worker-transitions-into-" + machineState.toString()))); + transitionIntoState = Collections.unmodifiableMap(EnumUtil + .toEnumMap(Slot.MachineState.class, + machineState -> metricsRegistry + .registerMeter("supervisor:num-worker-transitions-into-" + + machineState.toString()))); timeSpentInState = Collections.unmodifiableMap(EnumUtil.toEnumMap(Slot.MachineState.class, - machineState -> metricsRegistry.registerTimer("supervisor:time-worker-spent-in-state-" + machineState.toString() + "-ms"))); + machineState -> metricsRegistry.registerTimer("supervisor:time-worker-spent-in-state-" + + machineState.toString() + "-ms"))); } } diff --git a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/StandaloneSupervisor.java b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/StandaloneSupervisor.java index b002925dc3e..e65eaef0c0b 100644 --- a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/StandaloneSupervisor.java +++ b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/StandaloneSupervisor.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/Supervisor.java b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/Supervisor.java index 50b1bba9f44..4095c418774 100644 --- a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/Supervisor.java +++ b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/Supervisor.java @@ -19,7 +19,6 @@ package org.apache.storm.daemon.supervisor; import com.codahale.metrics.Meter; - import java.io.File; import java.io.IOException; import java.net.BindException; @@ -105,7 +104,7 @@ public class Supervisor implements DaemonCommon, AutoCloseable { private final StormTimer heartbeatTimer; private final StormTimer workerHeartbeatTimer; private final StormTimer eventTimer; - //Right now this is only used for sending metrics to nimbus, + // Right now this is only used for sending metrics to nimbus, // but we may want to combine it with the heartbeatTimer at some point // to really make this work well. private final ExecutorService heartbeatExecutor; @@ -120,9 +119,9 @@ public class Supervisor implements DaemonCommon, AutoCloseable { private FileWatcher keyStoreWatcher; private MultiThriftServer multiThriftServer; - //used for local cluster heartbeating + // used for local cluster heartbeating private Nimbus.Iface localNimbus; - //Passed to workers in local clusters, exposed by thrift server in distributed mode + // Passed to workers in local clusters, exposed by thrift server in distributed mode private org.apache.storm.generated.Supervisor.Iface supervisorThriftInterface; @SuppressWarnings("checkstyle:ParameterName") @@ -139,7 +138,8 @@ private Supervisor(ISupervisor iSupervisor, StormMetricsRegistry metricsRegistry * @param iSupervisor {@link ISupervisor} */ @SuppressWarnings("checkstyle:ParameterName") - public Supervisor(Map conf, IContext sharedContext, ISupervisor iSupervisor, StormMetricsRegistry metricsRegistry) + public Supervisor(Map conf, IContext sharedContext, ISupervisor iSupervisor, + StormMetricsRegistry metricsRegistry) throws IOException, IllegalAccessException, ClassNotFoundException, InstantiationException { this.conf = conf; this.metricsRegistry = metricsRegistry; @@ -154,7 +154,8 @@ public Supervisor(Map conf, IContext sharedContext, ISupervisor this.authorizationHandler = StormCommon.mkAuthorizationHandler( (String) conf.get(DaemonConfig.SUPERVISOR_AUTHORIZER), conf); if (authorizationHandler == null && conf.get(DaemonConfig.NIMBUS_AUTHORIZER) != null) { - throw new IllegalStateException("It looks like authorization is turned on for nimbus but not for the " + throw new IllegalStateException("It looks like authorization is turned on for nimbus " + + "but not for the " + "supervisor. ( " + DaemonConfig.SUPERVISOR_AUTHORIZER + " is not set)"); } @@ -188,14 +189,15 @@ public Supervisor(Map conf, IContext sharedContext, ISupervisor this.heartbeatTimer = new StormTimer("HBTimer", new DefaultUncaughtExceptionHandler()); - this.workerHeartbeatTimer = new StormTimer("WorkerHBTimer", new DefaultUncaughtExceptionHandler()); + this.workerHeartbeatTimer = new StormTimer("WorkerHBTimer", + new DefaultUncaughtExceptionHandler()); this.eventTimer = new StormTimer("EventTimer", new DefaultUncaughtExceptionHandler()); this.supervisorThriftInterface = createSupervisorIface(); } /** - * supervisor daemon enter entrance. + * Supervisor daemon enter entrance. */ public static void main(String[] args) throws Exception { Utils.setupDefaultUncaughtExceptionHandler(); @@ -306,8 +308,10 @@ public void launch() throws Exception { SupervisorHeartbeat hb = new SupervisorHeartbeat(conf, this); hb.run(); - // should synchronize supervisor so it doesn't launch anything after being down (optimization) - Integer heartbeatFrequency = ObjectReader.getInt(conf.get(DaemonConfig.SUPERVISOR_HEARTBEAT_FREQUENCY_SECS)); + // should synchronize supervisor so it doesn't launch anything after being down + // (optimization) + Integer heartbeatFrequency = ObjectReader.getInt(conf + .get(DaemonConfig.SUPERVISOR_HEARTBEAT_FREQUENCY_SECS)); heartbeatTimer.scheduleRecurring(0, heartbeatFrequency, hb); this.eventManager = new EventManagerImp(false); @@ -316,7 +320,8 @@ public void launch() throws Exception { asyncLocalizer.start(); if ((Boolean) conf.get(DaemonConfig.SUPERVISOR_ENABLE)) { - // This isn't strictly necessary, but it doesn't hurt and ensures that the machine stays up + // This isn't strictly necessary, but it doesn't hurt and ensures that the machine stays + // up // to date even if callbacks don't all work exactly right eventTimer.scheduleRecurring(0, 10, new EventManagerPushCallback(new SynchronizeAssignments(this, null, readState), eventManager)); @@ -326,14 +331,15 @@ public void launch() throws Exception { } ReportWorkerHeartbeats reportWorkerHeartbeats = new ReportWorkerHeartbeats(conf, this); - Integer workerHeartbeatFrequency = ObjectReader.getInt(conf.get(Config.WORKER_HEARTBEAT_FREQUENCY_SECS)); + Integer workerHeartbeatFrequency = ObjectReader.getInt(conf + .get(Config.WORKER_HEARTBEAT_FREQUENCY_SECS)); workerHeartbeatTimer.scheduleRecurring(0, workerHeartbeatFrequency, reportWorkerHeartbeats); LOG.info("Starting supervisor with id {} at host {}.", getId(), getHostName()); } /** - * start distribute supervisor. + * Start distribute supervisor. */ public void launchDaemon() { LOG.info("Starting supervisor for storm version '{}'.", VersionInfo.getVersion()); @@ -344,10 +350,12 @@ public void launchDaemon() { } launch(); - metricsRegistry.registerGauge("supervisor:num-slots-used-gauge", () -> SupervisorUtils.supervisorWorkerIds(conf).size()); - //This will only get updated once + metricsRegistry.registerGauge("supervisor:num-slots-used-gauge", () -> SupervisorUtils + .supervisorWorkerIds(conf).size()); + // This will only get updated once metricsRegistry.registerMeter("supervisor:num-launched").mark(); - metricsRegistry.registerMeter("supervisor:num-shell-exceptions", ShellUtils.numShellExceptions); + metricsRegistry.registerMeter("supervisor:num-shell-exceptions", + ShellUtils.numShellExceptions); metricsRegistry.registerMeter(Constants.SUPERVISOR_HEALTH_CHECK_TIMEOUTS); killErrorMeter = metricsRegistry.registerMeter("supervisor:num-kill-worker-errors"); metricsRegistry.registerMeter("supervisor:workerTokenAuthorizer-get-password-failures", @@ -358,7 +366,8 @@ public void launchDaemon() { this.close(); }); - // blocking call under the hood, must invoke after launch cause some services must be initialized + // blocking call under the hood, must invoke after launch cause some services must be + // initialized launchSupervisorThriftServer(conf); } catch (Exception e) { LOG.error("Failed to start supervisor\n", e); @@ -378,7 +387,8 @@ public void checkAuthorization(String topoName, Map topoConf, St } @VisibleForTesting - public void checkAuthorization(String topoName, Map topoConf, String operation, ReqContext context) + public void checkAuthorization(String topoName, Map topoConf, String operation, + ReqContext context) throws AuthorizationException { if (context == null) { context = ReqContext.context(); @@ -391,7 +401,8 @@ public void checkAuthorization(String topoName, Map topoConf, St } if (context.isImpersonating()) { - LOG.info("principal: {} is trying to impersonate principal: {}", context.realPrincipal(), + LOG.info("principal: {} is trying to impersonate principal: {}", context + .realPrincipal(), context.principal()); throw new WrappedAuthorizationException("Supervisor does not support impersonation"); } @@ -399,12 +410,15 @@ public void checkAuthorization(String topoName, Map topoConf, St IAuthorizer aclHandler = authorizationHandler; if (aclHandler != null) { if (!aclHandler.permit(context, operation, checkConf)) { - ThriftAccessLogger.logAccess(context.requestID(), context.remoteAddress(), context.principal(), + ThriftAccessLogger.logAccess(context.requestID(), context.remoteAddress(), context + .principal(), operation, topoName, "access-denied"); - throw new WrappedAuthorizationException(operation + (topoName != null ? " on topology " + topoName : "") + throw new WrappedAuthorizationException(operation + (topoName != null + ? " on topology " + topoName : "") + " is not authorized"); } else { - ThriftAccessLogger.logAccess(context.requestID(), context.remoteAddress(), context.principal(), + ThriftAccessLogger.logAccess(context.requestID(), context.remoteAddress(), context + .principal(), operation, topoName, "access-granted"); } } @@ -416,8 +430,10 @@ private org.apache.storm.generated.Supervisor.Iface createSupervisorIface() { public void sendSupervisorAssignments(SupervisorAssignments assignments) throws AuthorizationException, TException { checkAuthorization("sendSupervisorAssignments"); - LOG.info("Got an assignments from master, will start to sync with assignments: {}", assignments); - SynchronizeAssignments syn = new SynchronizeAssignments(getSupervisor(), assignments, + LOG.info("Got an assignments from master, will start to sync with assignments: {}", + assignments); + SynchronizeAssignments syn = new SynchronizeAssignments(getSupervisor(), + assignments, getReadClusterState()); getEventManger().add(syn); } @@ -452,7 +468,8 @@ public void sendSupervisorWorkerHeartbeat(SupervisorWorkerHeartbeat heartbeat) topoConf = ConfigUtils.readSupervisorStormConf(conf, id); } catch (IOException e) { LOG.warn("Topology config is not localized yet..."); - throw new WrappedNotAliveException(id + " does not appear to be alive, you should probably exit"); + throw new WrappedNotAliveException(id + + " does not appear to be alive, you should probably exit"); } checkAuthorization(id, topoConf, "sendSupervisorWorkerHeartbeat"); } @@ -470,13 +487,17 @@ private void launchSupervisorThriftServer(Map conf) throws IOExc ServerSocket socket = new ServerSocket(port); socket.close(); } catch (BindException e) { - LOG.error("{} is not available. Check if another process is already listening on {}", port, port); + LOG.error("{} is not available. Check if another process is already listening on {}", + port, port); throw new RuntimeException(e); } - TProcessor processor = new org.apache.storm.generated.Supervisor.Processor<>(supervisorThriftInterface); - boolean useTls = ObjectReader.getBoolean(conf.get(Config.SUPERVISOR_THRIFT_CLIENT_USE_TLS), false); - ThriftConnectionType type = useTls ? ThriftConnectionType.SUPERVISOR_TLS : ThriftConnectionType.SUPERVISOR; + TProcessor processor = + new org.apache.storm.generated.Supervisor.Processor<>(supervisorThriftInterface); + boolean useTls = ObjectReader.getBoolean(conf.get(Config.SUPERVISOR_THRIFT_CLIENT_USE_TLS), + false); + ThriftConnectionType type = useTls + ? ThriftConnectionType.SUPERVISOR_TLS : ThriftConnectionType.SUPERVISOR; String confPath = ThriftConnectionType.SUPERVISOR_TLS.getServerKeyStorePath(conf); this.multiThriftServer = new MultiThriftServer<>("supervisor-thrift-server"); @@ -490,7 +511,7 @@ private void launchSupervisorThriftServer(Map conf) throws IOExc * @param assignments {@link SupervisorAssignments} */ public void sendSupervisorAssignments(SupervisorAssignments assignments) { - //for local test + // for local test if (Time.isSimulating() && !(Boolean) conf.get(DaemonConfig.SUPERVISOR_ENABLE)) { return; } @@ -525,7 +546,8 @@ public void close() { } } - void killWorkers(Collection workerIds, ContainerLauncher launcher) throws InterruptedException, IOException { + void killWorkers(Collection workerIds, + ContainerLauncher launcher) throws InterruptedException, IOException { HashSet containers = new HashSet<>(); for (String workerId : workerIds) { try { @@ -540,7 +562,8 @@ void killWorkers(Collection workerIds, ContainerLauncher launcher) throw LOG.error("Error trying to kill {}", workerId, e); } } - int shutdownSleepSecs = ObjectReader.getInt(conf.get(Config.SUPERVISOR_WORKER_SHUTDOWN_SLEEP_SECS)); + int shutdownSleepSecs = ObjectReader.getInt(conf + .get(Config.SUPERVISOR_WORKER_SHUTDOWN_SLEEP_SECS)); if (!containers.isEmpty()) { Time.sleepSecs(shutdownSleepSecs); } @@ -553,7 +576,8 @@ void killWorkers(Collection workerIds, ContainerLauncher launcher) throw killErrorMeter.mark(); } throw new RuntimeException("Giving up on killing " + k - + " after " + (Time.currentTimeMillis() - start) + " ms"); + + " after " + (Time.currentTimeMillis() - start) + + " ms"); } k.forceKill(); Time.sleep(100); @@ -565,12 +589,14 @@ void killWorkers(Collection workerIds, ContainerLauncher launcher) throw } } - public void shutdownAllWorkers(BiConsumer onWarnTimeout, UniFunc onErrorTimeout) { + public void shutdownAllWorkers(BiConsumer onWarnTimeout, + UniFunc onErrorTimeout) { if (readState != null) { readState.shutdownAllWorkers(onWarnTimeout, onErrorTimeout); } else { try { - ContainerLauncher launcher = ContainerLauncher.make(getConf(), getId(), getThriftServerPort(), + ContainerLauncher launcher = ContainerLauncher.make(getConf(), getId(), + getThriftServerPort(), getSharedContext(), getMetricsRegistry(), getContainerMemoryTracker(), supervisorThriftInterface); killWorkers(SupervisorUtils.supervisorWorkerIds(conf), launcher); diff --git a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/SupervisorUtils.java b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/SupervisorUtils.java index 5948a5483c8..5ea85580fd8 100644 --- a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/SupervisorUtils.java +++ b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/SupervisorUtils.java @@ -29,7 +29,6 @@ import java.util.List; import java.util.Map; import java.util.Set; - import org.apache.storm.DaemonConfig; import org.apache.storm.generated.LSWorkerHeartbeat; import org.apache.storm.localizer.LocalResource; @@ -41,7 +40,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; - public class SupervisorUtils { private static final Logger LOG = LoggerFactory.getLogger(SupervisorUtils.class); @@ -59,6 +57,7 @@ public static void resetInstance() { /** * getNumaIdForPort for a specific supervisor. + * * @param port port * @param supervisorConf supervisorConf * @return getNumaIdForPort @@ -76,7 +75,8 @@ public static String getNumaIdForPort(Integer port, Map supervis } /** - * gets the set of all configured numa ports for a specific supervisor. + * Gets the set of all configured numa ports for a specific supervisor. + * * @param supervisorConf supervisorConf * @return set of all numa ports */ @@ -93,13 +93,15 @@ public static Set getNumaPorts(Map supervisorConf) { public static List getSlotsPorts(Map supervisorConf) { List slotsPorts = new ArrayList<>(); - List ports = (List) supervisorConf.getOrDefault(DaemonConfig.SUPERVISOR_SLOTS_PORTS, - new ArrayList<>()); + List ports = (List) supervisorConf + .getOrDefault(DaemonConfig.SUPERVISOR_SLOTS_PORTS, + new ArrayList<>()); for (Number port : ports) { slotsPorts.add(port.intValue()); } - // It's possible we have numaPorts specified that weren't configured in SUPERVISOR_SLOTS_PORTS. Make + // It's possible we have numaPorts specified that weren't configured in + // SUPERVISOR_SLOTS_PORTS. Make // sure we handle these ports as well. Set numaPorts = SupervisorUtils.getNumaPorts(supervisorConf); numaPorts.removeAll(slotsPorts); @@ -107,7 +109,8 @@ public static List getSlotsPorts(Map supervisorConf) { return slotsPorts; } - public static void rmrAsUser(Map conf, String id, String path) throws IOException { + public static void rmrAsUser(Map conf, String id, + String path) throws IOException { String user = ServerUtils.getFileOwner(path); String logPreFix = "rmr " + id; List commands = new ArrayList<>(); @@ -120,7 +123,8 @@ public static void rmrAsUser(Map conf, String id, String path) t } /** - * Given the blob information returns the value of the uncompress field, handling it being a boolean value, or if + * Given the blob information returns the value of the uncompress field, handling it being a + * boolean value, or if * it's not specified then returns false. */ public static boolean shouldUncompressBlob(Map blobInfo) { @@ -128,7 +132,8 @@ public static boolean shouldUncompressBlob(Map blobInfo) { } /** - * Given the blob information returns the value of the workerRestart field, handling it being a boolean value, or if + * Given the blob information returns the value of the workerRestart field, handling it being a + * boolean value, or if * it's not specified then returns false. * * @param blobInfo the info for the blob. @@ -141,13 +146,15 @@ public static boolean blobNeedsWorkerRestart(Map blobInfo) { /** * Returns a list of LocalResources based on the blobstore-map passed in. */ - public static List blobstoreMapToLocalresources(Map> blobstoreMap) { + public static List blobstoreMapToLocalresources(Map> blobstoreMap) { List localResourceList = new ArrayList<>(); if (blobstoreMap != null) { for (Map.Entry> map : blobstoreMap.entrySet()) { Map blobConf = map.getValue(); LocalResource localResource = - new LocalResource(map.getKey(), shouldUncompressBlob(blobConf), blobNeedsWorkerRestart(blobConf)); + new LocalResource(map.getKey(), shouldUncompressBlob(blobConf), + blobNeedsWorkerRestart(blobConf)); localResourceList.add(localResource); } } @@ -170,12 +177,14 @@ public static Map readWorkerHeartbeats(Map conf, String workerId) { + private static LSWorkerHeartbeat readWorkerHeartbeat(Map conf, + String workerId) { return _instance.readWorkerHeartbeatImpl(conf, workerId); } /** * Return supervisor numa configuration. + * * @param stormConf stormConf * @return getNumaMap */ @@ -205,7 +214,8 @@ protected LSWorkerHeartbeat readWorkerHeartbeatImpl(Map conf, St LocalState localState = ConfigUtils.workerState(conf, workerId); return localState.getWorkerHeartBeat(); } catch (Exception e) { - LOG.warn("Failed to read local heartbeat for workerId : {},Ignoring exception.", workerId, e); + LOG.warn("Failed to read local heartbeat for workerId : {},Ignoring exception.", + workerId, e); return null; } } diff --git a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/TimerDecoratedAssignment.java b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/TimerDecoratedAssignment.java index ad2e6bb6846..a1621406b13 100644 --- a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/TimerDecoratedAssignment.java +++ b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/TimerDecoratedAssignment.java @@ -1,21 +1,24 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. - * See the NOTICE file distributed with this work for additional information regarding copyright ownership. + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. + * See the NOTICE file distributed with this work for additional information regarding copyright + * ownership. * The ASF licenses this file to you under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. You may obtain a copy of the License at + * you may not use this file except in compliance with the License. You may obtain a copy of the + * License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, + *

      Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and limitations under the License. + * See the License for the specific language governing permissions and limitations under the + * License. */ package org.apache.storm.daemon.supervisor; import com.codahale.metrics.Timer; - import org.apache.storm.generated.LocalAssignment; import org.apache.storm.metric.timed.TimerDecorated; diff --git a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/UniFunc.java b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/UniFunc.java index b0dfd946974..67efb98dabc 100644 --- a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/UniFunc.java +++ b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/UniFunc.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/timer/ReportWorkerHeartbeats.java b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/timer/ReportWorkerHeartbeats.java index c9582c20c1a..1b577d386a2 100644 --- a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/timer/ReportWorkerHeartbeats.java +++ b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/timer/ReportWorkerHeartbeats.java @@ -1,12 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

      * http://www.apache.org/licenses/LICENSE-2.0 *

      - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -32,8 +37,10 @@ import org.slf4j.LoggerFactory; /** - * Runnable reporting local worker reported heartbeats to master, supervisor should take care the of the heartbeats - * integrity for the master heartbeats recovery, a non-null node id means that the heartbeats are full, + * Runnable reporting local worker reported heartbeats to master, supervisor should take care the of + * the heartbeats + * integrity for the master heartbeats recovery, a non-null node id means that the heartbeats are + * full, * and master can go on to check and wait others nodes when doing a heartbeats recovery. */ public class ReportWorkerHeartbeats implements Runnable { @@ -47,7 +54,8 @@ public class ReportWorkerHeartbeats implements Runnable { public ReportWorkerHeartbeats(Map conf, Supervisor supervisor) { this.conf = conf; this.supervisor = supervisor; - this.workerTimeoutSecs = ObjectReader.getInt(conf.get(Config.SUPERVISOR_WORKER_TIMEOUT_SECS)); + this.workerTimeoutSecs = ObjectReader.getInt(conf + .get(Config.SUPERVISOR_WORKER_TIMEOUT_SECS)); this.workerMaxTimeoutSecs = ObjectReader.getInt(conf.get(Config.WORKER_MAX_TIMEOUT_SECS)); } @@ -63,13 +71,15 @@ private SupervisorWorkerHeartbeats getAndResetWorkerHeartbeats() { localHeartbeats = SupervisorUtils.readWorkerHeartbeats(this.conf); return getSupervisorWorkerHeartbeatsFromLocal(localHeartbeats); } catch (Exception e) { - LOG.error("Read local worker heartbeats error, skipping heartbeats for this round, msg:{}", e.getMessage()); + LOG.error("Read local worker heartbeats error, skipping heartbeats for this round, " + + "msg:{}", e.getMessage()); return null; } } @VisibleForTesting - SupervisorWorkerHeartbeats getSupervisorWorkerHeartbeatsFromLocal(Map localHeartbeats) { + SupervisorWorkerHeartbeats getSupervisorWorkerHeartbeatsFromLocal(Map localHeartbeats) { SupervisorWorkerHeartbeats supervisorWorkerHeartbeats = new SupervisorWorkerHeartbeats(); List heartbeatList = new ArrayList<>(); @@ -90,7 +100,8 @@ SupervisorWorkerHeartbeats getSupervisorWorkerHeartbeatsFromLocal(Map effectiveTimeoutSecs) { - LOG.debug("Skipping stale heartbeat for topology {}: age {}s > effective worker timeout {}s", + LOG.debug("Skipping stale heartbeat for topology {}: age {}s > effective worker " + + "timeout {}s", topologyId, hbAgeSecs, effectiveTimeoutSecs); continue; } @@ -121,14 +133,20 @@ SupervisorWorkerHeartbeats getSupervisorWorkerHeartbeatsFromLocal(MapNimbus clamps {@link Config#TOPOLOGY_WORKER_TIMEOUT_SECS} to {@link Config#WORKER_MAX_TIMEOUT_SECS} - * at submission time and persists the clamped value, so the on-disk conf both this method and Slot read is - * already bounded; for that value the two computations match. Unlike Slot, this method re-applies the cap - * to the topology override defensively, guarding against an un-clamped conf. The cap is applied to the - * override component only (not the final result), so the global timeout is never shrunk below the value + *

      Nimbus clamps {@link Config#TOPOLOGY_WORKER_TIMEOUT_SECS} to {@link + * Config#WORKER_MAX_TIMEOUT_SECS} + * at submission time and persists the clamped value, so the on-disk conf both this method and + * Slot read is + * already bounded; for that value the two computations match. Unlike Slot, this method + * re-applies the cap + * to the topology override defensively, guarding against an un-clamped conf. The cap is applied + * to the + * override component only (not the final result), so the global timeout is never shrunk below + * the value * Slot would use. * - *

      Orphaned worker directories often outlive their topology conf; when the conf cannot be read we fall + *

      Orphaned worker directories often outlive their topology conf; when the conf cannot be + * read we fall * back to the global timeout, which is exactly the behavior wanted for an already-dead worker. */ @VisibleForTesting @@ -137,7 +155,8 @@ int effectiveWorkerTimeoutSecs(String topologyId) { try { topoConf = ConfigUtils.readSupervisorStormConf(conf, topologyId); } catch (Exception e) { - LOG.debug("Cannot read topology conf for {}; using supervisor worker timeout {}s. msg: {}", + LOG.debug("Cannot read topology conf for {}; using supervisor worker timeout {}s. " + + "msg: {}", topologyId, workerTimeoutSecs, e.getMessage()); return workerTimeoutSecs; } @@ -145,7 +164,8 @@ int effectiveWorkerTimeoutSecs(String topologyId) { if (topoTimeout == null) { return workerTimeoutSecs; } - int cappedTopoTimeoutSecs = Math.min(ObjectReader.getInt(topoTimeout), workerMaxTimeoutSecs); + int cappedTopoTimeoutSecs = Math.min(ObjectReader.getInt(topoTimeout), + workerMaxTimeoutSecs); return Math.max(workerTimeoutSecs, cappedTopoTimeoutSecs); } @@ -155,13 +175,15 @@ private void reportWorkerHeartbeats(SupervisorWorkerHeartbeats supervisorWorkerH return; } if (supervisor.getStormClusterState().isPacemakerStateStore()) { - LOG.debug("Worker are using pacemaker to send worker heartbeats so skip reporting by supervisor."); + LOG.debug("Worker are using pacemaker to send worker heartbeats so skip reporting by " + + "supervisor."); return; } // if it is local mode, just get the local nimbus instance and set the heartbeats if (ConfigUtils.isLocalMode(conf)) { try { - this.supervisor.getLocalNimbus().sendSupervisorWorkerHeartbeats(supervisorWorkerHeartbeats); + this.supervisor.getLocalNimbus() + .sendSupervisorWorkerHeartbeats(supervisorWorkerHeartbeats); } catch (TException tex) { LOG.error("Send local supervisor heartbeats error", tex); } diff --git a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/timer/SupervisorHeartbeat.java b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/timer/SupervisorHeartbeat.java index b613f36ee9a..3a9dc4cdd74 100644 --- a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/timer/SupervisorHeartbeat.java +++ b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/timer/SupervisorHeartbeat.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -18,7 +24,6 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; - import org.apache.storm.Config; import org.apache.storm.DaemonConfig; import org.apache.storm.ServerConstants; @@ -43,7 +48,6 @@ public class SupervisorHeartbeat implements Runnable { public static final Logger LOG = LoggerFactory.getLogger(SupervisorHeartbeat.class); - public SupervisorHeartbeat(Map conf, Supervisor supervisor) { this.stormClusterState = supervisor.getStormClusterState(); this.supervisorId = supervisor.getId(); @@ -51,7 +55,8 @@ public SupervisorHeartbeat(Map conf, Supervisor supervisor) { this.conf = conf; } - private Map buildSupervisorInfo(Map conf, Supervisor supervisor, + private Map buildSupervisorInfo(Map conf, + Supervisor supervisor, Map validatedNumaMap) { List metaDatas = (List) supervisor.getiSupervisor().getMetadata(); List allPortList = new ArrayList<>(); @@ -67,7 +72,8 @@ private Map buildSupervisorInfo(Map conf List allUsedPorts = new ArrayList<>(); allUsedPorts.addAll(supervisor.getCurrAssignment().get().keySet()); Map totalSupervisorResources = mkSupervisorCapacities(conf); - NormalizedResourceOffer totalSupervisorNormalizedResources = new NormalizedResourceOffer(totalSupervisorResources); + NormalizedResourceOffer totalSupervisorNormalizedResources = + new NormalizedResourceOffer(totalSupervisorResources); Map result = new HashMap(); @@ -77,12 +83,14 @@ private Map buildSupervisorInfo(Map conf supervisorInfo.set_time_secs(Time.currentTimeSecsLong()); supervisorInfo.set_hostname(supervisor.getHostName()); supervisorInfo.set_assignment_id( - supervisor.getAssignmentId() + ServerConstants.NUMA_ID_SEPARATOR + numaMapEntry.getKey() + supervisor.getAssignmentId() + ServerConstants.NUMA_ID_SEPARATOR + + numaMapEntry.getKey() ); supervisorInfo.set_server_port(supervisor.getThriftServerPort()); Map numaMap = (Map) numaMapEntry.getValue(); - List numaPortList = ((List) numaMap.get(ServerConstants.NUMA_PORTS)).stream() + List numaPortList = ((List) numaMap.get(ServerConstants.NUMA_PORTS)) + .stream() .map(e -> e.longValue()).collect(Collectors.toList()); List usedNumaPorts = ListUtils.intersection(numaPortList, allUsedPorts); @@ -95,16 +103,20 @@ private Map buildSupervisorInfo(Map conf ); supervisorInfo.set_uptime_secs(supervisor.getUpTime().upTime()); supervisorInfo.set_version(supervisor.getStormVersion()); - Map supervisorCapacitiesFromNumaMap = mkSupervisorCapacitiesFromNumaMap(numaMap); - NormalizedResourceOffer numaNormalizedResources = new NormalizedResourceOffer(supervisorCapacitiesFromNumaMap); + Map supervisorCapacitiesFromNumaMap = + mkSupervisorCapacitiesFromNumaMap(numaMap); + NormalizedResourceOffer numaNormalizedResources = + new NormalizedResourceOffer(supervisorCapacitiesFromNumaMap); totalSupervisorNormalizedResources.remove(numaNormalizedResources); supervisorInfo.set_resources_map(supervisorCapacitiesFromNumaMap); - result.put(supervisor.getId() + ServerConstants.NUMA_ID_SEPARATOR + numaMapEntry.getKey(), supervisorInfo); + result.put(supervisor.getId() + ServerConstants.NUMA_ID_SEPARATOR + numaMapEntry + .getKey(), supervisorInfo); } } if (totalSupervisorNormalizedResources.getTotalCpu() > 0 - && totalSupervisorNormalizedResources.getTotalMemoryMb() > 0 && !allPortList.isEmpty()) { + && totalSupervisorNormalizedResources.getTotalMemoryMb() > 0 && !allPortList + .isEmpty()) { SupervisorInfo supervisorInfo = new SupervisorInfo(); supervisorInfo.set_time_secs(Time.currentTimeSecsLong()); supervisorInfo.set_hostname(supervisor.getHostName()); @@ -112,7 +124,8 @@ private Map buildSupervisorInfo(Map conf supervisorInfo.set_server_port(supervisor.getThriftServerPort()); supervisorInfo.set_used_ports(allUsedPorts); supervisorInfo.set_meta(allPortList); - supervisorInfo.set_scheduler_meta((Map) conf.get(DaemonConfig.SUPERVISOR_SCHEDULER_META)); + supervisorInfo.set_scheduler_meta((Map) conf + .get(DaemonConfig.SUPERVISOR_SCHEDULER_META)); supervisorInfo.set_uptime_secs(supervisor.getUpTime().upTime()); supervisorInfo.set_version(supervisor.getStormVersion()); supervisorInfo.set_resources_map(totalSupervisorNormalizedResources.toNormalizedMap()); @@ -131,7 +144,8 @@ private Map mkSupervisorCapacitiesFromNumaMap(Map) numaMap.getOrDefault(ServerConstants.NUMA_GENERIC_RESOURCES_MAP, Collections.emptyMap())); + ret.putAll((Map) numaMap + .getOrDefault(ServerConstants.NUMA_GENERIC_RESOURCES_MAP, Collections.emptyMap())); return NormalizedResources.RESOURCE_NAME_NORMALIZER.normalizedResourceMap(ret); } @@ -153,16 +167,20 @@ private Map mkSupervisorCapacities(Map conf) { ret.put(stringNumberEntry.getKey(), stringNumberEntry.getValue().doubleValue()); } - LOG.debug(NormalizedResources.RESOURCE_NAME_NORMALIZER.normalizedResourceMap(ret).toString()); + LOG.debug(NormalizedResources.RESOURCE_NAME_NORMALIZER.normalizedResourceMap(ret) + .toString()); return NormalizedResources.RESOURCE_NAME_NORMALIZER.normalizedResourceMap(ret); } @Override public void run() { Map validatedNumaMap = SupervisorUtils.getNumaMap(conf); - Map supervisorInfoList = buildSupervisorInfo(conf, supervisor, validatedNumaMap); - for (Map.Entry supervisorInfoEntry : supervisorInfoList.entrySet()) { - stormClusterState.supervisorHeartbeat(supervisorInfoEntry.getKey(), supervisorInfoEntry.getValue()); + Map supervisorInfoList = buildSupervisorInfo(conf, supervisor, + validatedNumaMap); + for (Map.Entry supervisorInfoEntry : supervisorInfoList + .entrySet()) { + stormClusterState.supervisorHeartbeat(supervisorInfoEntry.getKey(), supervisorInfoEntry + .getValue()); } } } diff --git a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/timer/SynchronizeAssignments.java b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/timer/SynchronizeAssignments.java index da7e676aa04..970d49dba89 100644 --- a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/timer/SynchronizeAssignments.java +++ b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/timer/SynchronizeAssignments.java @@ -1,12 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

      * http://www.apache.org/licenses/LICENSE-2.0 *

      - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -17,7 +22,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; - import org.apache.storm.ServerConstants; import org.apache.storm.cluster.IStormClusterState; import org.apache.storm.daemon.supervisor.ReadClusterState; @@ -46,11 +50,13 @@ public class SynchronizeAssignments implements Runnable { /** * Constructor. + * * @param supervisor {@link Supervisor} * @param assignments {@link SupervisorAssignments} * @param readClusterState {@link ReadClusterState} */ - public SynchronizeAssignments(Supervisor supervisor, SupervisorAssignments assignments, ReadClusterState readClusterState) { + public SynchronizeAssignments(Supervisor supervisor, SupervisorAssignments assignments, + ReadClusterState readClusterState) { this.supervisor = supervisor; this.assignments = assignments; this.readClusterState = readClusterState; @@ -59,16 +65,17 @@ public SynchronizeAssignments(Supervisor supervisor, SupervisorAssignments assig private static void assignedAssignmentsToLocal(IStormClusterState clusterState, List supervisorAssignments) { if (null == supervisorAssignments || supervisorAssignments.isEmpty()) { - //unknown error, just skip + // unknown error, just skip return; } Map serAssignments = new HashMap<>(); for (SupervisorAssignments supervisorAssignment : supervisorAssignments) { if (supervisorAssignment == null) { - //unknown error, just skip + // unknown error, just skip continue; } - for (Map.Entry entry : supervisorAssignment.get_storm_assignment().entrySet()) { + for (Map.Entry entry : supervisorAssignment.get_storm_assignment() + .entrySet()) { serAssignments.put(entry.getKey(), Utils.serialize(entry.getValue())); } } @@ -79,9 +86,11 @@ private static void assignedAssignmentsToLocal(IStormClusterState clusterState, public void run() { // first sync assignments to local, then sync processes. if (null == assignments) { - getAssignmentsFromMaster(this.supervisor.getConf(), this.supervisor.getStormClusterState(), this.supervisor.getAssignmentId()); + getAssignmentsFromMaster(this.supervisor.getConf(), this.supervisor + .getStormClusterState(), this.supervisor.getAssignmentId()); } else { - assignedAssignmentsToLocal(this.supervisor.getStormClusterState(), Collections.singletonList(assignments)); + assignedAssignmentsToLocal(this.supervisor.getStormClusterState(), Collections + .singletonList(assignments)); } this.readClusterState.run(); } @@ -106,6 +115,7 @@ public List getAllAssignmentsFromNumaSupervisors( /** * Used by {@link Supervisor} to fetch assignments when start up. + * * @param conf config * @param clusterState {@link IStormClusterState} * @param node id of node @@ -123,8 +133,10 @@ public void getAssignmentsFromMaster(Map conf, IStormClusterState clusterState, } } else { try (NimbusClient master = NimbusClient.Builder.withConf(conf).forDaemon().build()) { - List supervisorAssignmentsList = getAllAssignmentsFromNumaSupervisors(master.getClient(), node); - LOG.debug("Sync an assignments from master, will start to sync with assignments: {}", supervisorAssignmentsList); + List supervisorAssignmentsList = + getAllAssignmentsFromNumaSupervisors(master.getClient(), node); + LOG.debug("Sync an assignments from master, will start to sync with assignments: " + + "{}", supervisorAssignmentsList); assignedAssignmentsToLocal(clusterState, supervisorAssignmentsList); } catch (Exception t) { LOG.error("Get assignments from master exception", t); diff --git a/storm-server/src/main/java/org/apache/storm/event/EventManager.java b/storm-server/src/main/java/org/apache/storm/event/EventManager.java index ff7342b6770..068deacb8bc 100644 --- a/storm-server/src/main/java/org/apache/storm/event/EventManager.java +++ b/storm-server/src/main/java/org/apache/storm/event/EventManager.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-server/src/main/java/org/apache/storm/event/EventManagerImp.java b/storm-server/src/main/java/org/apache/storm/event/EventManagerImp.java index a5273153ba3..7518414975a 100644 --- a/storm-server/src/main/java/org/apache/storm/event/EventManagerImp.java +++ b/storm-server/src/main/java/org/apache/storm/event/EventManagerImp.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -51,9 +57,11 @@ public void run() { } catch (Throwable t) { if (Utils.exceptionCauseIsInstanceOf(InterruptedIOException.class, t)) { LOG.info("Event manager interrupted while doing IO"); - } else if (Utils.exceptionCauseIsInstanceOf(ClosedByInterruptException.class, t)) { + } else if (Utils + .exceptionCauseIsInstanceOf(ClosedByInterruptException.class, t)) { LOG.info("Event manager interrupted while doing NIO"); - } else if (Utils.exceptionCauseIsInstanceOf(InterruptedException.class, t)) { + } else if (Utils.exceptionCauseIsInstanceOf(InterruptedException.class, + t)) { LOG.info("Event manager interrupted"); } else { LOG.error("{} Error when processing event", t); diff --git a/storm-server/src/main/java/org/apache/storm/healthcheck/HealthChecker.java b/storm-server/src/main/java/org/apache/storm/healthcheck/HealthChecker.java index ab26881bcb8..55762d0a5e1 100644 --- a/storm-server/src/main/java/org/apache/storm/healthcheck/HealthChecker.java +++ b/storm-server/src/main/java/org/apache/storm/healthcheck/HealthChecker.java @@ -19,7 +19,6 @@ package org.apache.storm.healthcheck; import com.codahale.metrics.Meter; - import java.io.BufferedReader; import java.io.File; import java.io.IOException; @@ -29,7 +28,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; - import org.apache.storm.Constants; import org.apache.storm.DaemonConfig; import org.apache.storm.metric.StormMetricsRegistry; @@ -78,12 +76,14 @@ public static int healthCheck(Map conf, StormMetricsRegistry met } else if (results.contains(TIMEOUT)) { LOG.warn("The supervisor healthchecks timedout!!!"); if (metricRegistry != null) { - Meter timeoutMeter = metricRegistry.getMeter(Constants.SUPERVISOR_HEALTH_CHECK_TIMEOUTS); + Meter timeoutMeter = metricRegistry + .getMeter(Constants.SUPERVISOR_HEALTH_CHECK_TIMEOUTS); if (timeoutMeter != null) { timeoutMeter.mark(); } } - Boolean failOnTimeouts = ObjectReader.getBoolean(conf.get(DaemonConfig.STORM_HEALTH_CHECK_FAIL_ON_TIMEOUTS), true); + Boolean failOnTimeouts = ObjectReader.getBoolean(conf + .get(DaemonConfig.STORM_HEALTH_CHECK_FAIL_ON_TIMEOUTS), true); if (failOnTimeouts) { return 1; } else { @@ -101,7 +101,8 @@ public static String processScript(Map conf, String script) { Process process = null; try { process = Runtime.getRuntime().exec(script); - final long timeout = ObjectReader.getLong(conf.get(DaemonConfig.STORM_HEALTH_CHECK_TIMEOUT_MS), 5000L); + final long timeout = ObjectReader.getLong(conf + .get(DaemonConfig.STORM_HEALTH_CHECK_TIMEOUT_MS), 5000L); final Thread curThread = Thread.currentThread(); // kill process when timeout interruptThread = new Thread(new Runnable() { @@ -127,8 +128,8 @@ public void run() { LOG.warn("The healthcheck process {} exited with code: {}; output: {}; err: {}.", script, process.exitValue(), outMessage, errMessage); - //Keep this for backwards compatibility. - //It relies on "ERROR" at the beginning of stdout to determine FAILED status + // Keep this for backwards compatibility. + // It relies on "ERROR" at the beginning of stdout to determine FAILED status if (outMessage.startsWith("ERROR")) { return FAILED; } diff --git a/storm-server/src/main/java/org/apache/storm/localizer/AsyncLocalizer.java b/storm-server/src/main/java/org/apache/storm/localizer/AsyncLocalizer.java index c06e38f3dc8..f753c0175fe 100644 --- a/storm-server/src/main/java/org/apache/storm/localizer/AsyncLocalizer.java +++ b/storm-server/src/main/java/org/apache/storm/localizer/AsyncLocalizer.java @@ -72,7 +72,8 @@ public class AsyncLocalizer implements AutoCloseable { private static final Logger LOG = LoggerFactory.getLogger(AsyncLocalizer.class); - private static final CompletableFuture ALL_DONE_FUTURE = CompletableFuture.completedFuture(null); + private static final CompletableFuture ALL_DONE_FUTURE = CompletableFuture + .completedFuture(null); private static final int ATTEMPTS_INTERVAL_TIME = 100; private final Timer blobCacheUpdateDuration; @@ -81,8 +82,10 @@ public class AsyncLocalizer implements AutoCloseable { private final Meter updateBlobExceptions; // track resources - user to resourceSet - //ConcurrentHashMap is explicitly used everywhere in this class because it uses locks to guarantee atomicity for compute and - // computeIfAbsent where as ConcurrentMap allows for a retry of the function passed in, and would require the function to have + // ConcurrentHashMap is explicitly used everywhere in this class because it uses locks to + // guarantee atomicity for compute and + // computeIfAbsent where as ConcurrentMap allows for a retry of the function passed in, and + // would require the function to have // no side effects. protected final ConcurrentHashMap> userFiles = new ConcurrentHashMap<>(); protected final ConcurrentHashMap> userArchives = new ConcurrentHashMap<>(); @@ -92,8 +95,10 @@ public class AsyncLocalizer implements AutoCloseable { private final Map conf; private final AdvancedFSOps fsOps; private final boolean symlinksDisabled; - private final ConcurrentHashMap topologyBlobs = new ConcurrentHashMap<>(); - private final ConcurrentHashMap> topologyBasicDownloaded = new ConcurrentHashMap<>(); + private final ConcurrentHashMap topologyBlobs = + new ConcurrentHashMap<>(); + private final ConcurrentHashMap> topologyBasicDownloaded = + new ConcurrentHashMap<>(); private final Path localBaseDir; private final int blobDownloadRetries; private final ScheduledExecutorService downloadExecService; @@ -106,19 +111,25 @@ public class AsyncLocalizer implements AutoCloseable { protected long cacheTargetSize; @VisibleForTesting - AsyncLocalizer(Map conf, AdvancedFSOps ops, String baseDir, StormMetricsRegistry metricsRegistry) throws IOException { + AsyncLocalizer(Map conf, AdvancedFSOps ops, String baseDir, + StormMetricsRegistry metricsRegistry) throws IOException { this.conf = conf; - this.blobCacheUpdateDuration = metricsRegistry.registerTimer("supervisor:blob-cache-update-duration"); - this.blobLocalizationDuration = metricsRegistry.registerTimer("supervisor:blob-localization-duration"); + this.blobCacheUpdateDuration = metricsRegistry + .registerTimer("supervisor:blob-cache-update-duration"); + this.blobLocalizationDuration = metricsRegistry + .registerTimer("supervisor:blob-localization-duration"); this.localResourceFileNotFoundWhenReleasingSlot - = metricsRegistry.registerMeter("supervisor:local-resource-file-not-found-when-releasing-slot"); - this.updateBlobExceptions = metricsRegistry.registerMeter("supervisor:update-blob-exceptions"); + = metricsRegistry + .registerMeter("supervisor:local-resource-file-not-found-when-releasing-slot"); + this.updateBlobExceptions = metricsRegistry + .registerMeter("supervisor:update-blob-exceptions"); this.metricsRegistry = metricsRegistry; isLocalMode = ConfigUtils.isLocalMode(conf); fsOps = ops; localBaseDir = Paths.get(baseDir); // default cache size 10GB, converted to Bytes - cacheTargetSize = ObjectReader.getInt(conf.get(DaemonConfig.SUPERVISOR_LOCALIZER_CACHE_TARGET_SIZE_MB), + cacheTargetSize = ObjectReader.getInt(conf + .get(DaemonConfig.SUPERVISOR_LOCALIZER_CACHE_TARGET_SIZE_MB), 10 * 1024).longValue() << 20; // default 30 seconds. (we cache the size so it is cheap to do) cacheCleanupPeriod = ObjectReader.getInt(conf.get( @@ -129,18 +140,22 @@ public class AsyncLocalizer implements AutoCloseable { blobDownloadRetries = ObjectReader.getInt(conf.get( DaemonConfig.SUPERVISOR_BLOBSTORE_DOWNLOAD_MAX_RETRIES), 3); - int downloadThreadPoolSize = ObjectReader.getInt(conf.get(DaemonConfig.SUPERVISOR_BLOBSTORE_DOWNLOAD_THREAD_COUNT), 5); + int downloadThreadPoolSize = ObjectReader.getInt(conf + .get(DaemonConfig.SUPERVISOR_BLOBSTORE_DOWNLOAD_THREAD_COUNT), 5); downloadExecService = Executors.newScheduledThreadPool(downloadThreadPoolSize, - new ThreadFactoryBuilder().setNameFormat("AsyncLocalizer Download Executor - %d").build()); + new ThreadFactoryBuilder().setNameFormat("AsyncLocalizer Download Executor - %d") + .build()); taskExecService = Executors.newScheduledThreadPool(3, - new ThreadFactoryBuilder().setNameFormat("AsyncLocalizer Task Executor - %d").build()); + new ThreadFactoryBuilder().setNameFormat("AsyncLocalizer Task Executor - %d") + .build()); reconstructLocalizedResources(); symlinksDisabled = (boolean) conf.getOrDefault(Config.DISABLE_SYMLINKS, false); blobPending = new ConcurrentHashMap<>(); } - public AsyncLocalizer(Map conf, StormMetricsRegistry metricsRegistry) throws IOException { + public AsyncLocalizer(Map conf, + StormMetricsRegistry metricsRegistry) throws IOException { this(conf, AdvancedFSOps.make(conf), ConfigUtils.supervisorLocalDir(conf), metricsRegistry); } @@ -196,34 +211,42 @@ private LocalizedResource getUserArchive(String user, String key) { if (user == null) { throw new AssertionError("All user archives require a user present"); } - ConcurrentMap keyToResource = userArchives.computeIfAbsent(user, (u) -> new ConcurrentHashMap<>()); + ConcurrentMap keyToResource = userArchives.computeIfAbsent(user, + (u) -> new ConcurrentHashMap<>()); return keyToResource.computeIfAbsent(key, - (k) -> new LocalizedResource(key, localBaseDir, true, fsOps, conf, user, metricsRegistry)); + (k) -> new LocalizedResource(key, localBaseDir, true, fsOps, conf, user, + metricsRegistry)); } private LocalizedResource getUserFile(String user, String key) { if (user == null) { throw new AssertionError("All user archives require a user present"); } - ConcurrentMap keyToResource = userFiles.computeIfAbsent(user, (u) -> new ConcurrentHashMap<>()); + ConcurrentMap keyToResource = userFiles.computeIfAbsent(user, + (u) -> new ConcurrentHashMap<>()); return keyToResource.computeIfAbsent(key, - (k) -> new LocalizedResource(key, localBaseDir, false, fsOps, conf, user, metricsRegistry)); + (k) -> new LocalizedResource(key, localBaseDir, false, fsOps, conf, user, + metricsRegistry)); } /** - * Request that all of the blobs necessary for this topology be downloaded. Note that this adds references to + * Request that all of the blobs necessary for this topology be downloaded. Note that this adds + * references to * blobs asynchronously in background threads. * * @param assignment the assignment that needs the blobs * @param port the port the assignment is a part of - * @param cb a callback for when the blobs change. This is only for blobs that are tied to the lifetime of the worker. + * @param cb a callback for when the blobs change. This is only for blobs that are tied to the + * lifetime of the worker. * @return a Future that indicates when they are all downloaded. * * @throws IOException if there was an error while trying doing it. */ - public CompletableFuture requestDownloadTopologyBlobs(final LocalAssignment assignment, final int port, + public CompletableFuture requestDownloadTopologyBlobs(final LocalAssignment assignment, + final int port, final BlobChangingCallback cb) throws IOException { - final PortAndAssignment pna = new TimePortAndAssignment(new PortAndAssignmentImpl(port, assignment), blobLocalizationDuration); + final PortAndAssignment pna = new TimePortAndAssignment(new PortAndAssignmentImpl(port, + assignment), blobLocalizationDuration); final String topologyId = pna.getToplogyId(); LOG.info("requestDownloadTopologyBlobs for {}", pna); @@ -233,7 +256,8 @@ public CompletableFuture requestDownloadTopologyBlobs(final LocalAssignmen blobPending.compute(topologyId, (tid, old) -> { CompletableFuture ret = old; if (ret == null) { - ret = CompletableFuture.supplyAsync(new DownloadBlobs(pna, cb), taskExecService); + ret = CompletableFuture.supplyAsync(new DownloadBlobs(pna, cb), + taskExecService); } else { try { addReferencesToBlobs(pna, cb); @@ -250,7 +274,8 @@ public CompletableFuture requestDownloadTopologyBlobs(final LocalAssignmen } @VisibleForTesting - CompletableFuture requestDownloadBaseTopologyBlobs(PortAndAssignment pna, BlobChangingCallback cb) { + CompletableFuture requestDownloadBaseTopologyBlobs(PortAndAssignment pna, + BlobChangingCallback cb) { final String topologyId = pna.getToplogyId(); final LocallyCachedBlob topoJar = getTopoJar(topologyId, pna.getAssignment().get_owner()); @@ -291,7 +316,8 @@ private CompletableFuture downloadOrUpdate(Collection blobDownloadRetries) { throw new RuntimeException("Could not download...", e); } - LOG.warn("Failed to download blob {} will try again in {} ms", blob, ATTEMPTS_INTERVAL_TIME, e); + LOG.warn("Failed to download blob {} will try again in {} ms", blob, + ATTEMPTS_INTERVAL_TIME, e); Utils.sleep(ATTEMPTS_INTERVAL_TIME); } } @@ -315,7 +341,8 @@ private long getRemoteBlobstoreUpdateTime() { } /** - * Downloads all blobs listed in the topology configuration for all topologies assigned to this supervisor, and creates version files + * Downloads all blobs listed in the topology configuration for all topologies assigned to this + * supervisor, and creates version files * with a suffix. The runnable is intended to be run periodically by a timer, created elsewhere. */ @VisibleForTesting @@ -344,13 +371,16 @@ void updateBlobs() { } /** - * Start any background threads needed. This includes updating blobs and cleaning up unused blobs over the configured size limit. + * Start any background threads needed. This includes updating blobs and cleaning up unused + * blobs over the configured size limit. */ public void start() { LOG.debug("Scheduling updateBlobs every {} seconds", updateBlobPeriod); - taskExecService.scheduleWithFixedDelay(this::updateBlobs, updateBlobPeriod, updateBlobPeriod, TimeUnit.SECONDS); + taskExecService.scheduleWithFixedDelay(this::updateBlobs, updateBlobPeriod, + updateBlobPeriod, TimeUnit.SECONDS); LOG.debug("Scheduling cleanup every {} millis", cacheCleanupPeriod); - taskExecService.scheduleAtFixedRate(this::cleanup, cacheCleanupPeriod, cacheCleanupPeriod, TimeUnit.MILLISECONDS); + taskExecService.scheduleAtFixedRate(this::cleanup, cacheCleanupPeriod, cacheCleanupPeriod, + TimeUnit.MILLISECONDS); } @Override @@ -364,7 +394,8 @@ private List getLocalResources(PortAndAssignment pna) throws IOEx Map topoConf = ConfigUtils.readSupervisorStormConf(conf, topologyId); @SuppressWarnings("unchecked") - Map> blobstoreMap = (Map>) topoConf.get(Config.TOPOLOGY_BLOBSTORE_MAP); + Map> blobstoreMap = (Map>) topoConf + .get(Config.TOPOLOGY_BLOBSTORE_MAP); List ret = new ArrayList<>(); if (blobstoreMap != null) { @@ -402,7 +433,8 @@ void addReferencesToBlobs(PortAndAssignment pna, BlobChangingCallback cb) * * @param currentAssignment the assignment for the topology. * @param port the port the assignment is on. - * @param cb a callback for when the blobs are updated. This will only be for blobs that indicate that if they change + * @param cb a callback for when the blobs are updated. This will only be for blobs that + * indicate that if they change * the worker should be restarted. * @throws IOException on any error trying to recover the state. */ @@ -421,7 +453,8 @@ public void recoverRunningTopology(final LocalAssignment currentAssignment, fina LocallyCachedBlob topoConf = getTopoConf(topologyId, pna.getAssignment().get_owner()); topoConf.addReference(pna, cb); - CompletableFuture localResource = blobPending.computeIfAbsent(topologyId, (tid) -> ALL_DONE_FUTURE); + CompletableFuture localResource = blobPending.computeIfAbsent(topologyId, + (tid) -> ALL_DONE_FUTURE); try { addReferencesToBlobs(pna, cb); @@ -533,7 +566,8 @@ private void reconstructLocalizedResources() { void removeBlobReference(String key, PortAndAssignment pna, boolean uncompress) { String user = pna.getOwner(); String topo = pna.getToplogyId(); - ConcurrentMap lrsrcSet = uncompress ? userArchives.get(user) : userFiles.get(user); + ConcurrentMap lrsrcSet = uncompress ? userArchives + .get(user) : userFiles.get(user); if (lrsrcSet != null) { LocalizedResource lrsrc = lrsrcSet.get(key); if (lrsrc != null) { @@ -544,7 +578,8 @@ void removeBlobReference(String key, PortAndAssignment pna, boolean uncompress) + " topo: " + topo); } } else { - LOG.warn("trying to remove blob for non-existent resource set for user: " + user + " key: " + LOG.warn("trying to remove blob for non-existent resource set for user: " + user + + " key: " + key + " topo: " + topo); } } @@ -554,13 +589,17 @@ protected ClientBlobStore getClientBlobStore() { } /** - * This function either returns the blobs in the existing cache or if they don't exist in the cache, it downloads them in parallel (up - * to SUPERVISOR_BLOBSTORE_DOWNLOAD_THREAD_COUNT) and will block until all of them have been downloaded. + * This function either returns the blobs in the existing cache or if they don't exist in the + * cache, it downloads them in parallel (up + * to SUPERVISOR_BLOBSTORE_DOWNLOAD_THREAD_COUNT) and will block until all of them have been + * downloaded. */ - List getBlobs(List localResources, PortAndAssignment pna, BlobChangingCallback cb) + List getBlobs(List localResources, PortAndAssignment pna, + BlobChangingCallback cb) throws AuthorizationException, KeyNotFoundException, IOException { if ((boolean) conf.getOrDefault(Config.DISABLE_SYMLINKS, false)) { - throw new WrappedKeyNotFoundException("symlinks are disabled so blobs cannot be downloaded."); + throw new WrappedKeyNotFoundException("symlinks are disabled so blobs cannot be " + + "downloaded."); } String user = pna.getOwner(); ArrayList results = new ArrayList<>(); @@ -570,7 +609,8 @@ List getBlobs(List localResources, PortAndAssi for (LocalResource localResource : localResources) { String key = localResource.getBlobName(); boolean uncompress = localResource.shouldUncompress(); - LocalizedResource lrsrc = uncompress ? getUserArchive(user, key) : getUserFile(user, key); + LocalizedResource lrsrc = uncompress ? getUserArchive(user, key) : getUserFile(user, + key); // go off to blobstore and get it // assume dir passed in exists and has correct permission @@ -613,16 +653,21 @@ private void forEachTopologyDistDir(ConsumePathAndId consumer) throws IOExceptio void cleanup() { try { LOG.info("Starting cleanup"); - LocalizedResourceRetentionSet toClean = new LocalizedResourceRetentionSet(cacheTargetSize); + LocalizedResourceRetentionSet toClean = + new LocalizedResourceRetentionSet(cacheTargetSize); // need one large set of all and then clean via LRU - for (Map.Entry> t : userArchives.entrySet()) { + for (Map.Entry> t : userArchives + .entrySet()) { toClean.addResources(t.getValue()); - LOG.debug("Resources to be cleaned after adding {} archives : {}", t.getKey(), toClean); + LOG.debug("Resources to be cleaned after adding {} archives : {}", t.getKey(), + toClean); } - for (Map.Entry> t : userFiles.entrySet()) { + for (Map.Entry> t : userFiles + .entrySet()) { toClean.addResources(t.getValue()); - LOG.debug("Resources to be cleaned after adding {} files : {}", t.getKey(), toClean); + LOG.debug("Resources to be cleaned after adding {} files : {}", t.getKey(), + toClean); } toClean.addResources(topologyBlobs); @@ -641,10 +686,11 @@ void cleanup() { for (String blobKey : topologyBlobs.keySet()) { safeTopologyIds.add(ConfigUtils.getIdFromBlobKey(blobKey)); } - LOG.debug("Topologies {} can no longer be considered fully downloaded", topologiesWithDeletes); + LOG.debug("Topologies {} can no longer be considered fully downloaded", + topologiesWithDeletes); safeTopologyIds.removeAll(topologiesWithDeletes); - //Deleting this early does not hurt anything + // Deleting this early does not hurt anything topologyBasicDownloaded.keySet().removeIf(topoId -> !safeTopologyIds.contains(topoId)); blobPending.keySet().removeIf(topoId -> !safeTopologyIds.contains(topoId)); @@ -711,7 +757,8 @@ public Void get() { String topologyId = pna.getToplogyId(); String topoOwner = pna.getOwner(); String stormroot = ConfigUtils.supervisorStormDistRoot(conf, topologyId); - Map topoConf = ConfigUtils.readSupervisorStormConf(conf, topologyId); + Map topoConf = ConfigUtils.readSupervisorStormConf(conf, + topologyId); @SuppressWarnings("unchecked") Map> blobstoreMap = @@ -723,12 +770,13 @@ public Void get() { if (!fsOps.fileExists(userDir)) { fsOps.forceMkdir(userDir); } - List localizedResources = getBlobs(localResourceList, pna, cb); + List localizedResources = getBlobs(localResourceList, pna, + cb); fsOps.setupBlobPermissions(userDir, topoOwner); if (!symlinksDisabled) { for (LocalizedResource localizedResource : localizedResources) { String keyName = localizedResource.getKey(); - //The sym link we are pointing to + // The sym link we are pointing to File rsrcFilePath = localizedResource.getCurrentSymlinkPath().toFile(); String symlinkName = null; @@ -743,8 +791,11 @@ public Void get() { // all things are from dependencies symlinkName = keyName; } - // the localname may come from the topology conf, it must not point outside of stormroot - fsOps.createSymlink(ServerUtils.resolveTopologyConfSuppliedName(new File(stormroot), symlinkName), + // the localname may come from the topology conf, it must not point + // outside of stormroot + fsOps.createSymlink(ServerUtils + .resolveTopologyConfSuppliedName(new File(stormroot), + symlinkName), rsrcFilePath); } } diff --git a/storm-server/src/main/java/org/apache/storm/localizer/BlobChangingCallback.java b/storm-server/src/main/java/org/apache/storm/localizer/BlobChangingCallback.java index 2ad96cebe17..7d3277995d1 100644 --- a/storm-server/src/main/java/org/apache/storm/localizer/BlobChangingCallback.java +++ b/storm-server/src/main/java/org/apache/storm/localizer/BlobChangingCallback.java @@ -26,10 +26,12 @@ public interface BlobChangingCallback { /** - * Informs the listener that a blob has changed and is ready to update and replace a localized blob that has been marked as tied to the + * Informs the listener that a blob has changed and is ready to update and replace a localized + * blob that has been marked as tied to the * life cycle of the worker process. * - *

      If `go.getLatch()` is never called before the method completes it is assumed that the listener is good with the blob changing. + *

      If `go.getLatch()` is never called before the method completes it is assumed that the + * listener is good with the blob changing. * * @param assignment the assignment this resource and callback are registered with. * @param port the port that this resource and callback are registered with. diff --git a/storm-server/src/main/java/org/apache/storm/localizer/GoodToGo.java b/storm-server/src/main/java/org/apache/storm/localizer/GoodToGo.java index 5050e865d98..af12a6991d5 100644 --- a/storm-server/src/main/java/org/apache/storm/localizer/GoodToGo.java +++ b/storm-server/src/main/java/org/apache/storm/localizer/GoodToGo.java @@ -22,8 +22,10 @@ import java.util.concurrent.Future; /** - * Used as a way to give feedback that the listener is ready for the caller to change the blob. By calling @{link GoodToGo#getLatch()} the - * listener indicates that it wants to block changing the blob until the CountDownLatch is triggered with a call to @{link + * Used as a way to give feedback that the listener is ready for the caller to change the blob. By + * calling @{link GoodToGo#getLatch()} the + * listener indicates that it wants to block changing the blob until the CountDownLatch is triggered + * with a call to @{link * CountDownLatch#countDown()}. */ public class GoodToGo { diff --git a/storm-server/src/main/java/org/apache/storm/localizer/IOFunction.java b/storm-server/src/main/java/org/apache/storm/localizer/IOFunction.java index 0a732b739a7..a8f47c34ba7 100644 --- a/storm-server/src/main/java/org/apache/storm/localizer/IOFunction.java +++ b/storm-server/src/main/java/org/apache/storm/localizer/IOFunction.java @@ -1,15 +1,19 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. - * See the NOTICE file distributed with this work for additional information regarding copyright ownership. + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. + * See the NOTICE file distributed with this work for additional information regarding copyright + * ownership. * The ASF licenses this file to you under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. You may obtain a copy of the License at + * you may not use this file except in compliance with the License. You may obtain a copy of the + * License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, + *

      Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and limitations under the License. + * See the License for the specific language governing permissions and limitations under the + * License. */ package org.apache.storm.localizer; diff --git a/storm-server/src/main/java/org/apache/storm/localizer/LocalResource.java b/storm-server/src/main/java/org/apache/storm/localizer/LocalResource.java index f5dfc77d9da..1561100ee38 100644 --- a/storm-server/src/main/java/org/apache/storm/localizer/LocalResource.java +++ b/storm-server/src/main/java/org/apache/storm/localizer/LocalResource.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -22,9 +28,11 @@ public class LocalResource { /** * Constructor. + * * @param keyname the key of the blob to download. * @param uncompress should the blob be uncompressed or not. - * @param needsCallback if the blobs changes should a callback happen so the worker is restarted. + * @param needsCallback if the blobs changes should a callback happen so the worker is + * restarted. */ public LocalResource(String keyname, boolean uncompress, boolean needsCallback) { blobKey = keyname; diff --git a/storm-server/src/main/java/org/apache/storm/localizer/LocalizedResource.java b/storm-server/src/main/java/org/apache/storm/localizer/LocalizedResource.java index 123920aef54..6781c7dcc98 100644 --- a/storm-server/src/main/java/org/apache/storm/localizer/LocalizedResource.java +++ b/storm-server/src/main/java/org/apache/storm/localizer/LocalizedResource.java @@ -43,7 +43,6 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; - import org.apache.commons.io.FileUtils; import org.apache.storm.Config; import org.apache.storm.blobstore.ClientBlobStore; @@ -62,8 +61,10 @@ import org.slf4j.LoggerFactory; /** - * Represents a resource that is localized on the supervisor. A localized resource has a .current symlink to the current version file which - * is named filename.{current version}. There is also a filename.version which contains the latest version. + * Represents a resource that is localized on the supervisor. A localized resource has a .current + * symlink to the current version file which + * is named filename.{current version}. There is also a filename.version which contains the latest + * version. */ public class LocalizedResource extends LocallyCachedBlob { @VisibleForTesting @@ -94,7 +95,8 @@ public class LocalizedResource extends LocallyCachedBlob { // size of the resource private long size = -1; - LocalizedResource(String key, Path localBaseDir, boolean shouldUncompress, IAdvancedFSOps fsOps, Map conf, + LocalizedResource(String key, Path localBaseDir, boolean shouldUncompress, IAdvancedFSOps fsOps, + Map conf, String user, StormMetricsRegistry metricRegistry) { super(key + (shouldUncompress ? " archive" : " file"), key, metricRegistry); Path base = getLocalUserFileCacheDir(localBaseDir, user); @@ -106,7 +108,7 @@ public class LocalizedResource extends LocallyCachedBlob { versionFilePath = constructVersionFileName(baseDir, key); symlinkPath = constructBlobCurrentSymlinkName(baseDir, key); this.shouldUncompress = shouldUncompress; - //Set the size in case we are recovering an already downloaded object + // Set the size in case we are recovering an already downloaded object setSize(); } @@ -256,15 +258,17 @@ public long fetchUnzipToTemp(ClientBlobStore store) throws IOException, KeyNotFo } DownloadMeta downloadMeta = fetch(store, key, v -> { - Path path = shouldUncompress ? tmpOutputLocation() : constructBlobWithVersionFileName(baseDir, getKey(), v); + Path path = shouldUncompress + ? tmpOutputLocation() : constructBlobWithVersionFileName(baseDir, getKey(), + v); // we need to download to temp file and then unpack into the one requested Path parent = path.getParent(); if (!Files.exists(parent)) { - //There is a race here that we can still lose + // There is a race here that we can still lose try { Files.createDirectories(parent); } catch (FileAlreadyExistsException e) { - //Ignored + // Ignored } catch (IOException e) { LOG.error("Failed to create parent directory {}", parent, e); throw e; @@ -278,7 +282,8 @@ public long fetchUnzipToTemp(ClientBlobStore store) throws IOException, KeyNotFo Path finalLocation = downloadMeta.getDownloadPath(); if (shouldUncompress) { Path downloadFile = finalLocation; - finalLocation = constructBlobWithVersionFileName(baseDir, getKey(), downloadMeta.getVersion()); + finalLocation = constructBlobWithVersionFileName(baseDir, getKey(), downloadMeta + .getVersion()); ServerUtils.unpack(downloadFile.toFile(), finalLocation.toFile(), symLinksDisabled); LOG.debug("Uncompressed {} to: {}", downloadFile, finalLocation); } @@ -307,7 +312,7 @@ protected void commitNewVersion(long version) throws IOException { Path currentSymLink = getCurrentSymlinkPath(); Files.move(tmpSymlink, currentSymLink, ATOMIC_MOVE); - //Update the size of the objects + // Update the size of the objects setSize(); } @@ -322,7 +327,8 @@ private void setBlobPermissions(Map conf, String user, Path path String stormHome = System.getProperty(ConfigUtils.STORM_HOME); wlCommand = stormHome + "/bin/worker-launcher"; } - List command = new ArrayList<>(Arrays.asList(wlCommand, user, "blob", path.toString())); + List command = new ArrayList<>(Arrays.asList(wlCommand, user, "blob", path + .toString())); String[] commandArray = command.toArray(new String[command.size()]); ShellUtils.ShellCommandExecutor shExec = new ShellUtils.ShellCommandExecutor(commandArray); @@ -336,7 +342,8 @@ private void setBlobPermissions(Map conf, String user, Path path LOG.warn("Exit code from worker-launcher is: {}", exitCode, e); LOG.debug("output: {}", shExec.getOutput()); throw new IOException("Setting blob permissions failed" - + " (exitCode=" + exitCode + ") with output: " + shExec.getOutput(), e); + + " (exitCode=" + exitCode + ") with output: " + shExec + .getOutput(), e); } } @@ -345,18 +352,19 @@ private Path tmpOutputLocation() { } private Path tmpSymlinkLocation() { - return baseDir.resolve(Paths.get(LocalizedResource.TO_UNCOMPRESS + getKey() + CURRENT_BLOB_SUFFIX)); + return baseDir.resolve(Paths.get(LocalizedResource.TO_UNCOMPRESS + getKey() + + CURRENT_BLOB_SUFFIX)); } @Override public void cleanupOrphanedData() throws IOException { - //There are a few possible files that we would want to clean up - //baseDir + "/" + "_tmp_" + baseName - //baseDir + "/" + "_tmp_" + baseName + ".current" - //baseDir + "/" + baseName. - //baseDir + "/" + baseName.current - //baseDir + "/" + baseName.version - //In general we always want to delete the _tmp_ files if they are there. + // There are a few possible files that we would want to clean up + // baseDir + "/" + "_tmp_" + baseName + // baseDir + "/" + "_tmp_" + baseName + ".current" + // baseDir + "/" + baseName. + // baseDir + "/" + baseName.current + // baseDir + "/" + baseName.version + // In general we always want to delete the _tmp_ files if they are there. Path tmpOutput = tmpOutputLocation(); Files.deleteIfExists(tmpOutput); @@ -368,7 +376,7 @@ public void cleanupOrphanedData() throws IOException { long version = getLocalVersion(); Path current = getCurrentSymlinkPath(); - //If .current and .version do not match, we roll back the .version file to match + // If .current and .version do not match, we roll back the .version file to match // what .current is pointing to. if (Files.exists(current) && Files.isSymbolicLink(current)) { Path versionFile = Files.readSymbolicLink(current); @@ -376,8 +384,9 @@ public void cleanupOrphanedData() throws IOException { if (m.matches()) { long foundVersion = Long.valueOf(m.group(2)); if (foundVersion != version) { - LOG.error("{} does not match the version file so fix the version file", current); - //The versions are different so roll back to whatever current is + LOG.error("{} does not match the version file so fix the version file", + current); + // The versions are different so roll back to whatever current is try (PrintWriter restoreWriter = new PrintWriter( new BufferedWriter(new FileWriter(versionFilePath.toFile(), false)))) { restoreWriter.println(foundVersion); @@ -387,7 +396,8 @@ public void cleanupOrphanedData() throws IOException { } } - // Finally delete any baseName. files that are not pointed to by the current version + // Finally delete any baseName. files that are not pointed to by the current + // version final long finalVersion = version; LOG.debug("Looking to clean up after {} in {}", getKey(), baseDir); try (DirectoryStream ds = fsOps.newDirectoryStream(baseDir, (path) -> { @@ -408,7 +418,8 @@ public void cleanupOrphanedData() throws IOException { } } } catch (NoSuchFileException e) { - LOG.warn("Nothing to cleanup with baseDir {} even though we expected there to be something there", baseDir); + LOG.warn("Nothing to cleanup with baseDir {} even though we expected there to be " + + "something there", baseDir); } } @@ -445,7 +456,8 @@ public boolean isFullyDownloaded() { public boolean equals(Object other) { if (other instanceof LocalizedResource) { LocalizedResource l = (LocalizedResource) other; - return getKey().equals(l.getKey()) && shouldUncompress == l.shouldUncompress && baseDir.equals(l.baseDir); + return getKey().equals(l.getKey()) && shouldUncompress == l.shouldUncompress && baseDir + .equals(l.baseDir); } return false; } diff --git a/storm-server/src/main/java/org/apache/storm/localizer/LocalizedResourceRetentionSet.java b/storm-server/src/main/java/org/apache/storm/localizer/LocalizedResourceRetentionSet.java index 4d4360ff97b..f6a14a74230 100644 --- a/storm-server/src/main/java/org/apache/storm/localizer/LocalizedResourceRetentionSet.java +++ b/storm-server/src/main/java/org/apache/storm/localizer/LocalizedResourceRetentionSet.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -47,7 +53,8 @@ public class LocalizedResourceRetentionSet { } LocalizedResourceRetentionSet(long targetSize, - SortedMap> retain) { + SortedMap> retain) { this.noReferences = retain; this.targetSize = targetSize; } @@ -59,25 +66,31 @@ protected int getSizeWithNoReferences() { /** * Add blobs to be checked if they can be deleted. - * @param blobs a map of blob name to the blob object. The blobs in this map will be deleted from the map + * + * @param blobs a map of blob name to the blob object. The blobs in this map will be deleted + * from the map * if they are deleted on disk too. */ public void addResources(ConcurrentMap blobs) { for (LocallyCachedBlob b : blobs.values()) { currentSize += b.getSizeOnDisk(); if (b.isUsed()) { - LOG.debug("NOT going to clean up {}, {} depends on it", b.getKey(), b.getDependencies()); + LOG.debug("NOT going to clean up {}, {} depends on it", b.getKey(), b + .getDependencies()); // always retain resources in use continue; } - LOG.debug("Possibly going to clean up {} ts {} size {}", b.getKey(), b.getLastUsed(), b.getSizeOnDisk()); + LOG.debug("Possibly going to clean up {} ts {} size {}", b.getKey(), b.getLastUsed(), b + .getSizeOnDisk()); noReferences.put(b, blobs); } } /** * Actually cleanup the blobs to try and get below the target cache size. - * @param store the blobs store client used to check if the blob has been deleted from the blobstore. If it has, the blob will be + * + * @param store the blobs store client used to check if the blob has been deleted from the + * blobstore. If it has, the blob will be * deleted even if the cache is not over the target size. * @return a set containing any deleted blobs. */ @@ -85,14 +98,15 @@ public Set cleanup(ClientBlobStore store) { Set deleted = new HashSet<>(); LOG.debug("cleanup target size: {} current size is: {}", targetSize, currentSize); long bytesOver = currentSize - targetSize; - //First delete everything that no longer exists... - for (Iterator>> i = noReferences.entrySet().iterator(); + // First delete everything that no longer exists... + for (Iterator>> i = noReferences.entrySet().iterator(); i.hasNext(); ) { Map.Entry> rsrc = i.next(); LocallyCachedBlob resource = rsrc.getKey(); try { if (!store.isRemoteBlobExists(resource.getKey())) { - //The key was removed so we should delete it too. + // The key was removed so we should delete it too. Map set = rsrc.getValue(); if (removeBlob(resource, set)) { bytesOver -= resource.getSizeOnDisk(); @@ -102,11 +116,12 @@ public Set cleanup(ClientBlobStore store) { } } } catch (AuthorizationException e) { - //Ignored + // Ignored } } - for (Iterator>> i = noReferences.entrySet().iterator(); + for (Iterator>> i = noReferences.entrySet().iterator(); bytesOver > 0 && i.hasNext(); ) { Map.Entry> rsrc = i.next(); LocallyCachedBlob resource = rsrc.getKey(); @@ -121,7 +136,8 @@ public Set cleanup(ClientBlobStore store) { return deleted; } - private boolean removeBlob(LocallyCachedBlob blob, Map blobs) { + private boolean removeBlob(LocallyCachedBlob blob, Map blobs) { synchronized (blob) { if (!blob.isUsed()) { try { diff --git a/storm-server/src/main/java/org/apache/storm/localizer/LocallyCachedBlob.java b/storm-server/src/main/java/org/apache/storm/localizer/LocallyCachedBlob.java index 56ca33891ca..5ac43260c4d 100644 --- a/storm-server/src/main/java/org/apache/storm/localizer/LocallyCachedBlob.java +++ b/storm-server/src/main/java/org/apache/storm/localizer/LocallyCachedBlob.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

      - * http://www.apache.org/licenses/LICENSE-2.0 - *

      - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

      http://www.apache.org/licenses/LICENSE-2.0 + * + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -15,7 +21,6 @@ import com.codahale.metrics.Histogram; import com.codahale.metrics.Meter; import com.codahale.metrics.Timer; - import java.io.File; import java.io.IOException; import java.io.OutputStream; @@ -31,7 +36,6 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Stream; - import org.apache.storm.blobstore.ClientBlobStore; import org.apache.storm.blobstore.InputStreamWithMeta; import org.apache.storm.generated.AuthorizationException; @@ -50,7 +54,8 @@ public abstract class LocallyCachedBlob { // A callback that does nothing. private static final BlobChangingCallback NOOP_CB = (assignment, port, resource, go) -> { }; - private final ConcurrentHashMap references = new ConcurrentHashMap<>(); + private final ConcurrentHashMap references = + new ConcurrentHashMap<>(); private final String blobDescription; private final String blobKey; private AtomicLong lastUsed = new AtomicLong(Time.currentTimeMillis()); @@ -63,15 +68,20 @@ public abstract class LocallyCachedBlob { /** * Create a new LocallyCachedBlob. * - * @param blobDescription a description of the blob this represents. Typically it should at least be the blob key, but ideally also - * include if it is an archive or not, what user or topology it is for, or if it is a storm.jar etc. + * @param blobDescription a description of the blob this represents. Typically it should at + * least be the blob key, but ideally also + * include if it is an archive or not, what user or topology it is for, or if it is a + * storm.jar etc. */ - protected LocallyCachedBlob(String blobDescription, String blobKey, StormMetricsRegistry metricsRegistry) { + protected LocallyCachedBlob(String blobDescription, String blobKey, + StormMetricsRegistry metricsRegistry) { this.blobDescription = blobDescription; this.blobKey = blobKey; this.fetchingRate = metricsRegistry.registerHistogram("supervisor:blob-fetching-rate-MB/s"); - this.numBlobUpdateVersionChanged = metricsRegistry.registerMeter("supervisor:num-blob-update-version-changed"); - this.singleBlobLocalizationDuration = metricsRegistry.registerTimer("supervisor:single-blob-localization-duration"); + this.numBlobUpdateVersionChanged = metricsRegistry + .registerMeter("supervisor:num-blob-update-version-changed"); + this.singleBlobLocalizationDuration = metricsRegistry + .registerTimer("supervisor:single-blob-localization-duration"); } /** @@ -79,10 +89,12 @@ protected LocallyCachedBlob(String blobDescription, String blobKey, StormMetrics * * @param store Blob store to fetch blobs from * @param key Key to retrieve blobs - * @param pathSupplier A function that supplies the download destination of a blob. It guarantees the validity + * @param pathSupplier A function that supplies the download destination of a blob. It + * guarantees the validity * of path or throws {@link IOException} * @param outStreamSupplier A function that supplies the {@link OutputStream} object - * @return The metadata of the download session, including blob's version and download destination + * @return The metadata of the download session, including blob's version and download + * destination * @throws KeyNotFoundException Thrown if key to retrieve blob is invalid * @throws AuthorizationException Thrown if the retrieval is not under security authorization * @throws IOException Thrown if any IO error occurs @@ -96,10 +108,11 @@ protected DownloadMeta fetch(ClientBlobStore store, String key, long newVersion = in.getVersion(); long currentVersion = getLocalVersion(); if (newVersion == currentVersion) { - LOG.warn("The version did not change, but going to download again {} {}", currentVersion, key); + LOG.warn("The version did not change, but going to download again {} {}", + currentVersion, key); } - //Make sure the parent directory is there and ready to go + // Make sure the parent directory is there and ready to go Path downloadPath = pathSupplier.apply(newVersion); LOG.debug("Downloading {} to {}", key, downloadPath); @@ -120,7 +133,8 @@ protected DownloadMeta fetch(ClientBlobStore store, String key, long expectedSize = in.getFileLength(); if (totalRead != expectedSize) { - throw new IOException("We expected to download " + expectedSize + " bytes but found we got " + totalRead); + throw new IOException("We expected to download " + expectedSize + + " bytes but found we got " + totalRead); } else { double downloadRate = ((double) totalRead * 1e3) / duration; fetchingRate.update(Math.round(downloadRate)); @@ -130,18 +144,21 @@ protected DownloadMeta fetch(ClientBlobStore store, String key, } /** - * Get the version of the blob cached locally. If the version is unknown or it has not been downloaded NOT_DOWNLOADED_VERSION should be + * Get the version of the blob cached locally. If the version is unknown or it has not been + * downloaded NOT_DOWNLOADED_VERSION should be * returned. PRECONDITION: this can only be called with a lock on this instance held. */ public abstract long getLocalVersion(); /** - * Get the version of the blob in the blob store. PRECONDITION: this can only be called with a lock on this instance held. + * Get the version of the blob in the blob store. PRECONDITION: this can only be called with a + * lock on this instance held. */ public abstract long getRemoteVersion(ClientBlobStore store) throws KeyNotFoundException, AuthorizationException; /** - * Download the latest version to a temp location. This may also include unzipping some or all of the data to a temp location. + * Download the latest version to a temp location. This may also include unzipping some or all + * of the data to a temp location. * PRECONDITION: this can only be called with a lock on this instance held. * * @param store the store to us to download the data. @@ -159,22 +176,26 @@ protected DownloadMeta fetch(ClientBlobStore store, String key, protected abstract void commitNewVersion(long version) throws IOException; /** - * Clean up any temporary files. This will be called after updating a blob, either successfully or if an error has occurred. + * Clean up any temporary files. This will be called after updating a blob, either successfully + * or if an error has occurred. * The goal is to find any files that may be left over and remove them so space is not leaked. * PRECONDITION: this can only be called with a lock on this instance held. */ public abstract void cleanupOrphanedData() throws IOException; /** - * Completely remove anything that is cached locally for this blob and all tracking files also stored for it. + * Completely remove anything that is cached locally for this blob and all tracking files also + * stored for it. * This will be called after the blob was determined to no longer be needed in the cache. * PRECONDITION: this can only be called with a lock on this instance held. */ public abstract void completelyRemove() throws IOException; /** - * Get the amount of disk space that is used by this blob. If the blob is uncompressed it should be the sum of the space used by all - * of the uncompressed files. In general this will not be called with any locks held so it is a good idea to cache it and updated it + * Get the amount of disk space that is used by this blob. If the blob is uncompressed it should + * be the sum of the space used by all + * of the uncompressed files. In general this will not be called with any locks held so it is a + * good idea to cache it and updated it * when committing a new version. */ public abstract long getSizeOnDisk(); @@ -191,10 +212,11 @@ protected static long getSizeOnDisk(Path p) throws IOException { } else if (Files.isRegularFile(p)) { return Files.size(p); } else { - //We will not follow sym links + // We will not follow sym links try (Stream stream = Files.walk(p)) { return - stream.filter((subp) -> Files.isRegularFile(subp, LinkOption.NOFOLLOW_LINKS)) + stream.filter((subp) -> Files.isRegularFile(subp, + LinkOption.NOFOLLOW_LINKS)) .mapToLong((subp) -> { try { return Files.size(subp); @@ -223,7 +245,8 @@ public long getLastUsed() { } /** - * Return true if this blob is actively being used, else false (meaning it can be deleted, but might not be). + * Return true if this blob is actively being used, else false (meaning it can be deleted, but + * might not be). */ public boolean isUsed() { return !references.isEmpty(); @@ -233,11 +256,13 @@ public boolean isUsed() { * Mark that a given port and assignment are using this. * * @param pna the slot and assignment that are using this blob. - * @param cb an optional callback indicating that they want to know/synchronize when a blob is updated. + * @param cb an optional callback indicating that they want to know/synchronize when a blob is + * updated. */ public void addReference(final PortAndAssignment pna, BlobChangingCallback cb) { touch(); - LOG.info("Adding reference {} with timestamp {} to {}", pna, getLastUsed(), blobDescription); + LOG.info("Adding reference {} with timestamp {} to {}", pna, getLastUsed(), + blobDescription); if (cb == null) { cb = NOOP_CB; } @@ -267,15 +292,18 @@ public boolean removeReference(final PortAndAssignment pna) { touch(); return true; } else { - LOG.warn("{} had no reservation for {}, current references are {} with last update at {}", + LOG.warn("{} had no reservation for {}, current references are {} with last update at " + + "{}", pna, blobDescription, getDependencies(), getLastUsed()); return false; } } /** - * Inform all of the callbacks that a change is going to happen and then wait for them to all get back that it is OK to make that - * change. Commit the new version once all callbacks are ready. Finally inform all callbacks that the commit is complete. + * Inform all of the callbacks that a change is going to happen and then wait for them to all + * get back that it is OK to make that + * change. Commit the new version once all callbacks are ready. Finally inform all callbacks + * that the commit is complete. */ public synchronized void informReferencesAndCommitNewVersion(long newVersion) throws IOException { CompletableFuture doneUpdating = informAllOfChangeAndWaitForConsensus(); @@ -306,7 +334,7 @@ private CompletableFuture informAllOfChangeAndWaitForConsensus() { try { cdl.await(3, TimeUnit.MINUTES); } catch (InterruptedException e) { - //Interrupted is thrown when we are shutting down. + // Interrupted is thrown when we are shutting down. // So just ignore it for now... } return doneUpdating; @@ -319,7 +347,6 @@ public String getKey() { return blobKey; } - public Collection getDependencies() { return references.keySet(); } @@ -335,7 +362,8 @@ public Collection getDependencies() { * @throws KeyNotFoundException if the remote blob is missing * @throws AuthorizationException if authorization is failed */ - boolean requiresUpdate(ClientBlobStore blobStore, long remoteBlobstoreUpdateTime) throws KeyNotFoundException, AuthorizationException { + boolean requiresUpdate(ClientBlobStore blobStore, + long remoteBlobstoreUpdateTime) throws KeyNotFoundException, AuthorizationException { if (!this.isUsed()) { return false; } @@ -348,7 +376,8 @@ boolean requiresUpdate(ClientBlobStore blobStore, long remoteBlobstoreUpdateTime // the remote blobstore for the remote file. This reduces Hadoop namenode impact of // 100's of supervisors querying multiple blobs. if (remoteBlobstoreUpdateTime > 0 && this.localUpdateTime == remoteBlobstoreUpdateTime) { - LOG.debug("{} is up to date, blob localUpdateTime matches remote timestamp {}", this, remoteBlobstoreUpdateTime); + LOG.debug("{} is up to date, blob localUpdateTime matches remote timestamp {}", this, + remoteBlobstoreUpdateTime); return false; } @@ -357,7 +386,8 @@ boolean requiresUpdate(ClientBlobStore blobStore, long remoteBlobstoreUpdateTime if (localVersion != remoteVersion) { return true; } else { - // track that we are now up to date with respect to last time the remote blobstore was updated + // track that we are now up to date with respect to last time the remote blobstore was + // updated this.localUpdateTime = remoteBlobstoreUpdateTime; return false; } @@ -382,7 +412,8 @@ private void download(ClientBlobStore blobStore, long remoteBlobstoreUpdateTime) long newVersion = this.fetchUnzipToTemp(blobStore); this.informReferencesAndCommitNewVersion(newVersion); this.localUpdateTime = remoteBlobstoreUpdateTime; - LOG.debug("local blob {} downloaded, in sync with remote blobstore to time {}", this, remoteBlobstoreUpdateTime); + LOG.debug("local blob {} downloaded, in sync with remote blobstore to time {}", this, + remoteBlobstoreUpdateTime); } finally { timer.stop(); this.cleanupOrphanedData(); diff --git a/storm-server/src/main/java/org/apache/storm/localizer/LocallyCachedTopologyBlob.java b/storm-server/src/main/java/org/apache/storm/localizer/LocallyCachedTopologyBlob.java index 773b464fc54..1679cbbe18e 100644 --- a/storm-server/src/main/java/org/apache/storm/localizer/LocallyCachedTopologyBlob.java +++ b/storm-server/src/main/java/org/apache/storm/localizer/LocallyCachedTopologyBlob.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

      - * http://www.apache.org/licenses/LICENSE-2.0 - *

      - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

      http://www.apache.org/licenses/LICENSE-2.0 + * + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -48,7 +54,8 @@ public class LocallyCachedTopologyBlob extends LocallyCachedBlob { public static final long LOCAL_MODE_JAR_VERSION = 1; private static final Logger LOG = LoggerFactory.getLogger(LocallyCachedTopologyBlob.class); - private static final Pattern EXTRACT_BASE_NAME_AND_VERSION = Pattern.compile("^(.*)\\.([0-9]+)$"); + private static final Pattern EXTRACT_BASE_NAME_AND_VERSION = Pattern + .compile("^(.*)\\.([0-9]+)$"); private final TopologyBlobType type; private final String topologyId; private final boolean isLocalMode; @@ -61,11 +68,13 @@ public class LocallyCachedTopologyBlob extends LocallyCachedBlob { /** * Create a new LocallyCachedBlob. + * * @param topologyId the ID of the topology. * @param type the type of the blob. * @param owner the name of the user that owns this blob. */ - protected LocallyCachedTopologyBlob(final String topologyId, final boolean isLocalMode, final Map conf, + protected LocallyCachedTopologyBlob(final String topologyId, final boolean isLocalMode, + final Map conf, final AdvancedFSOps fsOps, final TopologyBlobType type, String owner, StormMetricsRegistry metricsRegistry) throws IOException { super(topologyId + " " + type.getFileName(), type.getKey(topologyId), metricsRegistry); @@ -75,7 +84,8 @@ protected LocallyCachedTopologyBlob(final String topologyId, final boolean isLoc this.fsOps = fsOps; this.owner = owner; this.conf = conf; - topologyBasicBlobsRootDir = Paths.get(ConfigUtils.supervisorStormDistRoot(conf, topologyId)); + topologyBasicBlobsRootDir = Paths.get(ConfigUtils.supervisorStormDistRoot(conf, + topologyId)); readVersion(); updateSizeOnDisk(); } @@ -140,10 +150,12 @@ public long fetchUnzipToTemp(ClientBlobStore store) } if (isLocalMode && type == TopologyBlobType.TOPO_JAR) { LOG.debug("DOWNLOADING LOCAL JAR to TEMP LOCATION... {}", topologyId); - //This is a special case where the jar was not uploaded so we will not download it (it is already on the classpath) + // This is a special case where the jar was not uploaded so we will not download it (it + // is already on the classpath) String resourcesJar = resourcesJar(); URL url = ServerUtils.getResourceFromClassloader(ServerConfigUtils.RESOURCES_SUBDIR); - Path extractionDest = topologyBasicBlobsRootDir.resolve(type.getTempExtractionDir(LOCAL_MODE_JAR_VERSION)); + Path extractionDest = topologyBasicBlobsRootDir.resolve(type + .getTempExtractionDir(LOCAL_MODE_JAR_VERSION)); if (resourcesJar != null) { LOG.info("Extracting resources from jar at {} to {}", resourcesJar, extractionDest); extractDirFromJar(resourcesJar, ServerConfigUtils.RESOURCES_SUBDIR, extractionDest); @@ -151,13 +163,16 @@ public long fetchUnzipToTemp(ClientBlobStore store) LOG.info("Copying resources at {} to {}", url, extractionDest); if ("jar".equals(url.getProtocol())) { JarURLConnection urlConnection = (JarURLConnection) url.openConnection(); - extractDirFromJar(urlConnection.getJarFileURL().getFile(), ServerConfigUtils.RESOURCES_SUBDIR, extractionDest); + extractDirFromJar(urlConnection.getJarFileURL().getFile(), + ServerConfigUtils.RESOURCES_SUBDIR, extractionDest); } else { fsOps.copyDirectory(new File(url.getFile()), extractionDest.toFile()); } } else if (!fsOps.fileExists(extractionDest)) { - // if we can't find the resources directory in a resources jar or in the classpath just create an empty - // resources directory. This way we can check later that the topology jar was fully downloaded. + // if we can't find the resources directory in a resources jar or in the classpath + // just create an empty + // resources directory. This way we can check later that the topology jar was fully + // downloaded. fsOps.forceMkdir(extractionDest); } return LOCAL_MODE_JAR_VERSION; @@ -174,8 +189,10 @@ public long fetchUnzipToTemp(ClientBlobStore store) Path tmpLocation = downloadMeta.getDownloadPath(); if (type.needsExtraction()) { - Path extractionDest = topologyBasicBlobsRootDir.resolve(type.getTempExtractionDir(downloadMeta.getVersion())); - extractDirFromJar(tmpLocation.toAbsolutePath().toString(), ServerConfigUtils.RESOURCES_SUBDIR, + Path extractionDest = topologyBasicBlobsRootDir.resolve(type + .getTempExtractionDir(downloadMeta.getVersion())); + extractDirFromJar(tmpLocation.toAbsolutePath().toString(), + ServerConfigUtils.RESOURCES_SUBDIR, extractionDest); } return downloadMeta.getVersion(); @@ -184,7 +201,8 @@ public long fetchUnzipToTemp(ClientBlobStore store) protected void extractDirFromJar(String jarpath, String dir, Path dest) throws IOException { LOG.debug("EXTRACTING {} from {} and placing it at {}", dir, jarpath, dest); if (!Files.exists(dest)) { - //Create the directory no matter what. This is so we can check if it was downloaded in the future. + // Create the directory no matter what. This is so we can check if it was downloaded in + // the future. Files.createDirectories(dest); } try (JarFile jarFile = new JarFile(jarpath)) { @@ -210,17 +228,19 @@ public boolean isFullyDownloaded() { @Override protected void commitNewVersion(long newVersion) throws IOException { - //This is not atomic (so if something bad happens in the middle we need to be able to recover + // This is not atomic (so if something bad happens in the middle we need to be able to + // recover Path tempLoc = topologyBasicBlobsRootDir.resolve(type.getTempFileName(newVersion)); Path dest = topologyBasicBlobsRootDir.resolve(type.getFileName()); Path versionFile = topologyBasicBlobsRootDir.resolve(type.getVersionFileName()); LOG.debug("Removing version file {} to force download on failure", versionFile); - fsOps.deleteIfExists(versionFile.toFile()); //So if we fail we are forced to try again + fsOps.deleteIfExists(versionFile.toFile()); // So if we fail we are forced to try again LOG.debug("Removing destination file {} in preparation for move", dest); fsOps.deleteIfExists(dest.toFile()); if (type.needsExtraction()) { - Path extractionTemp = topologyBasicBlobsRootDir.resolve(type.getTempExtractionDir(newVersion)); + Path extractionTemp = topologyBasicBlobsRootDir.resolve(type + .getTempExtractionDir(newVersion)); Path extractionDest = topologyBasicBlobsRootDir.resolve(type.getExtractionDir()); LOG.debug("Removing extraction dest {} in preparation for extraction", extractionDest); fsOps.deleteIfExists(extractionDest.toFile()); @@ -229,19 +249,28 @@ protected void commitNewVersion(long newVersion) throws IOException { } } if (!(isLocalMode && type == TopologyBlobType.TOPO_JAR)) { - //Don't try to move the JAR file in local mode, it does not exist because it was not uploaded + // Don't try to move the JAR file in local mode, it does not exist because it was not + // uploaded fsOps.moveFile(tempLoc.toFile(), dest.toFile()); } synchronized (LocallyCachedTopologyBlob.class) { - //This is a bit ugly, but it works. In order to maintain the same directory structure that existed before - // we need to have storm conf, storm jar, and storm code in a shared directory, and we need to set the - // permissions for that entire directory, but the tracking is on a per item basis, so we are going to end - // up running the permission modification code once for each blob that is downloaded (3 times in this case). - // Because the permission modification code runs in a separate process we are doing a global lock to avoid - // any races between multiple versions running at the same time. Ideally this would be on a per topology - // basis, but that is a lot harder and the changes run fairly quickly so it should not be a big deal. + // This is a bit ugly, but it works. In order to maintain the same directory structure + // that existed before + // we need to have storm conf, storm jar, and storm code in a shared directory, and we + // need to set the + // permissions for that entire directory, but the tracking is on a per item basis, so we + // are going to end + // up running the permission modification code once for each blob that is downloaded (3 + // times in this case). + // Because the permission modification code runs in a separate process we are doing a + // global lock to avoid + // any races between multiple versions running at the same time. Ideally this would be + // on a per topology + // basis, but that is a lot harder and the changes run fairly quickly so it should not + // be a big deal. fsOps.setupStormCodeDir(owner, topologyBasicBlobsRootDir.toFile()); - File sharedMemoryDirFinalLocation = new File(ConfigUtils.sharedByTopologyDir(conf, topologyId)); + File sharedMemoryDirFinalLocation = new File(ConfigUtils.sharedByTopologyDir(conf, + topologyId)); sharedMemoryDirFinalLocation.mkdirs(); fsOps.setupWorkerArtifactsDir(owner, sharedMemoryDirFinalLocation); } @@ -269,7 +298,8 @@ private void cleanUpTemp(String baseName) throws IOException { return m.matches() && baseName.equals(m.group(1)); }) ) { - //children is only ever null if topologyBasicBlobsRootDir does not exist. This happens during unit tests + // children is only ever null if topologyBasicBlobsRootDir does not exist. This happens + // during unit tests // And because a non-existant directory is by definition clean we are ignoring it. if (children != null) { for (Path p : children) { @@ -309,7 +339,8 @@ public long getSizeOnDisk() { public boolean equals(Object other) { if (other instanceof LocallyCachedTopologyBlob) { LocallyCachedTopologyBlob o = (LocallyCachedTopologyBlob) other; - return topologyId.equals(o.topologyId) && type == o.type && topologyBasicBlobsRootDir.equals(o.topologyBasicBlobsRootDir); + return topologyId.equals(o.topologyId) && type == o.type && topologyBasicBlobsRootDir + .equals(o.topologyBasicBlobsRootDir); } return false; } diff --git a/storm-server/src/main/java/org/apache/storm/localizer/PortAndAssignment.java b/storm-server/src/main/java/org/apache/storm/localizer/PortAndAssignment.java index fdf9f8b7de8..dcf35628abb 100644 --- a/storm-server/src/main/java/org/apache/storm/localizer/PortAndAssignment.java +++ b/storm-server/src/main/java/org/apache/storm/localizer/PortAndAssignment.java @@ -1,15 +1,19 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. - * See the NOTICE file distributed with this work for additional information regarding copyright ownership. + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. + * See the NOTICE file distributed with this work for additional information regarding copyright + * ownership. * The ASF licenses this file to you under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

      - * http://www.apache.org/licenses/LICENSE-2.0 - *

      - * Unless required by applicable law or agreed to in writing, + * you may not use this file except in compliance with the License. You may obtain a copy of the + * License at + * + *

      http://www.apache.org/licenses/LICENSE-2.0 + * + *

      Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and limitations under the License. + * See the License for the specific language governing permissions and limitations under the + * License. */ package org.apache.storm.localizer; @@ -31,7 +35,8 @@ default void complete() { default boolean isEquivalentTo(PortAndAssignment other) { if (getPort() == other.getPort()) { - return EquivalenceUtils.areLocalAssignmentsEquivalent(getAssignment(), other.getAssignment()); + return EquivalenceUtils.areLocalAssignmentsEquivalent(getAssignment(), other + .getAssignment()); } return false; } diff --git a/storm-server/src/main/java/org/apache/storm/localizer/PortAndAssignmentImpl.java b/storm-server/src/main/java/org/apache/storm/localizer/PortAndAssignmentImpl.java index 509ef14495f..3feb1a94819 100644 --- a/storm-server/src/main/java/org/apache/storm/localizer/PortAndAssignmentImpl.java +++ b/storm-server/src/main/java/org/apache/storm/localizer/PortAndAssignmentImpl.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -70,7 +76,7 @@ public int getPort() { } /** - * return the assigment for this. + * Return the assigment for this. */ @Override public LocalAssignment getAssignment() { diff --git a/storm-server/src/main/java/org/apache/storm/localizer/TimePortAndAssignment.java b/storm-server/src/main/java/org/apache/storm/localizer/TimePortAndAssignment.java index 82bef5c4b0b..4a4b419dad6 100644 --- a/storm-server/src/main/java/org/apache/storm/localizer/TimePortAndAssignment.java +++ b/storm-server/src/main/java/org/apache/storm/localizer/TimePortAndAssignment.java @@ -1,15 +1,19 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. - * See the NOTICE file distributed with this work for additional information regarding copyright ownership. + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. + * See the NOTICE file distributed with this work for additional information regarding copyright + * ownership. * The ASF licenses this file to you under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

      - * http://www.apache.org/licenses/LICENSE-2.0 - *

      - * Unless required by applicable law or agreed to in writing, + * you may not use this file except in compliance with the License. You may obtain a copy of the + * License at + * + *

      http://www.apache.org/licenses/LICENSE-2.0 + * + *

      Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and limitations under the License. + * See the License for the specific language governing permissions and limitations under the + * License. */ package org.apache.storm.localizer; @@ -51,7 +55,8 @@ public void complete() { @Override public String toString() { - return "TimePortAndAssignment{" + getAssignment().get_topology_id() + " on " + getPort() + "}"; + return "TimePortAndAssignment{" + getAssignment().get_topology_id() + " on " + getPort() + + "}"; } /** diff --git a/storm-server/src/main/java/org/apache/storm/logging/ThriftAccessLogger.java b/storm-server/src/main/java/org/apache/storm/logging/ThriftAccessLogger.java index 78ab822b2ce..f2295402fc0 100644 --- a/storm-server/src/main/java/org/apache/storm/logging/ThriftAccessLogger.java +++ b/storm-server/src/main/java/org/apache/storm/logging/ThriftAccessLogger.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -20,7 +26,8 @@ public class ThriftAccessLogger { private static final Logger LOG = LoggerFactory.getLogger(ThriftAccessLogger.class); - private static String accessLogBase = "Request ID: {} access from: {} principal: {} operation: {}"; + private static String accessLogBase = + "Request ID: {} access from: {} principal: {} operation: {}"; public static void logAccessFunction(Integer requestId, InetAddress remoteAddress, Principal principal, String operation, diff --git a/storm-server/src/main/java/org/apache/storm/logging/filters/AccessLoggingFilter.java b/storm-server/src/main/java/org/apache/storm/logging/filters/AccessLoggingFilter.java index 09be1dfb484..401ab118f7d 100644 --- a/storm-server/src/main/java/org/apache/storm/logging/filters/AccessLoggingFilter.java +++ b/storm-server/src/main/java/org/apache/storm/logging/filters/AccessLoggingFilter.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

      - * http://www.apache.org/licenses/LICENSE-2.0 - *

      - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

      http://www.apache.org/licenses/LICENSE-2.0 + * + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -20,9 +26,7 @@ import jakarta.servlet.ServletResponse; import jakarta.servlet.http.HttpServletRequest; import jakarta.servlet.http.HttpServletResponse; - import java.io.IOException; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -32,24 +36,28 @@ public class AccessLoggingFilter implements Filter { @Override public void init(FilterConfig config) throws ServletException { - //NOOP + // NOOP } @Override - public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) throws IOException, ServletException { + public void doFilter(ServletRequest request, ServletResponse response, + FilterChain chain) throws IOException, ServletException { handle((HttpServletRequest) request, (HttpServletResponse) response, chain); } - public void handle(HttpServletRequest request, HttpServletResponse response, FilterChain chain) throws IOException, ServletException { + public void handle(HttpServletRequest request, HttpServletResponse response, + FilterChain chain) throws IOException, ServletException { if (request != null) { - LOG.info("Access from: {} url: {} principal: {}", request.getRemoteAddr(), request.getRequestURL(), - (request.getUserPrincipal() == null ? "" : request.getUserPrincipal().getName())); + LOG.info("Access from: {} url: {} principal: {}", request.getRemoteAddr(), request + .getRequestURL(), + (request.getUserPrincipal() == null ? "" : request.getUserPrincipal() + .getName())); } chain.doFilter(request, response); } @Override public void destroy() { - //NOOP + // NOOP } } diff --git a/storm-server/src/main/java/org/apache/storm/metric/ClusterMetricsConsumerExecutor.java b/storm-server/src/main/java/org/apache/storm/metric/ClusterMetricsConsumerExecutor.java index 6f2cd30e319..f17be0aedee 100644 --- a/storm-server/src/main/java/org/apache/storm/metric/ClusterMetricsConsumerExecutor.java +++ b/storm-server/src/main/java/org/apache/storm/metric/ClusterMetricsConsumerExecutor.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -22,7 +28,8 @@ public class ClusterMetricsConsumerExecutor { public static final Logger LOG = LoggerFactory.getLogger(ClusterMetricsConsumerExecutor.class); private static final String ERROR_MESSAGE_PREPARATION_CLUSTER_METRICS_CONSUMER_FAILED = "Preparation of Cluster Metrics Consumer failed. " - + "Please check your configuration and/or corresponding systems and relaunch Nimbus. " + + "Please check your configuration and/or corresponding systems and relaunch " + + "Nimbus. " + "Skipping handle metrics."; private IClusterMetricsConsumer metricsConsumer; @@ -36,10 +43,12 @@ public ClusterMetricsConsumerExecutor(String consumerClassName, Object registrat public void prepare() { try { - metricsConsumer = (IClusterMetricsConsumer) Class.forName(consumerClassName).newInstance(); + metricsConsumer = (IClusterMetricsConsumer) Class.forName(consumerClassName) + .newInstance(); metricsConsumer.prepare(registrationArgument); } catch (Exception e) { - LOG.error("Could not instantiate or prepare Cluster Metrics Consumer with fully qualified name " + LOG.error("Could not instantiate or prepare Cluster Metrics Consumer with fully " + + "qualified name " + consumerClassName, e); @@ -50,7 +59,8 @@ public void prepare() { } } - public void handleDataPoints(final IClusterMetricsConsumer.ClusterInfo clusterInfo, final Collection dataPoints) { + public void handleDataPoints(final IClusterMetricsConsumer.ClusterInfo clusterInfo, + final Collection dataPoints) { if (metricsConsumer == null) { LOG.error(ERROR_MESSAGE_PREPARATION_CLUSTER_METRICS_CONSUMER_FAILED); return; @@ -59,11 +69,13 @@ public void handleDataPoints(final IClusterMetricsConsumer.ClusterInfo clusterIn try { metricsConsumer.handleDataPoints(clusterInfo, dataPoints); } catch (Throwable e) { - LOG.error("Error while handling cluster data points, consumer class: " + consumerClassName, e); + LOG.error("Error while handling cluster data points, consumer class: " + + consumerClassName, e); } } - public void handleDataPoints(final IClusterMetricsConsumer.SupervisorInfo supervisorInfo, final Collection dataPoints) { + public void handleDataPoints(final IClusterMetricsConsumer.SupervisorInfo supervisorInfo, + final Collection dataPoints) { if (metricsConsumer == null) { LOG.error(ERROR_MESSAGE_PREPARATION_CLUSTER_METRICS_CONSUMER_FAILED); return; @@ -72,7 +84,8 @@ public void handleDataPoints(final IClusterMetricsConsumer.SupervisorInfo superv try { metricsConsumer.handleDataPoints(supervisorInfo, dataPoints); } catch (Throwable e) { - LOG.error("Error while handling cluster data points, consumer class: " + consumerClassName, e); + LOG.error("Error while handling cluster data points, consumer class: " + + consumerClassName, e); } } diff --git a/storm-server/src/main/java/org/apache/storm/metric/LoggingClusterMetricsConsumer.java b/storm-server/src/main/java/org/apache/storm/metric/LoggingClusterMetricsConsumer.java index 0e296fb28b7..42069a6e280 100644 --- a/storm-server/src/main/java/org/apache/storm/metric/LoggingClusterMetricsConsumer.java +++ b/storm-server/src/main/java/org/apache/storm/metric/LoggingClusterMetricsConsumer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-server/src/main/java/org/apache/storm/metric/StormMetricsRegistry.java b/storm-server/src/main/java/org/apache/storm/metric/StormMetricsRegistry.java index 5db347ebb87..7133026573b 100644 --- a/storm-server/src/main/java/org/apache/storm/metric/StormMetricsRegistry.java +++ b/storm-server/src/main/java/org/apache/storm/metric/StormMetricsRegistry.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -62,7 +68,7 @@ public void registerAll(MetricSet metrics) { } public void removeAll(MetricSet metrics) { - //Could be replaced when metrics support remove all functions + // Could be replaced when metrics support remove all functions // https://github.com/dropwizard/metrics/pull/1280 Map nameToMetric = metrics.getMetrics(); registry.removeMatching((name, metric) -> nameToMetric.containsKey(name)); diff --git a/storm-server/src/main/java/org/apache/storm/metric/api/DataPoint.java b/storm-server/src/main/java/org/apache/storm/metric/api/DataPoint.java index 8468722da6a..6bdc7d285da 100644 --- a/storm-server/src/main/java/org/apache/storm/metric/api/DataPoint.java +++ b/storm-server/src/main/java/org/apache/storm/metric/api/DataPoint.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-server/src/main/java/org/apache/storm/metric/api/IClusterMetricsConsumer.java b/storm-server/src/main/java/org/apache/storm/metric/api/IClusterMetricsConsumer.java index a0b0a8d04be..d10b3f21d32 100644 --- a/storm-server/src/main/java/org/apache/storm/metric/api/IClusterMetricsConsumer.java +++ b/storm-server/src/main/java/org/apache/storm/metric/api/IClusterMetricsConsumer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-server/src/main/java/org/apache/storm/metric/timed/Timed.java b/storm-server/src/main/java/org/apache/storm/metric/timed/Timed.java index f5e8a9678a2..f3ae0299ab7 100644 --- a/storm-server/src/main/java/org/apache/storm/metric/timed/Timed.java +++ b/storm-server/src/main/java/org/apache/storm/metric/timed/Timed.java @@ -1,15 +1,19 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. - * See the NOTICE file distributed with this work for additional information regarding copyright ownership. + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. + * See the NOTICE file distributed with this work for additional information regarding copyright + * ownership. * The ASF licenses this file to you under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. You may obtain a copy of the License at + * you may not use this file except in compliance with the License. You may obtain a copy of the + * License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, + *

      Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and limitations under the License. + * See the License for the specific language governing permissions and limitations under the + * License. */ package org.apache.storm.metric.timed; diff --git a/storm-server/src/main/java/org/apache/storm/metric/timed/TimedResource.java b/storm-server/src/main/java/org/apache/storm/metric/timed/TimedResource.java index ceca7587168..eac29aef8d2 100644 --- a/storm-server/src/main/java/org/apache/storm/metric/timed/TimedResource.java +++ b/storm-server/src/main/java/org/apache/storm/metric/timed/TimedResource.java @@ -1,15 +1,19 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. - * See the NOTICE file distributed with this work for additional information regarding copyright ownership. + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. + * See the NOTICE file distributed with this work for additional information regarding copyright + * ownership. * The ASF licenses this file to you under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. You may obtain a copy of the License at + * you may not use this file except in compliance with the License. You may obtain a copy of the + * License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, + *

      Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and limitations under the License. + * See the License for the specific language governing permissions and limitations under the + * License. */ package org.apache.storm.metric.timed; diff --git a/storm-server/src/main/java/org/apache/storm/metric/timed/TimerDecorated.java b/storm-server/src/main/java/org/apache/storm/metric/timed/TimerDecorated.java index abac6363357..76af4a9e18a 100644 --- a/storm-server/src/main/java/org/apache/storm/metric/timed/TimerDecorated.java +++ b/storm-server/src/main/java/org/apache/storm/metric/timed/TimerDecorated.java @@ -1,15 +1,19 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. - * See the NOTICE file distributed with this work for additional information regarding copyright ownership. + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. + * See the NOTICE file distributed with this work for additional information regarding copyright + * ownership. * The ASF licenses this file to you under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. You may obtain a copy of the License at + * you may not use this file except in compliance with the License. You may obtain a copy of the + * License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, + *

      Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and limitations under the License. + * See the License for the specific language governing permissions and limitations under the + * License. */ package org.apache.storm.metric.timed; diff --git a/storm-server/src/main/java/org/apache/storm/metricstore/AggLevel.java b/storm-server/src/main/java/org/apache/storm/metricstore/AggLevel.java index d8beaa091d6..5a638413394 100644 --- a/storm-server/src/main/java/org/apache/storm/metricstore/AggLevel.java +++ b/storm-server/src/main/java/org/apache/storm/metricstore/AggLevel.java @@ -1,11 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-server/src/main/java/org/apache/storm/metricstore/FilterOptions.java b/storm-server/src/main/java/org/apache/storm/metricstore/FilterOptions.java index 33d2097cbde..170b0b94eda 100644 --- a/storm-server/src/main/java/org/apache/storm/metricstore/FilterOptions.java +++ b/storm-server/src/main/java/org/apache/storm/metricstore/FilterOptions.java @@ -1,11 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -15,7 +21,8 @@ import java.util.Set; /** - * FilterOptions provides a method to select various filtering options for doing a scan of the metrics database. + * FilterOptions provides a method to select various filtering options for doing a scan of the + * metrics database. */ public class FilterOptions { private Set aggLevels = null; @@ -125,7 +132,8 @@ public void addAggLevel(AggLevel level) { */ public Set getAggLevels() { if (this.aggLevels == null) { - // assume filter choices have been made and since no selection was made, all levels are valid + // assume filter choices have been made and since no selection was made, all levels are + // valid this.aggLevels = new HashSet<>(4); aggLevels.add(AggLevel.AGG_LEVEL_NONE); aggLevels.add(AggLevel.AGG_LEVEL_1_MIN); diff --git a/storm-server/src/main/java/org/apache/storm/metricstore/Metric.java b/storm-server/src/main/java/org/apache/storm/metricstore/Metric.java index 5f8bee365fa..60c954a24b5 100644 --- a/storm-server/src/main/java/org/apache/storm/metricstore/Metric.java +++ b/storm-server/src/main/java/org/apache/storm/metricstore/Metric.java @@ -1,11 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -42,11 +48,11 @@ public class Metric implements Comparable { private double max = 0.0; private double sum = 0.0; - /** * Metric constructor. */ - public Metric(String name, Long timestamp, String topologyId, double value, String componentId, String executorId, + public Metric(String name, Long timestamp, String topologyId, double value, String componentId, + String executorId, String hostname, String streamId, int port, AggLevel aggLevel) throws MetricException { this.name = name; this.timestamp = timestamp; diff --git a/storm-server/src/main/java/org/apache/storm/metricstore/MetricException.java b/storm-server/src/main/java/org/apache/storm/metricstore/MetricException.java index 8236e7fd51d..fd6ed680594 100644 --- a/storm-server/src/main/java/org/apache/storm/metricstore/MetricException.java +++ b/storm-server/src/main/java/org/apache/storm/metricstore/MetricException.java @@ -1,11 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-server/src/main/java/org/apache/storm/metricstore/MetricStore.java b/storm-server/src/main/java/org/apache/storm/metricstore/MetricStore.java index 82e4e354d43..f05efff1496 100644 --- a/storm-server/src/main/java/org/apache/storm/metricstore/MetricStore.java +++ b/storm-server/src/main/java/org/apache/storm/metricstore/MetricStore.java @@ -1,11 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -23,7 +29,8 @@ public interface MetricStore extends AutoCloseable { * @param metricsRegistry The Nimbus daemon metrics registry * @throws MetricException on preparation error */ - void prepare(Map config, StormMetricsRegistry metricsRegistry) throws MetricException; + void prepare(Map config, + StormMetricsRegistry metricsRegistry) throws MetricException; /** * Stores a metric in the store. diff --git a/storm-server/src/main/java/org/apache/storm/metricstore/MetricStoreConfig.java b/storm-server/src/main/java/org/apache/storm/metricstore/MetricStoreConfig.java index 22cff7c72dd..50606d06524 100644 --- a/storm-server/src/main/java/org/apache/storm/metricstore/MetricStoreConfig.java +++ b/storm-server/src/main/java/org/apache/storm/metricstore/MetricStoreConfig.java @@ -1,11 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -15,17 +21,18 @@ import org.apache.storm.DaemonConfig; import org.apache.storm.metric.StormMetricsRegistry; - public class MetricStoreConfig { /** * Configures metrics store (running on Nimbus) to use the class specified in the conf. + * * @param conf Storm config map * @param metricsRegistry The Nimbus daemon metrics registry * @return MetricStore prepared store * @throws MetricException on misconfiguration */ - public static MetricStore configure(Map conf, StormMetricsRegistry metricsRegistry) throws MetricException { + public static MetricStore configure(Map conf, + StormMetricsRegistry metricsRegistry) throws MetricException { String storeClass = "None"; try { @@ -37,14 +44,18 @@ public static MetricStore configure(Map conf, StormMetricsRegist String rocksdbSpecificMsg = ""; if (storeClass.contains("rocksdb") && System.getenv("ROCKSDB_SHAREDLIB_DIR") == null) { - rocksdbSpecificMsg = ", missing env var ROCKSDB_SHAREDLIB_DIR required to load JNI library in org.rocksdb.RocksDB class"; + rocksdbSpecificMsg = + ", missing env var ROCKSDB_SHAREDLIB_DIR required to load JNI library in " + + "org.rocksdb.RocksDB class"; } - throw new MetricException("Failed to create metric store using store class " + storeClass + rocksdbSpecificMsg, e); + throw new MetricException("Failed to create metric store using store class " + + storeClass + rocksdbSpecificMsg, e); } } /** * Configures metric processor (running on supervisor) to use the class specified in the conf. + * * @param conf the supervisor config * @return WorkerMetricsProcessor prepared processor * @throws MetricException on misconfiguration @@ -53,7 +64,8 @@ public static WorkerMetricsProcessor configureMetricProcessor(Map conf) throws M try { String processorClass = (String) conf.get(DaemonConfig.STORM_METRIC_PROCESSOR_CLASS); - WorkerMetricsProcessor processor = (WorkerMetricsProcessor) (Class.forName(processorClass)).newInstance(); + WorkerMetricsProcessor processor = (WorkerMetricsProcessor) (Class + .forName(processorClass)).newInstance(); processor.prepare(conf); return processor; } catch (Exception e) { diff --git a/storm-server/src/main/java/org/apache/storm/metricstore/NimbusMetricProcessor.java b/storm-server/src/main/java/org/apache/storm/metricstore/NimbusMetricProcessor.java index 3ce1298a779..4e2bb50816a 100644 --- a/storm-server/src/main/java/org/apache/storm/metricstore/NimbusMetricProcessor.java +++ b/storm-server/src/main/java/org/apache/storm/metricstore/NimbusMetricProcessor.java @@ -1,11 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -22,7 +28,8 @@ */ public class NimbusMetricProcessor implements WorkerMetricsProcessor { @Override - public void processWorkerMetrics(Map conf, WorkerMetrics metrics) throws MetricException { + public void processWorkerMetrics(Map conf, + WorkerMetrics metrics) throws MetricException { try (NimbusClient client = NimbusClient.Builder.withConf(conf).forDaemon().build()) { client.getClient().processWorkerMetrics(metrics); } catch (TException | NimbusLeaderNotFoundException e) { diff --git a/storm-server/src/main/java/org/apache/storm/metricstore/NoOpMetricStore.java b/storm-server/src/main/java/org/apache/storm/metricstore/NoOpMetricStore.java index c9f67c186f1..34e09372e8e 100644 --- a/storm-server/src/main/java/org/apache/storm/metricstore/NoOpMetricStore.java +++ b/storm-server/src/main/java/org/apache/storm/metricstore/NoOpMetricStore.java @@ -1,11 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -21,7 +27,7 @@ public class NoOpMetricStore implements MetricStore { public void prepare(Map config, StormMetricsRegistry metricsRegistry) {} @Override - public void insert(Metric metric) { } + public void insert(Metric metric) {} @Override public boolean populateValue(Metric metric) { @@ -29,10 +35,10 @@ public boolean populateValue(Metric metric) { } @Override - public void close() { } + public void close() {} @Override - public void scan(FilterOptions filter, ScanCallback scanCallback) { } + public void scan(FilterOptions filter, ScanCallback scanCallback) {} } diff --git a/storm-server/src/main/java/org/apache/storm/metricstore/WorkerMetricsProcessor.java b/storm-server/src/main/java/org/apache/storm/metricstore/WorkerMetricsProcessor.java index 5a594735114..42244c8089e 100644 --- a/storm-server/src/main/java/org/apache/storm/metricstore/WorkerMetricsProcessor.java +++ b/storm-server/src/main/java/org/apache/storm/metricstore/WorkerMetricsProcessor.java @@ -1,11 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -18,14 +24,17 @@ public interface WorkerMetricsProcessor { /** * Process insertion of worker metrics. The implementation should be thread-safe. + * * @param conf the supervisor config * @param metrics the metrics to process * @throws MetricException on error */ - void processWorkerMetrics(Map conf, WorkerMetrics metrics) throws MetricException; + void processWorkerMetrics(Map conf, + WorkerMetrics metrics) throws MetricException; /** * Prepares the metric processor. + * * @param config Storm config map * @throws MetricException on error */ diff --git a/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/KeyType.java b/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/KeyType.java index 8f655916459..5803b2f8a0b 100644 --- a/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/KeyType.java +++ b/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/KeyType.java @@ -1,11 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/MetricsCleaner.java b/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/MetricsCleaner.java index 3c324747333..eadc2c1d7cc 100644 --- a/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/MetricsCleaner.java +++ b/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/MetricsCleaner.java @@ -1,11 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -31,7 +37,8 @@ public class MetricsCleaner implements Runnable, AutoCloseable { private long sleepMs = DEFAULT_SLEEP_MS; private long purgeTimestamp = 0L; - MetricsCleaner(RocksDbStore store, int retentionHours, int hourlyPeriod, Meter failureMeter, StormMetricsRegistry metricsRegistry) { + MetricsCleaner(RocksDbStore store, int retentionHours, int hourlyPeriod, Meter failureMeter, + StormMetricsRegistry metricsRegistry) { this.store = store; this.retentionHours = retentionHours; if (hourlyPeriod > 0) { diff --git a/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/ReadOnlyStringMetadataCache.java b/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/ReadOnlyStringMetadataCache.java index 7735550df17..1aeabd38433 100644 --- a/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/ReadOnlyStringMetadataCache.java +++ b/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/ReadOnlyStringMetadataCache.java @@ -1,11 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/RocksDbKey.java b/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/RocksDbKey.java index 5f014f482fd..de5688e5d77 100644 --- a/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/RocksDbKey.java +++ b/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/RocksDbKey.java @@ -1,11 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -22,9 +28,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; - /** - * Class representing the data used as a Key in RocksDB. Keys can be used either for metadata or metrics. + * Class representing the data used as a Key in RocksDB. Keys can be used either for metadata or + * metrics. * *

      Keys are 38 bytes in size. The fields for a key are: *

      <
      @@ -63,7 +69,8 @@ public class RocksDbKey implements Comparable {
            * Constructor for a RocksDB key for a metadata string.
            *
            * @param type  type of metadata string
      -     * @param metadataStringId  the string Id for the string (stored in the topologyId portion of the key)
      +     * @param metadataStringId the string Id for the string (stored in the topologyId portion of the
      +     *     key)
            */
           RocksDbKey(KeyType type, int metadataStringId) {
               byte[] key = new byte[KEY_SIZE];
      @@ -83,7 +90,6 @@ public class RocksDbKey implements Comparable {
               this.key = raw;
           }
       
      -
           /**
            * Get a zeroed key of the specified type.
            *
      @@ -95,7 +101,7 @@ static RocksDbKey getPrefix(KeyType type) {
           }
       
           /**
      -     * gets the first possible key value for the desired key type.
      +     * Gets the first possible key value for the desired key type.
            *
            * @return the initial key
            */
      @@ -104,7 +110,7 @@ static RocksDbKey getInitialKey(KeyType type) {
           }
       
           /**
      -     * gets the key just larger than the last possible key value for the desired key type.
      +     * Gets the key just larger than the last possible key value for the desired key type.
            *
            * @return the last key
            */
      @@ -118,7 +124,8 @@ static RocksDbKey getLastKey(KeyType type) {
            *
            * @return the generated key
            */
      -    static RocksDbKey createMetricKey(AggLevel aggLevel, int topologyId, long metricTimestamp, int metricId,
      +    static RocksDbKey createMetricKey(AggLevel aggLevel, int topologyId, long metricTimestamp,
      +            int metricId,
                                             int componentId, int executorId, int hostId, int port,
                                             int streamId) {
               byte[] raw = new byte[KEY_SIZE];
      @@ -139,7 +146,7 @@ static RocksDbKey createMetricKey(AggLevel aggLevel, int topologyId, long metric
           }
       
           /**
      -     * get the metadata string Id portion of the key for metadata keys.
      +     * Get the metadata string Id portion of the key for metadata keys.
            *
            * @return the metadata string Id
            * @throws RuntimeException  if the key is not a metadata type
      @@ -148,19 +155,20 @@ int getMetadataStringId() {
               if (this.getType().getValue() < KeyType.METADATA_STRING_END.getValue()) {
                   return ByteBuffer.wrap(key, 2, 4).getInt();
               } else {
      -            throw new RuntimeException("Cannot fetch metadata string for key of type " + this.getType());
      +            throw new RuntimeException("Cannot fetch metadata string for key of type " + this
      +                    .getType());
               }
           }
       
           /**
      -     * get the raw key bytes
      +     * Get the raw key bytes.
            */
           byte[] getRaw() {
               return this.key;
           }
       
           /**
      -     * get the type of key.
      +     * Get the type of key.
            *
            * @return the type of key
            */
      @@ -169,7 +177,7 @@ KeyType getType() {
           }
       
           /**
      -     * compares to keys on a byte by byte basis.
      +     * Compares to keys on a byte by byte basis.
            *
            * @return comparison of key byte values
            */
      diff --git a/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/RocksDbMetricsWriter.java b/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/RocksDbMetricsWriter.java
      index 029c51652fd..f2241e5bcbc 100644
      --- a/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/RocksDbMetricsWriter.java
      +++ b/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/RocksDbMetricsWriter.java
      @@ -1,11 +1,17 @@
       /**
      - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements.  See the NOTICE file distributed with
      - * this work for additional information regarding copyright ownership.  The ASF licenses this file to you under the Apache License, Version
      - * 2.0 (the "License"); you may not use this file except in compliance with the License.  You may obtain a copy of the License at
      + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
      + * agreements. See the NOTICE file distributed with
      + * this work for additional information regarding copyright ownership. The ASF licenses this file to
      + * you under the Apache License, Version
      + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
      + * obtain a copy of the License at
        * http://www.apache.org/licenses/LICENSE-2.0
        *
      - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
      - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the License for the specific language governing permissions
      + * 

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -32,12 +38,16 @@ /** *

      - * Class designed to perform all metrics inserts into RocksDB. Metrics are processed from a blocking queue. Inserts - * to RocksDB are done using a single thread to simplify design (such as looking up existing metric data for aggregation, + * Class designed to perform all metrics inserts into RocksDB. Metrics are processed from a blocking + * queue. Inserts + * to RocksDB are done using a single thread to simplify design (such as looking up existing metric + * data for aggregation, * and fetching/evicting metadata from the cache). This class is not thread safe. *

      - * A writable LRU StringMetadataCache is used to minimize looking up metadata string Ids. As entries are added to the full cache, older - * entries are evicted from the cache and need to be written to the database. This happens as the handleEvictedMetadata() + * A writable LRU StringMetadataCache is used to minimize looking up metadata string Ids. As entries + * are added to the full cache, older + * entries are evicted from the cache and need to be written to the database. This happens as the + * handleEvictedMetadata() * method callback. *

      * The following issues would need to be addressed to implement a multithreaded metrics writer: @@ -46,9 +56,12 @@ *
    • Ensuring newly created metadata strings are seen by all threads.
    • *
    • Maintaining a properly cached state of metadata for multiple writers. The current LRU cache * evicts data as new metadata is added.
    • - *
    • Processing the aggregation of a metric requires fetching and updating previous aggregates. A multithreaded - * design would need to ensure two metrics were not updating an aggregated metric at the same time.
    • - *
    • Investigate performance of multiple threads inserting into RocksDB versus a single ordered insert.
    • + *
    • Processing the aggregation of a metric requires fetching and updating previous aggregates. A + * multithreaded + * design would need to ensure two metrics were not updating an aggregated metric at the same + * time.
    • + *
    • Investigate performance of multiple threads inserting into RocksDB versus a single ordered + * insert.
    • * */ public class RocksDbMetricsWriter implements Runnable, AutoCloseable { @@ -57,7 +70,8 @@ public class RocksDbMetricsWriter implements Runnable, AutoCloseable { private BlockingQueue queue; private WritableStringMetadataCache stringMetadataCache; private Set unusedIds = new HashSet<>(); - private TreeMap insertBatch = new TreeMap<>(); // RocksDB should insert in sorted key order + private TreeMap insertBatch = + new TreeMap<>(); // RocksDB should insert in sorted key order private WriteOptions writeOpts = new WriteOptions(); private volatile boolean shutdown = false; private Meter failureMeter; @@ -116,22 +130,31 @@ private void processInsert(Metric metric) throws MetricException { // convert all strings to numeric Ids for the metric key and add to the metadata cache long metricTimestamp = metric.getTimestamp(); - Integer topologyId = storeMetadataString(KeyType.TOPOLOGY_STRING, metric.getTopologyId(), metricTimestamp); - Integer metricId = storeMetadataString(KeyType.METRIC_STRING, metric.getMetricName(), metricTimestamp); - Integer componentId = storeMetadataString(KeyType.COMPONENT_STRING, metric.getComponentId(), metricTimestamp); - Integer executorId = storeMetadataString(KeyType.EXEC_ID_STRING, metric.getExecutorId(), metricTimestamp); - Integer hostId = storeMetadataString(KeyType.HOST_STRING, metric.getHostname(), metricTimestamp); - Integer streamId = storeMetadataString(KeyType.STREAM_ID_STRING, metric.getStreamId(), metricTimestamp); - - RocksDbKey key = RocksDbKey.createMetricKey(AggLevel.AGG_LEVEL_NONE, topologyId, metric.getTimestamp(), metricId, - componentId, executorId, hostId, metric.getPort(), streamId); + Integer topologyId = storeMetadataString(KeyType.TOPOLOGY_STRING, metric.getTopologyId(), + metricTimestamp); + Integer metricId = storeMetadataString(KeyType.METRIC_STRING, metric.getMetricName(), + metricTimestamp); + Integer componentId = storeMetadataString(KeyType.COMPONENT_STRING, metric.getComponentId(), + metricTimestamp); + Integer executorId = storeMetadataString(KeyType.EXEC_ID_STRING, metric.getExecutorId(), + metricTimestamp); + Integer hostId = storeMetadataString(KeyType.HOST_STRING, metric.getHostname(), + metricTimestamp); + Integer streamId = storeMetadataString(KeyType.STREAM_ID_STRING, metric.getStreamId(), + metricTimestamp); + + RocksDbKey key = RocksDbKey.createMetricKey(AggLevel.AGG_LEVEL_NONE, topologyId, metric + .getTimestamp(), metricId, + componentId, executorId, hostId, metric + .getPort(), streamId); // save metric key/value to be batched RocksDbValue value = new RocksDbValue(metric); insertBatch.put(key, value); // Aggregate matching metrics over bucket timeframes. - // We'll process starting with the longest bucket. If the metric for this does not exist, we don't have to + // We'll process starting with the longest bucket. If the metric for this does not exist, we + // don't have to // search for the remaining bucket metrics. ListIterator li = aggBuckets.listIterator(aggBuckets.size()); boolean populate = true; @@ -144,7 +167,8 @@ private void processInsert(Metric metric) throws MetricException { long roundedToBucket = msToBucket * (metric.getTimestamp() / msToBucket); aggMetric.setTimestamp(roundedToBucket); - RocksDbKey aggKey = RocksDbKey.createMetricKey(bucket, topologyId, aggMetric.getTimestamp(), metricId, + RocksDbKey aggKey = RocksDbKey.createMetricKey(bucket, topologyId, aggMetric + .getTimestamp(), metricId, componentId, executorId, hostId, aggMetric.getPort(), streamId); if (populate) { @@ -152,7 +176,8 @@ private void processInsert(Metric metric) throws MetricException { if (store.populateFromKey(aggKey, aggMetric)) { aggMetric.addValue(metric.getValue()); } else { - // aggregating metric did not exist, don't look for further ones with smaller timestamps + // aggregating metric did not exist, don't look for further ones with smaller + // timestamps populate = false; } } @@ -169,7 +194,8 @@ private void processInsert(Metric metric) throws MetricException { // converts a metadata string into a unique integer. Updates the timestamp of the string // so we can track when it was last used for later deletion on database cleanup. - private int storeMetadataString(KeyType type, String s, long metricTimestamp) throws MetricException { + private int storeMetadataString(KeyType type, String s, + long metricTimestamp) throws MetricException { if (s == null) { throw new MetricException("No string for metric metadata string type " + type); } @@ -250,9 +276,11 @@ private void generateUniqueStringIds() throws MetricException { } } - // writes multiple metric values into the database as a batch operation. The tree map keeps the keys sorted + // writes multiple metric values into the database as a batch operation. The tree map keeps the + // keys sorted // for faster insertion to RocksDB. - private void processBatchInsert(TreeMap batchMap) throws MetricException { + private void processBatchInsert(TreeMap batchMap) throws MetricException { try (WriteBatch writeBatch = new WriteBatch()) { // take the batched metric data and write to the database for (RocksDbKey k : batchMap.keySet()) { @@ -267,7 +295,8 @@ private void processBatchInsert(TreeMap batchMap) thro } } - // evicted metadata needs to be stored immediately. Metadata lookups count on it being in the cache + // evicted metadata needs to be stored immediately. Metadata lookups count on it being in the + // cache // or database. void handleEvictedMetadata(RocksDbKey key, RocksDbValue val) { try { @@ -286,13 +315,15 @@ public void close() { this.shutdown = true; // get all metadata from the cache to put into the database - TreeMap batchMap = new TreeMap<>(); // use a new map to prevent threading issues with writer thread + TreeMap batchMap = + new TreeMap<>(); // use a new map to prevent threading issues with writer thread for (Map.Entry entry : stringMetadataCache.entrySet()) { String metadataString = (String) entry.getKey(); StringMetadata val = (StringMetadata) entry.getValue(); RocksDbValue rval = new RocksDbValue(val.getLastTimestamp(), metadataString); - for (KeyType type : val.getMetadataTypes()) { // save the metadata for all types of strings it matches + for (KeyType type : val + .getMetadataTypes()) { // save the metadata for all types of strings it matches RocksDbKey rkey = new RocksDbKey(type, val.getStringId()); batchMap.put(rkey, rval); } diff --git a/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/RocksDbStore.java b/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/RocksDbStore.java index 59355c1ed8d..13d7bce9484 100644 --- a/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/RocksDbStore.java +++ b/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/RocksDbStore.java @@ -1,11 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -39,7 +45,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; - public class RocksDbStore implements MetricStore, AutoCloseable { static final int INVALID_METADATA_STRING_ID = 0; private static final Logger LOG = LoggerFactory.getLogger(RocksDbStore.class); @@ -59,13 +64,15 @@ public class RocksDbStore implements MetricStore, AutoCloseable { * @throws MetricException on preparation error */ @Override - public void prepare(Map config, StormMetricsRegistry metricsRegistry) throws MetricException { + public void prepare(Map config, + StormMetricsRegistry metricsRegistry) throws MetricException { validateConfig(config); this.failureMeter = metricsRegistry.registerMeter("RocksDB:metric-failures"); RocksDB.loadLibrary(); - boolean createIfMissing = ObjectReader.getBoolean(config.get(DaemonConfig.STORM_ROCKSDB_CREATE_IF_MISSING), false); + boolean createIfMissing = ObjectReader.getBoolean(config + .get(DaemonConfig.STORM_ROCKSDB_CREATE_IF_MISSING), false); try (Options options = new Options().setCreateIfMissing(createIfMissing)) { // use the hash index for prefix searches @@ -75,7 +82,8 @@ public void prepare(Map config, StormMetricsRegistry metricsRegi options.useCappedPrefixExtractor(RocksDbKey.KEY_SIZE); String path = getRocksDbAbsoluteDir(config); - LOG.info("Opening RocksDB from {}, {}={}", path, DaemonConfig.STORM_ROCKSDB_CREATE_IF_MISSING, createIfMissing); + LOG.info("Opening RocksDB from {}, {}={}", path, + DaemonConfig.STORM_ROCKSDB_CREATE_IF_MISSING, createIfMissing); db = RocksDB.open(options, path); } catch (RocksDBException e) { String message = "Error opening RockDB database"; @@ -84,17 +92,21 @@ public void prepare(Map config, StormMetricsRegistry metricsRegi } // create thread to delete old metrics and metadata - Integer retentionHours = Integer.parseInt(config.get(DaemonConfig.STORM_ROCKSDB_METRIC_RETENTION_HOURS).toString()); + Integer retentionHours = Integer.parseInt(config + .get(DaemonConfig.STORM_ROCKSDB_METRIC_RETENTION_HOURS).toString()); Integer deletionPeriod = 0; if (config.containsKey(DaemonConfig.STORM_ROCKSDB_METRIC_DELETION_PERIOD_HOURS)) { - deletionPeriod = Integer.parseInt(config.get(DaemonConfig.STORM_ROCKSDB_METRIC_DELETION_PERIOD_HOURS).toString()); + deletionPeriod = Integer.parseInt(config + .get(DaemonConfig.STORM_ROCKSDB_METRIC_DELETION_PERIOD_HOURS).toString()); } - metricsCleaner = new MetricsCleaner(this, retentionHours, deletionPeriod, failureMeter, metricsRegistry); + metricsCleaner = new MetricsCleaner(this, retentionHours, deletionPeriod, failureMeter, + metricsRegistry); // create thread to process insertion of all metrics metricsWriter = new RocksDbMetricsWriter(this, this.queue, this.failureMeter); - int cacheCapacity = Integer.parseInt(config.get(DaemonConfig.STORM_ROCKSDB_METADATA_STRING_CACHE_CAPACITY).toString()); + int cacheCapacity = Integer.parseInt(config + .get(DaemonConfig.STORM_ROCKSDB_METADATA_STRING_CACHE_CAPACITY).toString()); StringMetadataCache.init(metricsWriter, cacheCapacity); readOnlyStringMetadataCache = StringMetadataCache.getReadOnlyStringMetadataCache(); metricsWriter.init(); // init the writer once the cache is setup @@ -110,39 +122,47 @@ public void prepare(Map config, StormMetricsRegistry metricsRegi } /** - * Implements configuration validation of Metrics Store, validates storm configuration for Metrics Store. + * Implements configuration validation of Metrics Store, validates storm configuration for + * Metrics Store. * * @param config Storm config to specify which store type, location of store and creation policy - * @throws MetricException if there is a missing required configuration or if the store does not exist but + * @throws MetricException if there is a missing required configuration or if the store does not + * exist but * the config specifies not to create the store */ private void validateConfig(Map config) throws MetricException { if (!(config.containsKey(DaemonConfig.STORM_ROCKSDB_LOCATION))) { - throw new MetricException("Not a vaild RocksDB configuration - Missing store location " + DaemonConfig.STORM_ROCKSDB_LOCATION); + throw new MetricException("Not a vaild RocksDB configuration - Missing store location " + + DaemonConfig.STORM_ROCKSDB_LOCATION); } if (!(config.containsKey(DaemonConfig.STORM_ROCKSDB_CREATE_IF_MISSING))) { - throw new MetricException("Not a vaild RocksDB configuration - Does not specify creation policy " + throw new MetricException("Not a vaild RocksDB configuration - Does not specify " + + "creation policy " + DaemonConfig.STORM_ROCKSDB_CREATE_IF_MISSING); } // validate path defined String storePath = getRocksDbAbsoluteDir(config); - boolean createIfMissing = ObjectReader.getBoolean(config.get(DaemonConfig.STORM_ROCKSDB_CREATE_IF_MISSING), false); + boolean createIfMissing = ObjectReader.getBoolean(config + .get(DaemonConfig.STORM_ROCKSDB_CREATE_IF_MISSING), false); if (!createIfMissing) { if (!(new File(storePath).exists())) { - throw new MetricException("Configuration specifies not to create a store but no store currently exists at " + storePath); + throw new MetricException("Configuration specifies not to create a store but no " + + "store currently exists at " + storePath); } } if (!(config.containsKey(DaemonConfig.STORM_ROCKSDB_METADATA_STRING_CACHE_CAPACITY))) { - throw new MetricException("Not a valid RocksDB configuration - Missing metadata string cache size " + throw new MetricException("Not a valid RocksDB configuration - Missing metadata " + + "string cache size " + DaemonConfig.STORM_ROCKSDB_METADATA_STRING_CACHE_CAPACITY); } if (!config.containsKey(DaemonConfig.STORM_ROCKSDB_METRIC_RETENTION_HOURS)) { - throw new MetricException("Not a valid RocksDB configuration - Missing metric retention " + throw new MetricException("Not a valid RocksDB configuration - Missing metric " + + "retention " + DaemonConfig.STORM_ROCKSDB_METRIC_RETENTION_HOURS); } } @@ -150,7 +170,8 @@ private void validateConfig(Map config) throws MetricException { private String getRocksDbAbsoluteDir(Map conf) throws MetricException { String storePath = (String) conf.get(DaemonConfig.STORM_ROCKSDB_LOCATION); if (storePath == null) { - throw new MetricException("Not a vaild RocksDB configuration - Missing store location " + DaemonConfig.STORM_ROCKSDB_LOCATION); + throw new MetricException("Not a vaild RocksDB configuration - Missing store location " + + DaemonConfig.STORM_ROCKSDB_LOCATION); } else { if (new File(storePath).isAbsolute()) { return storePath; @@ -200,33 +221,41 @@ public void insert(Metric metric) throws MetricException { public boolean populateValue(Metric metric) throws MetricException { Map localLookupCache = new HashMap<>(6); - int topologyId = lookupMetadataString(KeyType.TOPOLOGY_STRING, metric.getTopologyId(), localLookupCache); + int topologyId = lookupMetadataString(KeyType.TOPOLOGY_STRING, metric.getTopologyId(), + localLookupCache); if (INVALID_METADATA_STRING_ID == topologyId) { return false; } - int metricId = lookupMetadataString(KeyType.METRIC_STRING, metric.getMetricName(), localLookupCache); + int metricId = lookupMetadataString(KeyType.METRIC_STRING, metric.getMetricName(), + localLookupCache); if (INVALID_METADATA_STRING_ID == metricId) { return false; } - int componentId = lookupMetadataString(KeyType.COMPONENT_STRING, metric.getComponentId(), localLookupCache); + int componentId = lookupMetadataString(KeyType.COMPONENT_STRING, metric.getComponentId(), + localLookupCache); if (INVALID_METADATA_STRING_ID == componentId) { return false; } - int executorId = lookupMetadataString(KeyType.EXEC_ID_STRING, metric.getExecutorId(), localLookupCache); + int executorId = lookupMetadataString(KeyType.EXEC_ID_STRING, metric.getExecutorId(), + localLookupCache); if (INVALID_METADATA_STRING_ID == executorId) { return false; } - int hostId = lookupMetadataString(KeyType.HOST_STRING, metric.getHostname(), localLookupCache); + int hostId = lookupMetadataString(KeyType.HOST_STRING, metric.getHostname(), + localLookupCache); if (INVALID_METADATA_STRING_ID == hostId) { return false; } - int streamId = lookupMetadataString(KeyType.STREAM_ID_STRING, metric.getStreamId(), localLookupCache); + int streamId = lookupMetadataString(KeyType.STREAM_ID_STRING, metric.getStreamId(), + localLookupCache); if (INVALID_METADATA_STRING_ID == streamId) { return false; } - RocksDbKey key = RocksDbKey.createMetricKey(metric.getAggLevel(), topologyId, metric.getTimestamp(), metricId, - componentId, executorId, hostId, metric.getPort(), streamId); + RocksDbKey key = RocksDbKey.createMetricKey(metric.getAggLevel(), topologyId, metric + .getTimestamp(), metricId, + componentId, executorId, hostId, metric + .getPort(), streamId); return populateFromKey(key, metric); } @@ -251,9 +280,11 @@ boolean populateFromKey(RocksDbKey key, Metric metric) throws MetricException { return true; } - // attempts to lookup the unique Id for a string that may not exist yet. Returns INVALID_METADATA_STRING_ID + // attempts to lookup the unique Id for a string that may not exist yet. Returns + // INVALID_METADATA_STRING_ID // if it does not exist. - private int lookupMetadataString(KeyType type, String s, Map lookupCache) throws MetricException { + private int lookupMetadataString(KeyType type, String s, Map lookupCache) throws MetricException { if (s == null) { if (this.failureMeter != null) { this.failureMeter.mark(); @@ -283,7 +314,8 @@ private int lookupMetadataString(KeyType type, String s, Map lo if (stringMetadata != null) { id = stringMetadata.getStringId(); - // add to the callers cache. We can't add it to the stringMetadataCache, since that could cause an eviction + // add to the callers cache. We can't add it to the stringMetadataCache, since that + // could cause an eviction // database write, which we want to only occur from the inserting DB thread. lookupCache.put(s, id); @@ -310,8 +342,10 @@ StringMetadata rocksDbGetStringMetadata(KeyType type, String s) throws RocksDBEx return reference.get(); } - // scans from key start to the key before end, calling back until callback indicates not to process further - void scanRange(RocksDbKey start, RocksDbKey end, RocksDbScanCallback fn) throws RocksDBException { + // scans from key start to the key before end, calling back until callback indicates not to + // process further + void scanRange(RocksDbKey start, RocksDbKey end, + RocksDbScanCallback fn) throws RocksDBException { try (ReadOptions ro = new ReadOptions()) { ro.setTotalOrderSeek(true); try (RocksIterator iterator = db.newIterator(ro)) { @@ -361,12 +395,14 @@ public void scan(FilterOptions filter, ScanCallback scanCallback) throws MetricE * @param rawCallback callback for each Metric found * @throws MetricException on error */ - private void scanRaw(FilterOptions filter, RocksDbScanCallback rawCallback) throws MetricException { + private void scanRaw(FilterOptions filter, + RocksDbScanCallback rawCallback) throws MetricException { scanInternal(filter, null, rawCallback); } // perform a scan given filter options, and return results in either Metric or raw data. - private void scanInternal(FilterOptions filter, ScanCallback scanCallback, RocksDbScanCallback rawCallback) throws MetricException { + private void scanInternal(FilterOptions filter, ScanCallback scanCallback, + RocksDbScanCallback rawCallback) throws MetricException { Map stringToIdCache = new HashMap<>(); Map idToStringCache = new HashMap<>(); @@ -375,7 +411,8 @@ private void scanInternal(FilterOptions filter, ScanCallback scanCallback, Rocks int endTopologyId = 0xFFFFFFFF; String filterTopologyId = filter.getTopologyId(); if (filterTopologyId != null) { - int topologyId = lookupMetadataString(KeyType.TOPOLOGY_STRING, filterTopologyId, stringToIdCache); + int topologyId = lookupMetadataString(KeyType.TOPOLOGY_STRING, filterTopologyId, + stringToIdCache); if (INVALID_METADATA_STRING_ID == topologyId) { return; // string does not exist in database } @@ -390,7 +427,8 @@ private void scanInternal(FilterOptions filter, ScanCallback scanCallback, Rocks int endMetricId = 0xFFFFFFFF; String filterMetricName = filter.getMetricName(); if (filterMetricName != null) { - int metricId = lookupMetadataString(KeyType.METRIC_STRING, filterMetricName, stringToIdCache); + int metricId = lookupMetadataString(KeyType.METRIC_STRING, filterMetricName, + stringToIdCache); if (INVALID_METADATA_STRING_ID == metricId) { return; // string does not exist in database } @@ -402,7 +440,8 @@ private void scanInternal(FilterOptions filter, ScanCallback scanCallback, Rocks int endComponentId = 0xFFFFFFFF; String filterComponentId = filter.getComponentId(); if (filterComponentId != null) { - int componentId = lookupMetadataString(KeyType.COMPONENT_STRING, filterComponentId, stringToIdCache); + int componentId = lookupMetadataString(KeyType.COMPONENT_STRING, filterComponentId, + stringToIdCache); if (INVALID_METADATA_STRING_ID == componentId) { return; // string does not exist in database } @@ -414,7 +453,8 @@ private void scanInternal(FilterOptions filter, ScanCallback scanCallback, Rocks int endExecutorId = 0xFFFFFFFF; String filterExecutorName = filter.getExecutorId(); if (filterExecutorName != null) { - int executorId = lookupMetadataString(KeyType.EXEC_ID_STRING, filterExecutorName, stringToIdCache); + int executorId = lookupMetadataString(KeyType.EXEC_ID_STRING, filterExecutorName, + stringToIdCache); if (INVALID_METADATA_STRING_ID == executorId) { return; // string does not exist in database } @@ -446,7 +486,8 @@ private void scanInternal(FilterOptions filter, ScanCallback scanCallback, Rocks int endStreamId = 0xFFFFFFFF; String filterStreamId = filter.getStreamId(); if (filterStreamId != null) { - int streamId = lookupMetadataString(KeyType.HOST_STRING, filterStreamId, stringToIdCache); + int streamId = lookupMetadataString(KeyType.HOST_STRING, filterStreamId, + stringToIdCache); if (INVALID_METADATA_STRING_ID == streamId) { return; // string does not exist in database } @@ -459,9 +500,11 @@ private void scanInternal(FilterOptions filter, ScanCallback scanCallback, Rocks for (AggLevel aggLevel : filter.getAggLevels()) { - RocksDbKey startKey = RocksDbKey.createMetricKey(aggLevel, startTopologyId, startTime, startMetricId, + RocksDbKey startKey = RocksDbKey.createMetricKey(aggLevel, startTopologyId, + startTime, startMetricId, startComponentId, startExecutorId, startHostId, startPort, startStreamId); - RocksDbKey endKey = RocksDbKey.createMetricKey(aggLevel, endTopologyId, endTime, endMetricId, + RocksDbKey endKey = RocksDbKey.createMetricKey(aggLevel, endTopologyId, endTime, + endMetricId, endComponentId, endExecutorId, endHostId, endPort, endStreamId); try (RocksIterator iterator = db.newIterator(ro)) { @@ -510,14 +553,21 @@ private void scanInternal(FilterOptions filter, ScanCallback scanCallback, Rocks if (scanCallback != null) { try { // populate a metric - String metricName = metadataIdToString(KeyType.METRIC_STRING, key.getMetricId(), idToStringCache); - String topologyId = metadataIdToString(KeyType.TOPOLOGY_STRING, key.getTopologyId(), idToStringCache); - String componentId = metadataIdToString(KeyType.COMPONENT_STRING, key.getComponentId(), idToStringCache); - String executorId = metadataIdToString(KeyType.EXEC_ID_STRING, key.getExecutorId(), idToStringCache); - String hostname = metadataIdToString(KeyType.HOST_STRING, key.getHostnameId(), idToStringCache); - String streamId = metadataIdToString(KeyType.STREAM_ID_STRING, key.getStreamId(), idToStringCache); - - Metric metric = new Metric(metricName, timestamp, topologyId, 0.0, componentId, executorId, hostname, + String metricName = metadataIdToString(KeyType.METRIC_STRING, key + .getMetricId(), idToStringCache); + String topologyId = metadataIdToString(KeyType.TOPOLOGY_STRING, key + .getTopologyId(), idToStringCache); + String componentId = metadataIdToString(KeyType.COMPONENT_STRING, + key.getComponentId(), idToStringCache); + String executorId = metadataIdToString(KeyType.EXEC_ID_STRING, key + .getExecutorId(), idToStringCache); + String hostname = metadataIdToString(KeyType.HOST_STRING, key + .getHostnameId(), idToStringCache); + String streamId = metadataIdToString(KeyType.STREAM_ID_STRING, key + .getStreamId(), idToStringCache); + + Metric metric = new Metric(metricName, timestamp, topologyId, 0.0, + componentId, executorId, hostname, streamId, key.getPort(), aggLevel); val.populateMetric(metric); @@ -542,9 +592,11 @@ private void scanInternal(FilterOptions filter, ScanCallback scanCallback, Rocks } } - // Finds the metadata string that matches the string Id and type provided. The string should exist, as it is + // Finds the metadata string that matches the string Id and type provided. The string should + // exist, as it is // referenced from a metric. - private String metadataIdToString(KeyType type, int id, Map lookupCache) throws MetricException { + private String metadataIdToString(KeyType type, int id, Map lookupCache) throws MetricException { String s = readOnlyStringMetadataCache.getMetadataString(id); if (s != null) { return s; @@ -558,7 +610,8 @@ private String metadataIdToString(KeyType type, int id, Map loo try { byte[] value = db.get(key.getRaw()); if (value == null) { - throw new MetricException("Failed to find metadata string for id " + id + " of type " + type); + throw new MetricException("Failed to find metadata string for id " + id + + " of type " + type); } RocksDbValue rdbValue = new RocksDbValue(value); s = rdbValue.getMetdataString(); @@ -604,7 +657,8 @@ void deleteMetadataBefore(long firstValidTimestamp) throws MetricException { if (this.failureMeter != null) { this.failureMeter.mark(); } - throw new MetricException("Invalid timestamp for deleting metadata: " + firstValidTimestamp); + throw new MetricException("Invalid timestamp for deleting metadata: " + + firstValidTimestamp); } try (WriteBatch writeBatch = new WriteBatch(); @@ -644,7 +698,8 @@ void deleteMetadataBefore(long firstValidTimestamp) throws MetricException { } interface RocksDbScanCallback { - boolean cb(RocksDbKey key, RocksDbValue val) throws RocksDBException; // return false to stop scan + boolean cb(RocksDbKey key, + RocksDbValue val) throws RocksDBException; // return false to stop scan } } diff --git a/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/RocksDbValue.java b/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/RocksDbValue.java index 0da7b923b73..b028743e81b 100644 --- a/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/RocksDbValue.java +++ b/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/RocksDbValue.java @@ -1,11 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -14,9 +20,9 @@ import java.nio.ByteBuffer; import org.apache.storm.metricstore.Metric; - /** - * Class representing the data used as a Value in RocksDB. Values can be used either for metadata or metrics. + * Class representing the data used as a Value in RocksDB. Values can be used either for metadata or + * metrics. * *

      Formats for Metadata String values are: * @@ -24,7 +30,8 @@ * Field Size Offset * * Version 1 0 The current metadata version - allows migrating if the format changes in the future - * Timestamp 8 1 The time when the metadata was last used by a metric. Allows deleting of old metadata. + * Timestamp 8 1 The time when the metadata was last used by a metric. Allows deleting of old + * metadata. * Metadata String any 9 The metadata string *

      * @@ -104,7 +111,8 @@ String getMetdataString() { * Gets StringMetadata associated with the key/value pair. */ StringMetadata getStringMetadata(RocksDbKey key) { - return new StringMetadata(key.getType(), key.getMetadataStringId(), this.getLastTimestamp()); + return new StringMetadata(key.getType(), key.getMetadataStringId(), this + .getLastTimestamp()); } /** @@ -115,14 +123,14 @@ long getLastTimestamp() { } /** - * get the raw value bytes + * Get the raw value bytes. */ byte[] getRaw() { return this.value; } /** - * populate metric values from the raw data. + * Populate metric values from the raw data. */ void populateMetric(Metric metric) { ByteBuffer bb = ByteBuffer.wrap(this.value, 0, METRIC_VALUE_SIZE); diff --git a/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/StringMetadata.java b/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/StringMetadata.java index d8905823ee9..2e2c4a1f2c4 100644 --- a/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/StringMetadata.java +++ b/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/StringMetadata.java @@ -1,11 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -15,10 +21,12 @@ import java.util.List; /** - * Class that contains the information associated with a metadata string that remains cached in memory. + * Class that contains the information associated with a metadata string that remains cached in + * memory. */ class StringMetadata { - private List types = new ArrayList<>(1); // its possible a string is used by multiple types of metadata strings + private List types = + new ArrayList<>(1); // its possible a string is used by multiple types of metadata strings private int stringId; private long lastTimestamp; @@ -54,7 +62,8 @@ private void addKeyType(KeyType type) { } /** - * Updates the timestamp of when a metadata string was last used. Adds the type of the string if it is a new + * Updates the timestamp of when a metadata string was last used. Adds the type of the string if + * it is a new * type. * * @param metricTimestamp the timestamp of the metric using the metadata string diff --git a/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/StringMetadataCache.java b/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/StringMetadataCache.java index 5ee50a750b4..1327d7bfbe1 100644 --- a/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/StringMetadataCache.java +++ b/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/StringMetadataCache.java @@ -1,11 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -21,13 +27,18 @@ import org.slf4j.LoggerFactory; /** - * Class to create a use a cache that stores Metadata string information in memory. It allows searching for a - * Metadata string's unique id, or looking up the string by the unique id. The StringMetadata is stored in an - * LRU map. When an entry is added to the cache, an older entry may be evicted, which then needs to be + * Class to create a use a cache that stores Metadata string information in memory. It allows + * searching for a + * Metadata string's unique id, or looking up the string by the unique id. The StringMetadata is + * stored in an + * LRU map. When an entry is added to the cache, an older entry may be evicted, which then needs to + * be * immediately stored to the database to provide a consistent view of all the metadata strings. * - *

      All write operations adding metadata to RocksDB are done by a single thread (a RocksDbMetricsWriter), - * but multiple threads can read values from the cache. To clarify which permissions are accessible by various + *

      All write operations adding metadata to RocksDB are done by a single thread (a + * RocksDbMetricsWriter), + * but multiple threads can read values from the cache. To clarify which permissions are accessible + * by various * threads, the ReadOnlyStringMetadataCache and WritableStringMetadataCache are provided to be used. */ @@ -109,18 +120,22 @@ public StringMetadata get(String s) { /** * Add the string metadata to the cache. * - *

      NOTE: this can cause data to be evicted from the cache when full. When this occurs, the evictionCallback() method + *

      NOTE: this can cause data to be evicted from the cache when full. When this occurs, the + * evictionCallback() method * is called to store the metadata back into the RocksDB database. * *

      This method is only exposed to the WritableStringMetadataCache interface. * * @param s The string to add * @param stringMetadata The string's metadata - * @param newEntry Indicates the metadata is being used for the first time and should be written to RocksDB immediately - * @throws MetricException when evicted data fails to save to the database or when the database is shutdown + * @param newEntry Indicates the metadata is being used for the first time and should be written + * to RocksDB immediately + * @throws MetricException when evicted data fails to save to the database or when the database + * is shutdown */ @Override - public void put(String s, StringMetadata stringMetadata, boolean newEntry) throws MetricException { + public void put(String s, StringMetadata stringMetadata, + boolean newEntry) throws MetricException { if (dbWriter.isShutdown()) { // another thread could be writing out the metadata cache to the database. throw new MetricException("Shutting down"); @@ -159,7 +174,8 @@ private void writeMetadataToDisk(String key, StringMetadata val) { // save the evicted key/value to the database immediately RocksDbValue rval = new RocksDbValue(val.getLastTimestamp(), key); - for (KeyType type : val.getMetadataTypes()) { // save the metadata for all types of strings it matches + for (KeyType type : val + .getMetadataTypes()) { // save the metadata for all types of strings it matches RocksDbKey rkey = new RocksDbKey(type, val.getStringId()); dbWriter.handleEvictedMetadata(rkey, rval); } diff --git a/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/WritableStringMetadataCache.java b/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/WritableStringMetadataCache.java index d31fe9f393d..707c99f3ec5 100644 --- a/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/WritableStringMetadataCache.java +++ b/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/WritableStringMetadataCache.java @@ -1,11 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -16,7 +22,8 @@ import org.apache.storm.metricstore.MetricException; /** - * The writable interface to a StringMetadataCache intended to be used by a single RocksDBMetricwWriter instance. + * The writable interface to a StringMetadataCache intended to be used by a single + * RocksDBMetricwWriter instance. * This class is not thread safe. */ public interface WritableStringMetadataCache extends ReadOnlyStringMetadataCache { @@ -24,15 +31,18 @@ public interface WritableStringMetadataCache extends ReadOnlyStringMetadataCache /** * Add the string metadata to the cache. * - *

      NOTE: this can cause data to be evicted from the cache when full. When this occurs, the evictionCallback() method + *

      NOTE: this can cause data to be evicted from the cache when full. When this occurs, the + * evictionCallback() method * is called to store the metadata back into the RocksDB database. * *

      This method is only exposed to the WritableStringMetadataCache interface. * * @param s The string to add * @param stringMetadata The string's metadata - * @param newEntry Indicates the metadata is being used for the first time and should be written to RocksDB immediately - * @throws MetricException when evicted data fails to save to the database or when the database is shutdown + * @param newEntry Indicates the metadata is being used for the first time and should be written + * to RocksDB immediately + * @throws MetricException when evicted data fails to save to the database or when the database + * is shutdown */ void put(String s, StringMetadata stringMetadata, boolean newEntry) throws MetricException; diff --git a/storm-server/src/main/java/org/apache/storm/nimbus/AssignmentDistributionService.java b/storm-server/src/main/java/org/apache/storm/nimbus/AssignmentDistributionService.java index ce11722f950..d01fc2bc225 100644 --- a/storm-server/src/main/java/org/apache/storm/nimbus/AssignmentDistributionService.java +++ b/storm-server/src/main/java/org/apache/storm/nimbus/AssignmentDistributionService.java @@ -1,12 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

      * http://www.apache.org/licenses/LICENSE-2.0 *

      - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -21,7 +26,6 @@ import java.util.concurrent.Executors; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; - import org.apache.storm.Constants; import org.apache.storm.DaemonConfig; import org.apache.storm.daemon.supervisor.Supervisor; @@ -36,12 +40,14 @@ import org.slf4j.LoggerFactory; /** - * A service for distributing master assignments to supervisors, this service makes the assignments notification + * A service for distributing master assignments to supervisors, this service makes the assignments + * notification * asynchronous. * *

      We support multiple working threads to distribute assignment, every thread has a queue buffer. * - *

      Master will shuffle its node request to the queues, if the target queue is full, we just discard the request, + *

      Master will shuffle its node request to the queues, if the target queue is full, we just + * discard the request, * let the supervisors sync instead. * *

      Caution: this class is not thread safe. @@ -82,7 +88,7 @@ public class AssignmentDistributionService implements Closeable { private volatile Map> assignmentsQueue; /** - * local supervisors for local cluster assignments distribution. + * Local supervisors for local cluster assignments distribution. */ private Map localSupervisors; @@ -93,11 +99,13 @@ public class AssignmentDistributionService implements Closeable { /** * Factory method for initialize a instance. + * * @param conf config. * @param callback callback for sendAssignment results * @return an instance of {@link AssignmentDistributionService} */ - public static AssignmentDistributionService getInstance(Map conf, INodeAssignmentSentCallBack callback) { + public static AssignmentDistributionService getInstance(Map conf, + INodeAssignmentSentCallBack callback) { AssignmentDistributionService service = new AssignmentDistributionService(); service.prepare(conf, callback); return service; @@ -113,17 +121,19 @@ public void prepare(Map conf, INodeAssignmentSentCallBack callBack) { this.sendAssignmentCallback = callBack; this.random = new Random(47); - this.threadsNum = ObjectReader.getInt(conf.get(DaemonConfig.NIMBUS_ASSIGNMENTS_SERVICE_THREADS), 10); - this.queueSize = ObjectReader.getInt(conf.get(DaemonConfig.NIMBUS_ASSIGNMENTS_SERVICE_THREAD_QUEUE_SIZE), 100); + this.threadsNum = ObjectReader.getInt(conf + .get(DaemonConfig.NIMBUS_ASSIGNMENTS_SERVICE_THREADS), 10); + this.queueSize = ObjectReader.getInt(conf + .get(DaemonConfig.NIMBUS_ASSIGNMENTS_SERVICE_THREAD_QUEUE_SIZE), 100); this.assignmentsQueue = new HashMap<>(); for (int i = 0; i < threadsNum; i++) { this.assignmentsQueue.put(i, new LinkedBlockingQueue(queueSize)); } - //start the thread pool + // start the thread pool this.service = Executors.newFixedThreadPool(threadsNum); this.active = true; - //start the threads + // start the threads for (int i = 0; i < threadsNum; i++) { this.service.submit(new DistributeTask(this, i)); } @@ -148,25 +158,29 @@ public void close() throws IOException { /** * Add an assignments for a node/supervisor for distribution. + * * @param node node id of supervisor. * @param host host name for the node. * @param serverPort node thrift server port. * @param assignments the {@link org.apache.storm.generated.SupervisorAssignments} */ - public void addAssignmentsForNode(String node, String host, Integer serverPort, SupervisorAssignments assignments, + public void addAssignmentsForNode(String node, String host, Integer serverPort, + SupervisorAssignments assignments, StormMetricsRegistry metricsRegistry) { try { - //For some reasons, we can not get supervisor port info, eg: supervisor shutdown, - //Just skip for this scheduling round. + // For some reasons, we can not get supervisor port info, eg: supervisor shutdown, + // Just skip for this scheduling round. if (serverPort == null) { - LOG.warn("Discard an assignment distribution for node {} because server port info is missing.", node); + LOG.warn("Discard an assignment distribution for node {} because server port info " + + "is missing.", node); return; } boolean success = nextQueue().offer(NodeAssignments.getInstance(node, host, serverPort, assignments, metricsRegistry), 5L, TimeUnit.SECONDS); if (!success) { - LOG.warn("Discard an assignment distribution for node {} because the target sub queue is full.", node); + LOG.warn("Discard an assignment distribution for node {} because the target sub " + + "queue is full.", node); } } catch (InterruptedException e) { @@ -193,6 +207,7 @@ private LinkedBlockingQueue getQueueById(Integer queueIndex) { /** * Get an assignments from the target queue with the specific index. + * * @param queueIndex index of the queue * @return an {@link NodeAssignments} */ @@ -222,7 +237,8 @@ static class NodeAssignments { private SupervisorAssignments assignments; private StormMetricsRegistry metricsRegistry; - private NodeAssignments(String node, String host, Integer serverPort, SupervisorAssignments assignments, + private NodeAssignments(String node, String host, Integer serverPort, + SupervisorAssignments assignments, StormMetricsRegistry metricsRegistry) { this.node = node; this.host = host; @@ -236,7 +252,7 @@ public static NodeAssignments getInstance(String node, String host, Integer serv return new NodeAssignments(node, host, serverPort, assignments, metricsRegistry); } - //supervisor assignment id/supervisor id + // supervisor assignment id/supervisor id public String getNode() { return this.node; } @@ -278,7 +294,8 @@ public void run() { sendAssignmentsToNode(nodeAssignments); } catch (InterruptedException e) { if (service.isActive()) { - LOG.error("Get an unexpected interrupt when distributing assignments to node, {}", e.getCause()); + LOG.error("Get an unexpected interrupt when distributing assignments to " + + "node, {}", e.getCause()); } else { // service is off now just interrupt it. Thread.currentThread().interrupt(); @@ -289,31 +306,39 @@ public void run() { private void sendAssignmentsToNode(NodeAssignments assignments) { if (this.service.isLocalMode) { - //local node + // local node Supervisor supervisor = this.service.localSupervisors.get(assignments.getNode()); if (supervisor != null) { supervisor.sendSupervisorAssignments(assignments.getAssignments()); service.sendAssignmentCallback.nodeAssignmentSent(assignments.getNode(), true); } else { - LOG.error("Can not find node {} for assignments distribution", assignments.getNode()); + LOG.error("Can not find node {} for assignments distribution", assignments + .getNode()); service.sendAssignmentCallback.nodeAssignmentSent(assignments.getNode(), false); - throw new RuntimeException("null for node " + assignments.getNode() + " supervisor instance."); + throw new RuntimeException("null for node " + assignments.getNode() + + " supervisor instance."); } } else { // distributed mode try (SupervisorClient client = SupervisorClient.Builder.withConf(service.getConf()) - .withHostName(assignments.getHost()).withPort(assignments.getServerPort()).createSupervisorClient()) { + .withHostName(assignments.getHost()).withPort(assignments + .getServerPort()).createSupervisorClient()) { try { client.getIface().sendSupervisorAssignments(assignments.getAssignments()); - service.sendAssignmentCallback.nodeAssignmentSent(assignments.getNode(), true); + service.sendAssignmentCallback.nodeAssignmentSent(assignments.getNode(), + true); } catch (Exception e) { - assignments.getMetricsRegistry().getMeter(Constants.NIMBUS_SEND_ASSIGNMENT_EXCEPTIONS).mark(); - LOG.error("Exception when trying to send assignments to node {}", assignments.getNode(), e); - service.sendAssignmentCallback.nodeAssignmentSent(assignments.getNode(), false); + assignments.getMetricsRegistry() + .getMeter(Constants.NIMBUS_SEND_ASSIGNMENT_EXCEPTIONS).mark(); + LOG.error("Exception when trying to send assignments to node {}", + assignments.getNode(), e); + service.sendAssignmentCallback.nodeAssignmentSent(assignments.getNode(), + false); } } catch (Throwable e) { - //just ignore any error/exception. - LOG.error("Exception to create supervisor client for node {}", assignments.getNode(), e); + // just ignore any error/exception. + LOG.error("Exception to create supervisor client for node {}", assignments + .getNode(), e); } } } diff --git a/storm-server/src/main/java/org/apache/storm/nimbus/DefaultTopologyValidator.java b/storm-server/src/main/java/org/apache/storm/nimbus/DefaultTopologyValidator.java index c6bb208ede9..4d0eba73e7c 100644 --- a/storm-server/src/main/java/org/apache/storm/nimbus/DefaultTopologyValidator.java +++ b/storm-server/src/main/java/org/apache/storm/nimbus/DefaultTopologyValidator.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -28,19 +34,23 @@ public void prepare(Map stormConf) { } @Override - public void validate(String topologyName, Map topologyConf, StormTopology topology) throws InvalidTopologyException { + public void validate(String topologyName, Map topologyConf, + StormTopology topology) throws InvalidTopologyException { if (topologyName.contains(".")) { - LOG.warn("Metrics for topology name '{}' will be reported as '{}'.", topologyName, topologyName.replace('.', '_')); + LOG.warn("Metrics for topology name '{}' will be reported as '{}'.", topologyName, + topologyName.replace('.', '_')); } Map spouts = topology.get_spouts(); for (String spoutName : spouts.keySet()) { if (spoutName.contains(".")) { - LOG.warn("Metrics for spout name '{}' will be reported as '{}'.", spoutName, spoutName.replace('.', '_')); + LOG.warn("Metrics for spout name '{}' will be reported as '{}'.", spoutName, + spoutName.replace('.', '_')); } SpoutSpec spoutSpec = spouts.get(spoutName); for (String streamName : spoutSpec.get_common().get_streams().keySet()) { if (streamName.contains(".")) { - LOG.warn("Metrics for stream name '{}' will be reported as '{}'.", streamName, streamName.replace('.', '_')); + LOG.warn("Metrics for stream name '{}' will be reported as '{}'.", streamName, + streamName.replace('.', '_')); } } } @@ -48,12 +58,14 @@ public void validate(String topologyName, Map topologyConf, StormTopology topolo Map bolts = topology.get_bolts(); for (String boltName : bolts.keySet()) { if (boltName.contains(".")) { - LOG.warn("Metrics for bolt name '{}' will be reported as '{}'.", boltName, boltName.replace('.', '_')); + LOG.warn("Metrics for bolt name '{}' will be reported as '{}'.", boltName, boltName + .replace('.', '_')); } Bolt bolt = bolts.get(boltName); for (String streamName : bolt.get_common().get_streams().keySet()) { if (streamName.contains(".")) { - LOG.warn("Metrics for stream name '{}' will be reported as '{}'.", streamName, streamName.replace('.', '_')); + LOG.warn("Metrics for stream name '{}' will be reported as '{}'.", streamName, + streamName.replace('.', '_')); } } } diff --git a/storm-server/src/main/java/org/apache/storm/nimbus/ITopologyActionNotifierPlugin.java b/storm-server/src/main/java/org/apache/storm/nimbus/ITopologyActionNotifierPlugin.java index 1b5a6fb030a..ca0ef1b08d7 100644 --- a/storm-server/src/main/java/org/apache/storm/nimbus/ITopologyActionNotifierPlugin.java +++ b/storm-server/src/main/java/org/apache/storm/nimbus/ITopologyActionNotifierPlugin.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-server/src/main/java/org/apache/storm/nimbus/ITopologyValidator.java b/storm-server/src/main/java/org/apache/storm/nimbus/ITopologyValidator.java index ff17192c3df..46bc7619fd8 100644 --- a/storm-server/src/main/java/org/apache/storm/nimbus/ITopologyValidator.java +++ b/storm-server/src/main/java/org/apache/storm/nimbus/ITopologyValidator.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-server/src/main/java/org/apache/storm/nimbus/IWorkerHeartbeatsRecoveryStrategy.java b/storm-server/src/main/java/org/apache/storm/nimbus/IWorkerHeartbeatsRecoveryStrategy.java index 8457de8c0be..e8a0b1581a7 100644 --- a/storm-server/src/main/java/org/apache/storm/nimbus/IWorkerHeartbeatsRecoveryStrategy.java +++ b/storm-server/src/main/java/org/apache/storm/nimbus/IWorkerHeartbeatsRecoveryStrategy.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -22,19 +28,22 @@ public interface IWorkerHeartbeatsRecoveryStrategy { /** * Function to prepare the strategy. + * * @param conf config */ void prepare(Map conf); /** * Function to decide if the heartbeats is ready. + * * @param nodeIds all the node ids from current physical plan[assignments], read from {@code ClusterState} * @return true if all node worker heartbeats reported */ boolean isReady(Set nodeIds); /** - * report the node id to this strategy to help to decide {@code isReady}. + * Report the node id to this strategy to help to decide {@code isReady}. + * * @param nodeId the node id from reported SupervisorWorkerHeartbeats */ void reportNodeId(String nodeId); diff --git a/storm-server/src/main/java/org/apache/storm/nimbus/LeaderListenerCallback.java b/storm-server/src/main/java/org/apache/storm/nimbus/LeaderListenerCallback.java index 7255581f1c3..e9cf23aac0c 100644 --- a/storm-server/src/main/java/org/apache/storm/nimbus/LeaderListenerCallback.java +++ b/storm-server/src/main/java/org/apache/storm/nimbus/LeaderListenerCallback.java @@ -1,19 +1,23 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

      * http://www.apache.org/licenses/LICENSE-2.0 *

      - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.nimbus; import com.codahale.metrics.Meter; - import java.io.IOException; import java.util.HashSet; import java.util.List; @@ -21,7 +25,6 @@ import java.util.Set; import java.util.TreeSet; import javax.security.auth.Subject; - import org.apache.commons.io.IOUtils; import org.apache.storm.Config; import org.apache.storm.DaemonConfig; @@ -79,7 +82,8 @@ public class LeaderListenerCallback { * @param clusterState IStormClusterState * @param acls zookeeper acls */ - public LeaderListenerCallback(Map conf, CuratorFramework zk, BlobStore blobStore, ILeaderElector leaderElector, + public LeaderListenerCallback(Map conf, CuratorFramework zk, BlobStore blobStore, + ILeaderElector leaderElector, TopoCache tc, IStormClusterState clusterState, List acls, StormMetricsRegistry metricsRegistry) { this.blobStore = blobStore; this.tc = tc; @@ -90,9 +94,10 @@ public LeaderListenerCallback(Map conf, CuratorFramework zk, BlobStore blobStore this.acls = acls; this.numGainedLeader = metricsRegistry.registerMeter("nimbus:num-gained-leadership"); this.numLostLeader = metricsRegistry.registerMeter("nimbus:num-lost-leadership"); - //Since we only give up leadership if we're waiting for blobs to sync, - //it makes sense to wait a full sync cycle before trying for leadership again. - this.requeueDelayMs = ObjectReader.getInt(conf.get(DaemonConfig.NIMBUS_CODE_SYNC_FREQ_SECS)) * 1000; + // Since we only give up leadership if we're waiting for blobs to sync, + // it makes sense to wait a full sync cycle before trying for leadership again. + this.requeueDelayMs = ObjectReader.getInt(conf + .get(DaemonConfig.NIMBUS_CODE_SYNC_FREQ_SECS)) * 1000; } /** @@ -100,9 +105,9 @@ public LeaderListenerCallback(Map conf, CuratorFramework zk, BlobStore blobStore */ public void leaderCallBack(Object lock) { numGainedLeader.mark(); - //set up nimbus-info to zk + // set up nimbus-info to zk setUpNimbusInfo(acls); - //sync zk assignments/id-info to local + // sync zk assignments/id-info to local LOG.info("Locking sync remote assignments and id-info to local"); synchronized (lock) { clusterState.syncRemoteAssignments(null); @@ -118,25 +123,31 @@ public void leaderCallBack(Object lock) { Set allLocalTopologyBlobKeys = filterTopologyBlobKeys(allLocalBlobKeys); // this finds all active topologies blob keys from all local topology blob keys - Sets.SetView diffTopology = Sets.difference(activeTopologyBlobKeys, allLocalTopologyBlobKeys); + Sets.SetView diffTopology = Sets.difference(activeTopologyBlobKeys, + allLocalTopologyBlobKeys); LOG.info("active-topology-blobs [{}] local-topology-blobs [{}] diff-topology-blobs [{}]", generateJoinedString(activeTopologyIds), generateJoinedString(allLocalTopologyBlobKeys), generateJoinedString(diffTopology)); if (diffTopology.isEmpty()) { - Set activeTopologyDependencies = getTopologyDependencyKeys(activeTopologyCodeKeys); + Set activeTopologyDependencies = + getTopologyDependencyKeys(activeTopologyCodeKeys); // this finds all dependency blob keys from active topologies from all local blob keys - Sets.SetView diffDependencies = Sets.difference(activeTopologyDependencies, allLocalBlobKeys); - LOG.info("active-topology-dependencies [{}] local-blobs [{}] diff-topology-dependencies [{}]", + Sets.SetView diffDependencies = Sets.difference(activeTopologyDependencies, + allLocalBlobKeys); + LOG.info("active-topology-dependencies [{}] local-blobs [{}] " + + "diff-topology-dependencies [{}]", generateJoinedString(activeTopologyDependencies), generateJoinedString(allLocalBlobKeys), generateJoinedString(diffDependencies)); if (diffDependencies.isEmpty()) { - LOG.info("Accepting leadership, all active topologies and corresponding dependencies found locally."); + LOG.info("Accepting leadership, all active topologies and corresponding " + + "dependencies found locally."); tc.clear(); } else { - LOG.info("Code for all active topologies is available locally, but some dependencies are not found locally, " + LOG.info("Code for all active topologies is available locally, but some " + + "dependencies are not found locally, " + "giving up leadership."); surrenderLeadership(); } @@ -155,12 +166,14 @@ public void notLeaderCallback() { } private void setUpNimbusInfo(List acls) { - String leaderInfoPath = conf.get(Config.STORM_ZOOKEEPER_ROOT) + ClusterUtils.LEADERINFO_SUBTREE; + String leaderInfoPath = conf.get(Config.STORM_ZOOKEEPER_ROOT) + + ClusterUtils.LEADERINFO_SUBTREE; NimbusInfo nimbusInfo = NimbusInfo.fromConf(conf); if (ClientZookeeper.existsNode(zk, leaderInfoPath, false)) { ClientZookeeper.setData(zk, leaderInfoPath, Utils.javaSerialize(nimbusInfo)); } else { - ClientZookeeper.createNode(zk, leaderInfoPath, Utils.javaSerialize(nimbusInfo), CreateMode.PERSISTENT, acls); + ClientZookeeper.createNode(zk, leaderInfoPath, Utils.javaSerialize(nimbusInfo), + CreateMode.PERSISTENT, acls); } } @@ -206,7 +219,8 @@ private Set getTopologyDependencyKeys(Set activeTopologyCodeKeys for (String activeTopologyCodeKey : activeTopologyCodeKeys) { try (InputStreamWithMeta blob = blobStore.getBlob(activeTopologyCodeKey, subject)) { - byte[] blobContent = IOUtils.readFully(blob, new Long(blob.getFileLength()).intValue()); + byte[] blobContent = IOUtils.readFully(blob, new Long(blob.getFileLength()) + .intValue()); StormTopology stormCode = Utils.deserialize(blobContent, StormTopology.class); if (stormCode.is_set_dependency_jars()) { activeTopologyDependencies.addAll(stormCode.get_dependency_jars()); diff --git a/storm-server/src/main/java/org/apache/storm/nimbus/NimbusHeartbeatsPressureTest.java b/storm-server/src/main/java/org/apache/storm/nimbus/NimbusHeartbeatsPressureTest.java index 4aecdd5cb07..d0006284b46 100644 --- a/storm-server/src/main/java/org/apache/storm/nimbus/NimbusHeartbeatsPressureTest.java +++ b/storm-server/src/main/java/org/apache/storm/nimbus/NimbusHeartbeatsPressureTest.java @@ -1,12 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

      * http://www.apache.org/licenses/LICENSE-2.0 *

      - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -35,7 +40,7 @@ */ public class NimbusHeartbeatsPressureTest { /** - * the args below can be configured. + * The args below can be configured. */ private static String NIMBUS_HOST = "localhost"; private static int NIMBUS_PORT = 6627; @@ -55,6 +60,7 @@ public class NimbusHeartbeatsPressureTest { /** * Initialize a fake config. + * * @return conf */ private static Config initializedConfig() { @@ -181,7 +187,8 @@ static class HeartbeatSendTask implements Runnable { this.tryTimes = tryTimes; this.runtimesBook = new double[tryTimes]; try { - client = NimbusClient.Builder.withConf(initializedConfig()).buildWithNimbusHostPort(NIMBUS_HOST, NIMBUS_PORT); + client = NimbusClient.Builder.withConf(initializedConfig()) + .buildWithNimbusHostPort(NIMBUS_HOST, NIMBUS_PORT); } catch (TTransportException e) { e.printStackTrace(); } diff --git a/storm-server/src/main/java/org/apache/storm/nimbus/StrictTopologyValidator.java b/storm-server/src/main/java/org/apache/storm/nimbus/StrictTopologyValidator.java index c48ae871eb0..66d77586d7f 100644 --- a/storm-server/src/main/java/org/apache/storm/nimbus/StrictTopologyValidator.java +++ b/storm-server/src/main/java/org/apache/storm/nimbus/StrictTopologyValidator.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -29,19 +35,23 @@ public void prepare(Map stormConf) { } @Override - public void validate(String topologyName, Map topologyConf, StormTopology topology) throws InvalidTopologyException { + public void validate(String topologyName, Map topologyConf, + StormTopology topology) throws InvalidTopologyException { if (topologyName.contains(".")) { - throw new WrappedInvalidTopologyException(String.format("Topology name '%s' contains illegal character '.'", topologyName)); + throw new WrappedInvalidTopologyException(String + .format("Topology name '%s' contains illegal character '.'", topologyName)); } Map spouts = topology.get_spouts(); for (String spoutName : spouts.keySet()) { if (spoutName.contains(".")) { - throw new WrappedInvalidTopologyException(String.format("Spout name '%s' contains illegal character '.'", spoutName)); + throw new WrappedInvalidTopologyException(String + .format("Spout name '%s' contains illegal character '.'", spoutName)); } SpoutSpec spoutSpec = spouts.get(spoutName); for (String streamName : spoutSpec.get_common().get_streams().keySet()) { if (streamName.contains(".")) { - throw new WrappedInvalidTopologyException(String.format("Stream name '%s' contains illegal character '.'", streamName)); + throw new WrappedInvalidTopologyException(String + .format("Stream name '%s' contains illegal character '.'", streamName)); } } } @@ -49,12 +59,14 @@ public void validate(String topologyName, Map topologyConf, StormTopology topolo Map bolts = topology.get_bolts(); for (String boltName : bolts.keySet()) { if (boltName.contains(".")) { - throw new WrappedInvalidTopologyException(String.format("Bolt name '%s' contains illegal character '.'", boltName)); + throw new WrappedInvalidTopologyException(String + .format("Bolt name '%s' contains illegal character '.'", boltName)); } Bolt bolt = bolts.get(boltName); for (String streamName : bolt.get_common().get_streams().keySet()) { if (streamName.contains(".")) { - throw new WrappedInvalidTopologyException(String.format("Stream name '%s' contains illegal character '.'", streamName)); + throw new WrappedInvalidTopologyException(String + .format("Stream name '%s' contains illegal character '.'", streamName)); } } } diff --git a/storm-server/src/main/java/org/apache/storm/nimbus/TimeOutWorkerHeartbeatsRecoveryStrategy.java b/storm-server/src/main/java/org/apache/storm/nimbus/TimeOutWorkerHeartbeatsRecoveryStrategy.java index 1a8414036ba..61fa7e5e386 100644 --- a/storm-server/src/main/java/org/apache/storm/nimbus/TimeOutWorkerHeartbeatsRecoveryStrategy.java +++ b/storm-server/src/main/java/org/apache/storm/nimbus/TimeOutWorkerHeartbeatsRecoveryStrategy.java @@ -1,12 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

      * http://www.apache.org/licenses/LICENSE-2.0 *

      - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -25,17 +30,22 @@ import org.slf4j.LoggerFactory; /** - * Wait for a node to report worker heartbeats until a configured timeout. For cases below we have strategies: + * Wait for a node to report worker heartbeats until a configured timeout. For cases below we have + * strategies: * - *

      1: When nimbus gains leader ship, it will decide if the heartbeats are ready based on the reported node ids, - * supervisors/nodes will take care of the worker heartbeats recovery, a reported node id means all the workers + *

      1: When nimbus gains leader ship, it will decide if the heartbeats are ready based on the + * reported node ids, + * supervisors/nodes will take care of the worker heartbeats recovery, a reported node id means all + * the workers * heartbeats on the node are reported. * - *

      2: If several supervisor also crush and will never recover[or all crush for some unknown reason], + *

      2: If several supervisor also crush and will never recover[or all crush for some unknown + * reason], * workers will report their heartbeats directly to master, so it has not any effect. */ public class TimeOutWorkerHeartbeatsRecoveryStrategy implements IWorkerHeartbeatsRecoveryStrategy { - private static final Logger LOG = LoggerFactory.getLogger(TimeOutWorkerHeartbeatsRecoveryStrategy.class); + private static final Logger LOG = LoggerFactory + .getLogger(TimeOutWorkerHeartbeatsRecoveryStrategy.class); private static int NODE_MAX_TIMEOUT_SECS = 600; @@ -45,7 +55,8 @@ public class TimeOutWorkerHeartbeatsRecoveryStrategy implements IWorkerHeartbeat @Override public void prepare(Map conf) { - NODE_MAX_TIMEOUT_SECS = ObjectReader.getInt(conf.get(Config.SUPERVISOR_WORKER_HEARTBEATS_MAX_TIMEOUT_SECS), 600); + NODE_MAX_TIMEOUT_SECS = ObjectReader.getInt(conf + .get(Config.SUPERVISOR_WORKER_HEARTBEATS_MAX_TIMEOUT_SECS), 600); this.startTimeSecs = new AtomicLong(0L); this.reportedIds = new HashSet<>(); } @@ -54,8 +65,10 @@ public void prepare(Map conf) { public boolean isReady(Set nodeIds) { startTimeSecs.compareAndSet(0L, Time.currentTimeMillis() / 1000L); if (exceedsMaxTimeOut()) { - Set tmp = nodeIds.stream().filter(id -> !this.reportedIds.contains(id)).collect(toSet()); - LOG.warn("Failed to recover heartbeats for nodes: {} with timeout {}s", tmp, NODE_MAX_TIMEOUT_SECS); + Set tmp = nodeIds.stream().filter(id -> !this.reportedIds.contains(id)) + .collect(toSet()); + LOG.warn("Failed to recover heartbeats for nodes: {} with timeout {}s", tmp, + NODE_MAX_TIMEOUT_SECS); return true; } @@ -68,7 +81,8 @@ public void reportNodeId(String nodeId) { } private boolean exceedsMaxTimeOut() { - return (Time.currentTimeMillis() / 1000L - this.startTimeSecs.get()) > NODE_MAX_TIMEOUT_SECS; + return (Time.currentTimeMillis() / 1000L - this.startTimeSecs + .get()) > NODE_MAX_TIMEOUT_SECS; } } diff --git a/storm-server/src/main/java/org/apache/storm/nimbus/WorkerHeartbeatsRecoveryStrategyFactory.java b/storm-server/src/main/java/org/apache/storm/nimbus/WorkerHeartbeatsRecoveryStrategyFactory.java index b2dc777adbf..46dbacefcf1 100644 --- a/storm-server/src/main/java/org/apache/storm/nimbus/WorkerHeartbeatsRecoveryStrategyFactory.java +++ b/storm-server/src/main/java/org/apache/storm/nimbus/WorkerHeartbeatsRecoveryStrategyFactory.java @@ -1,12 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

      * http://www.apache.org/licenses/LICENSE-2.0 *

      - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -24,6 +29,7 @@ public class WorkerHeartbeatsRecoveryStrategyFactory { /** * Get instance of {@link IWorkerHeartbeatsRecoveryStrategy} with conf. + * * @param conf strategy config * @return an instance of {@link IWorkerHeartbeatsRecoveryStrategy} */ diff --git a/storm-server/src/main/java/org/apache/storm/pacemaker/IServerMessageHandler.java b/storm-server/src/main/java/org/apache/storm/pacemaker/IServerMessageHandler.java index dac6d444257..436c5517314 100644 --- a/storm-server/src/main/java/org/apache/storm/pacemaker/IServerMessageHandler.java +++ b/storm-server/src/main/java/org/apache/storm/pacemaker/IServerMessageHandler.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -17,8 +23,10 @@ /** * Handles heartbeat requests received by a Pacemaker server. * - * @deprecated Pacemaker is deprecated and only kept for backward compatibility; it will be removed in a future release. - * Use the default heartbeat path instead: workers heartbeat to their supervisor, which reports them to Nimbus over + * @deprecated Pacemaker is deprecated and only kept for backward compatibility; it will be removed + * in a future release. + * Use the default heartbeat path instead: workers heartbeat to their supervisor, which reports + * them to Nimbus over * Thrift, with the default ZooKeeper-based cluster state store ({@code org.apache.storm.cluster.ZKStateStorageFactory}). */ @Deprecated diff --git a/storm-server/src/main/java/org/apache/storm/pacemaker/Pacemaker.java b/storm-server/src/main/java/org/apache/storm/pacemaker/Pacemaker.java index a7809c407e2..db356dd7164 100644 --- a/storm-server/src/main/java/org/apache/storm/pacemaker/Pacemaker.java +++ b/storm-server/src/main/java/org/apache/storm/pacemaker/Pacemaker.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -36,8 +42,10 @@ /** * The Pacemaker daemon, an in-memory store for worker heartbeats. * - * @deprecated Pacemaker is deprecated and only kept for backward compatibility; it will be removed in a future release. - * Use the default heartbeat path instead: workers heartbeat to their supervisor, which reports them to Nimbus over + * @deprecated Pacemaker is deprecated and only kept for backward compatibility; it will be removed + * in a future release. + * Use the default heartbeat path instead: workers heartbeat to their supervisor, which reports + * them to Nimbus over * Thrift, with the default ZooKeeper-based cluster state store ({@code org.apache.storm.cluster.ZKStateStorageFactory}). */ @Deprecated @@ -59,13 +67,15 @@ public Pacemaker(Map conf, StormMetricsRegistry metricsRegistry) this.meterTotalReceivedSize = metricsRegistry.registerMeter("pacemaker:total-receive-size"); this.meterGetPulseCount = metricsRegistry.registerMeter("pacemaker:get-pulse=count"); this.meterTotalSentSize = metricsRegistry.registerMeter("pacemaker:total-sent-size"); - this.histogramHeartbeatSize = metricsRegistry.registerHistogram("pacemaker:heartbeat-size", new ExponentiallyDecayingReservoir()); + this.histogramHeartbeatSize = metricsRegistry.registerHistogram("pacemaker:heartbeat-size", + new ExponentiallyDecayingReservoir()); metricsRegistry.registerGauge("pacemaker:size-total-keys", heartbeats::size); } public static void main(String[] args) { SysOutOverSLF4J.sendSystemOutAndErrToSLF4J(); - Map conf = ConfigUtils.overrideLoginConfigWithSystemProperty(ConfigUtils.readStormConfig()); + Map conf = ConfigUtils.overrideLoginConfigWithSystemProperty(ConfigUtils + .readStormConfig()); StormMetricsRegistry metricsRegistry = new StormMetricsRegistry(); final Pacemaker serverHandler = new Pacemaker(conf, metricsRegistry); serverHandler.launchServer(); @@ -121,7 +131,8 @@ private HBMessage pathExists(String path, boolean authenticated) { if (authenticated) { boolean itDoes = heartbeats.containsKey(path); LOG.debug("Checking if path [ {} ] exists... {} .", path, itDoes); - response = new HBMessage(HBServerMessageType.EXISTS_RESPONSE, HBMessageData.boolval(itDoes)); + response = new HBMessage(HBServerMessageType.EXISTS_RESPONSE, HBMessageData + .boolval(itDoes)); } else { response = notAuthorized(); } @@ -169,7 +180,8 @@ private HBMessage getAllNodesForPath(String path, boolean authenticated) { } } HBMessageData hbMessageData = HBMessageData.nodes(new HBNodes(new ArrayList(pulseIds))); - return new HBMessage(HBServerMessageType.GET_ALL_NODES_FOR_PATH_RESPONSE, hbMessageData); + return new HBMessage(HBServerMessageType.GET_ALL_NODES_FOR_PATH_RESPONSE, + hbMessageData); } else { return notAuthorized(); } @@ -186,7 +198,8 @@ private HBMessage getPulse(String path, boolean authenticated) { HBPulse hbPulse = new HBPulse(); hbPulse.set_id(path); hbPulse.set_details(details); - return new HBMessage(HBServerMessageType.GET_PULSE_RESPONSE, HBMessageData.pulse(hbPulse)); + return new HBMessage(HBServerMessageType.GET_PULSE_RESPONSE, HBMessageData + .pulse(hbPulse)); } else { return notAuthorized(); } diff --git a/storm-server/src/main/java/org/apache/storm/pacemaker/PacemakerServer.java b/storm-server/src/main/java/org/apache/storm/pacemaker/PacemakerServer.java index 123f77aac1b..a8e80453126 100644 --- a/storm-server/src/main/java/org/apache/storm/pacemaker/PacemakerServer.java +++ b/storm-server/src/main/java/org/apache/storm/pacemaker/PacemakerServer.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -41,8 +47,10 @@ /** * Netty server of the Pacemaker daemon. * - * @deprecated Pacemaker is deprecated and only kept for backward compatibility; it will be removed in a future release. - * Use the default heartbeat path instead: workers heartbeat to their supervisor, which reports them to Nimbus over + * @deprecated Pacemaker is deprecated and only kept for backward compatibility; it will be removed + * in a future release. + * Use the default heartbeat path instead: workers heartbeat to their supervisor, which reports + * them to Nimbus over * Thrift, with the default ZooKeeper-based cluster state store ({@code org.apache.storm.cluster.ZKStateStorageFactory}). */ @Deprecated @@ -55,8 +63,10 @@ class PacemakerServer implements ISaslServer { private final IServerMessageHandler handler; private String secret; private final String topologyName; - private volatile ChannelGroup allChannels = new DefaultChannelGroup("storm-server", GlobalEventExecutor.INSTANCE); - private final ChannelGroup authenticatedChannels = new DefaultChannelGroup("authenticated-pacemaker-channels", + private volatile ChannelGroup allChannels = new DefaultChannelGroup("storm-server", + GlobalEventExecutor.INSTANCE); + private final ChannelGroup authenticatedChannels = + new DefaultChannelGroup("authenticated-pacemaker-channels", GlobalEventExecutor.INSTANCE); private final ThriftNettyServerCodec.AuthMethod authMethod; private final EventLoopGroup bossEventLoopGroup; @@ -72,10 +82,12 @@ class PacemakerServer implements ISaslServer { case "DIGEST": authMethod = ThriftNettyServerCodec.AuthMethod.DIGEST; - this.secret = ClientAuthUtils.makeDigestPayload(config, ClientAuthUtils.LOGIN_CONTEXT_PACEMAKER_DIGEST); + this.secret = ClientAuthUtils.makeDigestPayload(config, + ClientAuthUtils.LOGIN_CONTEXT_PACEMAKER_DIGEST); if (this.secret == null) { LOG.error("Can't start pacemaker server without digest secret."); - throw new RuntimeException("Can't start pacemaker server without digest secret."); + throw new RuntimeException("Can't start pacemaker server without digest " + + "secret."); } break; @@ -89,7 +101,8 @@ class PacemakerServer implements ISaslServer { default: LOG.error("Can't start pacemaker server without proper PACEMAKER_AUTH_METHOD."); - throw new RuntimeException("Can't start pacemaker server without proper PACEMAKER_AUTH_METHOD."); + throw new RuntimeException("Can't start pacemaker server without proper " + + "PACEMAKER_AUTH_METHOD."); } ThreadFactory bossFactory = new NettyRenameThreadFactory("server-boss"); @@ -98,9 +111,11 @@ class PacemakerServer implements ISaslServer { // 0 means DEFAULT_EVENT_LOOP_THREADS int maxWorkers = (int) config.get(DaemonConfig.PACEMAKER_MAX_THREADS); // https://github.com/netty/netty/blob/netty-4.1.24.Final/transport/src/main/java/io/netty/channel/MultithreadEventLoopGroup.java#L40 - this.workerEventLoopGroup = new NioEventLoopGroup(maxWorkers > 0 ? maxWorkers : 0, workerFactory); + this.workerEventLoopGroup = new NioEventLoopGroup(maxWorkers > 0 ? maxWorkers : 0, + workerFactory); - LOG.info("Create Netty Server " + name() + ", buffer_size: " + FIVE_MB_IN_BYTES + ", maxWorkers: " + maxWorkers); + LOG.info("Create Netty Server " + name() + ", buffer_size: " + FIVE_MB_IN_BYTES + + ", maxWorkers: " + maxWorkers); int thriftMessageMaxSize = (Integer) config.get(Config.PACEMAKER_THRIFT_MESSAGE_SIZE_MAX); ServerBootstrap bootstrap = new ServerBootstrap() @@ -109,9 +124,11 @@ class PacemakerServer implements ISaslServer { .childOption(ChannelOption.TCP_NODELAY, true) .childOption(ChannelOption.SO_SNDBUF, FIVE_MB_IN_BYTES) .childOption(ChannelOption.SO_KEEPALIVE, true) - .childOption(ChannelOption.WRITE_BUFFER_WATER_MARK, new WriteBufferWaterMark(8 * 1024, 32 * 1024)) + .childOption(ChannelOption.WRITE_BUFFER_WATER_MARK, new WriteBufferWaterMark(8 * 1024, + 32 * 1024)) .childOption(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT) - .childHandler(new ThriftNettyServerCodec(this, config, authMethod, thriftMessageMaxSize)); + .childHandler(new ThriftNettyServerCodec(this, config, authMethod, + thriftMessageMaxSize)); try { ChannelFuture channelFuture = bootstrap.bind(new InetSocketAddress(port)).sync(); @@ -149,7 +166,8 @@ public void received(Object mesg, String remote, Channel channel) throws Interru } cleanPipeline(channel); - boolean authenticated = (authMethod == ThriftNettyServerCodec.AuthMethod.NONE) || authenticatedChannels.contains(channel); + boolean authenticated = (authMethod == ThriftNettyServerCodec.AuthMethod.NONE) + || authenticatedChannels.contains(channel); HBMessage m = (HBMessage) mesg; LOG.debug("received message. Passing to handler. {} : {} : {}", handler.toString(), m.toString(), channel.toString()); @@ -163,7 +181,8 @@ public void received(Object mesg, String remote, Channel channel) throws Interru } /** - * Close all channels and stop the event loops of this server. The Pacemaker daemon itself runs until the JVM exits, + * Close all channels and stop the event loops of this server. The Pacemaker daemon itself runs + * until the JVM exits, * so this exists for tests to shut a server down deterministically. */ void close() { diff --git a/storm-server/src/main/java/org/apache/storm/pacemaker/codec/PacemakerServerHandler.java b/storm-server/src/main/java/org/apache/storm/pacemaker/codec/PacemakerServerHandler.java index 7bd8b1a82e9..0cb24486f67 100644 --- a/storm-server/src/main/java/org/apache/storm/pacemaker/codec/PacemakerServerHandler.java +++ b/storm-server/src/main/java/org/apache/storm/pacemaker/codec/PacemakerServerHandler.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -19,12 +25,16 @@ import org.slf4j.LoggerFactory; /** - * Pacemaker server handler. A failure while handling a request only affects the connection it arrived on: the - * connection is closed and the Pacemaker server keeps serving its other clients. Errors are still handled by + * Pacemaker server handler. A failure while handling a request only affects the connection it + * arrived on: the + * connection is closed and the Pacemaker server keeps serving its other clients. Errors are still + * handled by * {@link StormServerHandler}. * - * @deprecated Pacemaker is deprecated and only kept for backward compatibility; it will be removed in a future release. - * Use the default heartbeat path instead: workers heartbeat to their supervisor, which reports them to Nimbus over + * @deprecated Pacemaker is deprecated and only kept for backward compatibility; it will be removed + * in a future release. + * Use the default heartbeat path instead: workers heartbeat to their supervisor, which reports + * them to Nimbus over * Thrift, with the default ZooKeeper-based cluster state store ({@code org.apache.storm.cluster.ZKStateStorageFactory}). */ @Deprecated @@ -42,7 +52,8 @@ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { return; } try { - LOG.warn("Closing connection {} after failing to handle its request", ctx.channel(), cause); + LOG.warn("Closing connection {} after failing to handle its request", ctx.channel(), + cause); } finally { ctx.close(); } diff --git a/storm-server/src/main/java/org/apache/storm/pacemaker/codec/ThriftNettyServerCodec.java b/storm-server/src/main/java/org/apache/storm/pacemaker/codec/ThriftNettyServerCodec.java index 2b93372f6e6..a83418ae6b6 100644 --- a/storm-server/src/main/java/org/apache/storm/pacemaker/codec/ThriftNettyServerCodec.java +++ b/storm-server/src/main/java/org/apache/storm/pacemaker/codec/ThriftNettyServerCodec.java @@ -1,13 +1,19 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -31,8 +37,10 @@ /** * Builds the Pacemaker server pipeline. * - * @deprecated Pacemaker is deprecated and only kept for backward compatibility; it will be removed in a future release. - * Use the default heartbeat path instead: workers heartbeat to their supervisor, which reports them to Nimbus over + * @deprecated Pacemaker is deprecated and only kept for backward compatibility; it will be removed + * in a future release. + * Use the default heartbeat path instead: workers heartbeat to their supervisor, which reports + * them to Nimbus over * Thrift, with the default ZooKeeper-based cluster state store ({@code org.apache.storm.cluster.ZKStateStorageFactory}). */ @Deprecated @@ -74,7 +82,8 @@ protected void initChannel(Channel ch) throws Exception { LOG.debug("Adding KerberosSaslServerHandler to pacemaker server pipeline."); ArrayList authorizedUsers = new ArrayList<>(1); authorizedUsers.add((String) topoConf.get(DaemonConfig.NIMBUS_DAEMON_USER)); - pipeline.addLast(KERBEROS_HANDLER, new KerberosSaslServerHandler((ISaslServer) server, + pipeline.addLast(KERBEROS_HANDLER, + new KerberosSaslServerHandler((ISaslServer) server, topoConf, ClientAuthUtils.LOGIN_CONTEXT_PACEMAKER_SERVER, authorizedUsers)); diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/Cluster.java b/storm-server/src/main/java/org/apache/storm/scheduler/Cluster.java index c01b6fbd808..bb3aee97fa1 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/Cluster.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/Cluster.java @@ -24,8 +24,8 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Map.Entry; +import java.util.Map; import java.util.Set; import java.util.stream.Collectors; import org.apache.storm.Config; @@ -68,7 +68,7 @@ public class Cluster implements ISchedulingState { */ private final Map assignments = new HashMap<>(); /** - * key topologyId, Value: scheduler's status. + * Key topologyId, Value: scheduler's status. */ private final Map status = new HashMap<>(); /** @@ -80,7 +80,8 @@ public class Cluster implements ISchedulingState { private final Map> nodeToScheduledResourcesCache; private final Map> nodeToScheduledOffHeapNodeMemoryCache; // node -> topologyId -> double private final Map> nodeToUsedSlotsCache; - private final Map totalResourcesPerNodeCache = new HashMap<>(); + private final Map totalResourcesPerNodeCache = + new HashMap<>(); /** * Snapshot of cluster total resources (cpu, memory, generic). */ @@ -112,7 +113,8 @@ public Cluster( Map assignments, Topologies topologies, Map conf) { - this(nimbus, resourceMetrics, supervisors, assignments, topologies, conf, null, null, null, null, + this(nimbus, resourceMetrics, supervisors, assignments, topologies, conf, null, null, null, + null, Double.NaN, Double.NaN, null); } @@ -190,13 +192,16 @@ private Cluster( } this.conf = conf; this.topologies = topologies; - this.minWorkerCpu = ObjectReader.getDouble(conf.get(DaemonConfig.STORM_WORKER_MIN_CPU_PCORE_PERCENT), 0.0); + this.minWorkerCpu = ObjectReader.getDouble(conf + .get(DaemonConfig.STORM_WORKER_MIN_CPU_PCORE_PERCENT), 0.0); this.totalCpuResource = Double.isNaN(totalCpuResource) ? computeClusterCpuResource() : totalCpuResource; - this.totalMemoryResource = Double.isNaN(totalMemoryResource) ? computeClusterMemoryResource() : + this.totalMemoryResource = Double.isNaN(totalMemoryResource) + ? computeClusterMemoryResource() : totalMemoryResource; - this.totalGenericResources = totalGenericResources == null ? computeClusterGenericResources() : + this.totalGenericResources = totalGenericResources == null + ? computeClusterGenericResources() : totalGenericResources; ArrayList supervisorHostNames = new ArrayList<>(); @@ -204,9 +209,9 @@ private Cluster( supervisorHostNames.add(s.getHost()); } - //Initialize the network topography + // Initialize the network topography if (networkTopography == null || networkTopography.isEmpty()) { - //Initialize the network topography + // Initialize the network topography String clazz = (String) conf.get(Config.STORM_NETWORK_TOPOGRAPHY_PLUGIN); if (clazz == null || clazz.isEmpty()) { clazz = DefaultRackDNSToSwitchMapping.class.getName(); @@ -217,7 +222,8 @@ private Cluster( for (Map.Entry entry : resolvedSuperVisors.entrySet()) { String hostName = entry.getKey(); String rack = entry.getValue(); - List nodesForRack = this.networkTopography.computeIfAbsent(rack, k -> new ArrayList<>()); + List nodesForRack = this.networkTopography.computeIfAbsent(rack, + k -> new ArrayList<>()); nodesForRack.add(hostName); } } else { @@ -293,12 +299,13 @@ public static double getAssignedMemoryForSlot(final Map topConf) } /** - * Check if the given topology is allowed for modification right now. If not throw an IllegalArgumentException else go on. + * Check if the given topology is allowed for modification right now. If not throw an + * IllegalArgumentException else go on. * * @param topologyId the id of the topology to check */ protected void assertValidTopologyForModification(String topologyId) { - //NOOP + // NOOP } @Override @@ -318,7 +325,7 @@ public Set getBlacklistedHosts() { */ public void setBlacklistedHosts(Set hosts) { if (hosts == blackListedHosts) { - //NOOP + // NOOP return; } blackListedHosts.clear(); @@ -360,16 +367,22 @@ public List needsSchedulingTopologies() { public boolean needsScheduling(TopologyDetails topology) { int desiredNumWorkers = topology.getNumWorkers(); int assignedNumWorkers = this.getAssignedNumWorkers(topology); - return desiredNumWorkers > assignedNumWorkers || getUnassignedExecutors(topology).size() > 0; + return desiredNumWorkers > assignedNumWorkers || getUnassignedExecutors(topology) + .size() > 0; } /** * Returns true when {@code supervisor} is a stable, non-blacklisted supervisor whose slots are all currently free -- - * i.e. a returning idle supervisor the {@link EvenScheduler} idle-rebalance pass may relocate workers onto. The check - * is binary by design -- a supervisor either has zero used slots or it does not -- so the rebalance never fires for an - * "almost balanced" cluster. Stability is gated by {@link #hasMinimumIdleSupervisorStability(SupervisorDetails)} so a - * supervisor that has only just returned (and may still be flapping) is held back until it has been up long enough. The - * opt-in {@link DaemonConfig#NIMBUS_EVEN_REBALANCE_ON_IDLE_SUPERVISOR_ENABLED} flag is checked once by the caller + * i.e. a returning idle supervisor the {@link EvenScheduler} idle-rebalance pass may relocate + * workers onto. The check + * is binary by design -- a supervisor either has zero used slots or it does not -- so the + * rebalance never fires for an + * "almost balanced" cluster. Stability is gated by {@link + * #hasMinimumIdleSupervisorStability(SupervisorDetails)} so a + * supervisor that has only just returned (and may still be flapping) is held back until it has + * been up long enough. The + * opt-in {@link DaemonConfig#NIMBUS_EVEN_REBALANCE_ON_IDLE_SUPERVISOR_ENABLED} flag is checked + * once by the caller * ({@link EvenScheduler#redistributeOntoIdleSupervisors(Topologies, Cluster)}), not here. */ public boolean isIdleSupervisorAvailableForEvenRebalance(SupervisorDetails supervisor) { @@ -394,7 +407,8 @@ private boolean hasMinimumIdleSupervisorStability(SupervisorDetails supervisor) if (minStableRounds <= 0) { return true; } - int monitorFrequencySecs = ObjectReader.getInt(conf.get(DaemonConfig.SUPERVISOR_MONITOR_FREQUENCY_SECS), 3); + int monitorFrequencySecs = ObjectReader.getInt(conf + .get(DaemonConfig.SUPERVISOR_MONITOR_FREQUENCY_SECS), 3); long requiredUptimeSecs = (long) minStableRounds * Math.max(1, monitorFrequencySecs); return supervisor.getUptimeSecs() >= requiredUptimeSecs; } @@ -466,7 +480,8 @@ public Set getAssignablePorts(SupervisorDetails supervisor) { public List getNonBlacklistedAvailableSlots(List blacklistedSupervisorIds) { List slots = new ArrayList<>(); for (SupervisorDetails supervisor : this.supervisors.values()) { - if (!isBlackListed(supervisor.getId()) && !blacklistedSupervisorIds.contains(supervisor.getId())) { + if (!isBlackListed(supervisor.getId()) && !blacklistedSupervisorIds.contains(supervisor + .getId())) { slots.addAll(getAvailableSlots(supervisor)); } } @@ -552,7 +567,8 @@ public int getAssignedNumWorkers(TopologyDetails topology) { public NormalizedResourceOffer getAvailableResources(SupervisorDetails sd) { NormalizedResourceOffer ret = new NormalizedResourceOffer(sd.getTotalResources()); for (SchedulerAssignment assignment : assignments.values()) { - for (Entry entry : assignment.getScheduledResources().entrySet()) { + for (Entry entry : assignment.getScheduledResources() + .entrySet()) { if (sd.getId().equals(entry.getKey().getNodeId())) { ret.remove(entry.getValue(), getResourceMetrics()); } @@ -561,7 +577,8 @@ public NormalizedResourceOffer getAvailableResources(SupervisorDetails sd) { return ret; } - private void addResource(Map resourceMap, String resourceName, Double valueToBeAdded) { + private void addResource(Map resourceMap, String resourceName, + Double valueToBeAdded) { if (!resourceMap.containsKey(resourceName)) { resourceMap.put(resourceName, 0.0); } @@ -593,7 +610,8 @@ private WorkerResources calculateWorkerResources( Constants.COMMON_ONHEAP_MEMORY_RESOURCE_NAME, shared.get_on_heap() ); } - sharedTotalResources = NormalizedResources.RESOURCE_NAME_NORMALIZER.normalizedResourceMap(sharedTotalResources); + sharedTotalResources = NormalizedResources.RESOURCE_NAME_NORMALIZER + .normalizedResourceMap(sharedTotalResources); Map totalResourcesMap = totalResources.toNormalizedMap(); Double cpu = totalResources.getTotalCpu(); @@ -741,7 +759,8 @@ public void assign(WorkerSlot slot, String topologyId, Collection executorsOnNode = new HashSet<>(); - topoIdToNodeIdToSlotIdToExecutors.computeIfAbsent(td.getId(), Cluster::makeMap).computeIfAbsent(nodeId, Cluster::makeMap) + topoIdToNodeIdToSlotIdToExecutors.computeIfAbsent(td.getId(), Cluster::makeMap) + .computeIfAbsent(nodeId, Cluster::makeMap) .forEach((k, v) -> executorsOnNode.addAll(v)); if (extra != null) { executorsOnNode.add(extra); } - //Now check for overlap on the node + // Now check for overlap on the node double memorySharedWithinNode = 0.0; for (SharedMemory shared : td.getSharedMemoryRequests(executorsOnNode)) { memorySharedWithinNode += shared.get_off_heap_node(); @@ -847,22 +869,24 @@ public void freeSlot(WorkerSlot slot) { final String topologyId = assignment.getTopologyId(); assertValidTopologyForModification(topologyId); assignment.unassignBySlot(slot); - topoIdToNodeIdToSlotIdToExecutors.computeIfAbsent(topologyId, Cluster::makeMap).computeIfAbsent(nodeId, Cluster::makeMap) + topoIdToNodeIdToSlotIdToExecutors.computeIfAbsent(topologyId, Cluster::makeMap) + .computeIfAbsent(nodeId, Cluster::makeMap) .computeIfAbsent(slot.getId(), Cluster::makeSet) .clear(); TopologyDetails td = topologies.getById(topologyId); assignment.setTotalSharedOffHeapNodeMemory( nodeId, calculateSharedOffHeapNodeMemory(nodeId, td)); - nodeToScheduledResourcesCache.computeIfAbsent(nodeId, Cluster::makeMap).put(slot, new NormalizedResourceRequest()); + nodeToScheduledResourcesCache.computeIfAbsent(nodeId, Cluster::makeMap).put(slot, + new NormalizedResourceRequest()); nodeToUsedSlotsCache.computeIfAbsent(nodeId, Cluster::makeSet).remove(slot); } } - //Invalidate the cache as something on the node changed + // Invalidate the cache as something on the node changed totalResourcesPerNodeCache.remove(nodeId); } /** - * free the slots. + * Free the slots. * * @param slots multiple slots to free */ @@ -876,7 +900,8 @@ public void freeSlots(Collection slots) { @Override public boolean isSlotOccupied(WorkerSlot slot) { - return nodeToUsedSlotsCache.computeIfAbsent(slot.getNodeId(), Cluster::makeSet).contains(slot); + return nodeToUsedSlotsCache.computeIfAbsent(slot.getNodeId(), Cluster::makeSet) + .contains(slot); } @Override @@ -904,7 +929,8 @@ public SupervisorDetails getSupervisorById(String nodeId) { @Override public Collection getUsedSlots() { - return nodeToUsedSlotsCache.values().stream().flatMap(Set::stream).collect(Collectors.toSet()); + return nodeToUsedSlotsCache.values().stream().flatMap(Set::stream).collect(Collectors + .toSet()); } @Override @@ -932,7 +958,7 @@ public Map getAssignments() { public void setAssignments( Map newAssignments, boolean ignoreSingleExceptions) { if (newAssignments == assignments) { - //NOOP + // NOOP return; } for (SchedulerAssignment assignment : newAssignments.values()) { @@ -961,7 +987,8 @@ public NormalizedResourceOffer getNonBlacklistedClusterAvailableResources(Collec for (SupervisorDetails sup : supervisors.values()) { if (!isBlackListed(sup.getId()) && !blacklistedSupervisorIds.contains(sup.getId())) { available.add(sup.getTotalResources()); - available.remove(getAllScheduledResourcesForNode(sup.getId()), getResourceMetrics()); + available.remove(getAllScheduledResourcesForNode(sup.getId()), + getResourceMetrics()); } } return available; @@ -984,7 +1011,6 @@ public double getClusterTotalMemoryResource() { return this.totalMemoryResource; } - private double computeClusterMemoryResource() { return supervisors.values().stream() .mapToDouble(SupervisorDetails::getTotalMemory) @@ -1015,14 +1041,14 @@ public void setNetworkTopography(Map> networkTopography) { } /** - * set scheduler status for a topology. + * Set scheduler status for a topology. */ public void setStatus(TopologyDetails td, String statusMessage) { setStatus(td.getId(), statusMessage); } /** - * set scheduler status for a topology. + * Set scheduler status for a topology. */ public void setStatus(String topologyId, String statusMessage) { assertValidTopologyForModification(topologyId); @@ -1041,11 +1067,11 @@ public Map getStatusMap() { } /** - * set scheduler status map. + * Set scheduler status map. */ public void setStatusMap(Map statusMap) { if (statusMap == this.status) { - return; //This is a NOOP + return; // This is a NOOP } for (String topologyId : statusMap.keySet()) { assertValidTopologyForModification(topologyId); @@ -1076,7 +1102,8 @@ public Map getTopologyResourcesMap() { public Map getSupervisorsResourcesMap() { Map ret = new HashMap<>(); for (SupervisorDetails sd : supervisors.values()) { - ret.put(sd.getId(), new SupervisorResources(sd.getTotalMemory(), sd.getTotalCpu(), sd.getTotalGenericResources(), + ret.put(sd.getId(), new SupervisorResources(sd.getTotalMemory(), sd.getTotalCpu(), sd + .getTotalGenericResources(), 0, 0, new HashMap<>())); } for (SchedulerAssignmentImpl assignment : assignments.values()) { @@ -1092,7 +1119,8 @@ public Map getSupervisorsResourcesMap() { sr = sr.add(entry.getValue()); ret.put(id, sr); } - Map nodeIdToSharedOffHeap = assignment.getNodeIdToTotalSharedOffHeapNodeMemory(); + Map nodeIdToSharedOffHeap = assignment + .getNodeIdToTotalSharedOffHeapNodeMemory(); if (nodeIdToSharedOffHeap != null) { for (Entry entry : nodeIdToSharedOffHeap.entrySet()) { String id = entry.getKey(); @@ -1133,13 +1161,16 @@ public WorkerResources getWorkerResources(WorkerSlot ws) { /** * This method updates ScheduledResources and UsedSlots cache for given workerSlot. */ - private void updateCachesForWorkerSlot(WorkerSlot workerSlot, WorkerResources workerResources, String topologyId, + private void updateCachesForWorkerSlot(WorkerSlot workerSlot, WorkerResources workerResources, + String topologyId, Double sharedOffHeapNodeMemory) { String nodeId = workerSlot.getNodeId(); NormalizedResourceRequest normalizedResourceRequest = new NormalizedResourceRequest(); normalizedResourceRequest.add(workerResources); - nodeToScheduledResourcesCache.computeIfAbsent(nodeId, Cluster::makeMap).put(workerSlot, normalizedResourceRequest); - nodeToScheduledOffHeapNodeMemoryCache.computeIfAbsent(nodeId, Cluster::makeMap).put(topologyId, sharedOffHeapNodeMemory); + nodeToScheduledResourcesCache.computeIfAbsent(nodeId, Cluster::makeMap).put(workerSlot, + normalizedResourceRequest); + nodeToScheduledOffHeapNodeMemoryCache.computeIfAbsent(nodeId, Cluster::makeMap) + .put(topologyId, sharedOffHeapNodeMemory); nodeToUsedSlotsCache.computeIfAbsent(nodeId, Cluster::makeSet).add(workerSlot); } @@ -1152,12 +1183,14 @@ public NormalizedResourceRequest getAllScheduledResourcesForNode(String nodeId) return totalResourcesPerNodeCache.computeIfAbsent(nodeId, (nid) -> { // executor resources NormalizedResourceRequest totalScheduledResources = new NormalizedResourceRequest(); - for (NormalizedResourceRequest req : nodeToScheduledResourcesCache.computeIfAbsent(nodeId, Cluster::makeMap).values()) { + for (NormalizedResourceRequest req : nodeToScheduledResourcesCache + .computeIfAbsent(nodeId, Cluster::makeMap).values()) { totalScheduledResources.add(req); } // shared off heap node memory for (Double offHeapNodeMemory - : nodeToScheduledOffHeapNodeMemoryCache.computeIfAbsent(nid, Cluster::makeMap).values()) { + : nodeToScheduledOffHeapNodeMemoryCache.computeIfAbsent(nid, Cluster::makeMap) + .values()) { totalScheduledResources.addOffHeap(offHeapNodeMemory); } diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/Component.java b/storm-server/src/main/java/org/apache/storm/scheduler/Component.java index 7f5953a4087..f5a9c8d72a5 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/Component.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/Component.java @@ -42,7 +42,8 @@ public class Component { * @param compId the id of the component * @param execs the executors for this component. */ - public Component(ComponentType type, String compId, List execs, Map inputs) { + public Component(ComponentType type, String compId, List execs, + Map inputs) { this.type = type; this.id = compId; this.execs = execs; diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/DefaultScheduler.java b/storm-server/src/main/java/org/apache/storm/scheduler/DefaultScheduler.java index b8a287ec704..ee29d2e07d8 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/DefaultScheduler.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/DefaultScheduler.java @@ -22,15 +22,16 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Map.Entry; +import java.util.Map; import java.util.Set; import org.apache.storm.metric.StormMetricsRegistry; import org.apache.storm.utils.Utils; public class DefaultScheduler implements IScheduler { - private static Set badSlots(Map> existingSlots, int numExecutors, int numWorkers) { + private static Set badSlots(Map> existingSlots, + int numExecutors, int numWorkers) { if (numWorkers != 0) { Map distribution = Utils.integerDivided(numExecutors, numWorkers); Set slots = new HashSet(); @@ -72,15 +73,21 @@ public static Set slotsCanReassign(Cluster cluster, Set } public static void defaultSchedule(Topologies topologies, Cluster cluster) { - // Single full-set round-robin redistribute for the whole round. The per-topology scheduleTopologiesEvenly call - // below passes redistributeOntoIdle=false so the max.free.per.topology cap is not applied a second time on a + // Single full-set round-robin redistribute for the whole round. The per-topology + // scheduleTopologiesEvenly call + // below passes redistributeOntoIdle=false so the max.free.per.topology cap is not applied a + // second time on a // supervisor left idle by this pass (apache/storm#8778 follow-up). EvenScheduler.redistributeOntoIdleSupervisors(topologies, cluster); for (TopologyDetails topology : cluster.needsSchedulingTopologies()) { - // needsSchedulingTopologies() returns the cluster's full topology set, but this run is scoped to the - // topologies passed in: DefaultScheduler.schedule passes the full set (so the guard is a no-op), while - // IsolationScheduler delegates only its leftover, non-isolated topologies here. redistributeOntoIdleSupervisors - // above acted only on that passed-in set too. Skip topologies outside it so the leftover path never schedules + // needsSchedulingTopologies() returns the cluster's full topology set, but this run is + // scoped to the + // topologies passed in: DefaultScheduler.schedule passes the full set (so the guard is + // a no-op), while + // IsolationScheduler delegates only its leftover, non-isolated topologies here. + // redistributeOntoIdleSupervisors + // above acted only on that passed-in set too. Skip topologies outside it so the + // leftover path never schedules // one the caller excluded -- e.g. a down isolated topology on a reserved host. if (topologies.getById(topology.getId()) == null) { continue; @@ -96,7 +103,8 @@ public static void defaultSchedule(Topologies topologies, Cluster cluster) { } Set canReassignSlots = slotsCanReassign(cluster, aliveAssigned.keySet()); - int totalSlotsToUse = Math.min(topology.getNumWorkers(), canReassignSlots.size() + availableSlots.size()); + int totalSlotsToUse = Math.min(topology.getNumWorkers(), canReassignSlots.size() + + availableSlots.size()); Set badSlots = null; if (totalSlotsToUse > aliveAssigned.size() || !allExecutors.equals(aliveExecutors)) { @@ -112,7 +120,7 @@ public static void defaultSchedule(Topologies topologies, Cluster cluster) { @Override public void prepare(Map conf, StormMetricsRegistry metricsRegistry) { - //noop + // noop } @Override diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/EvenScheduler.java b/storm-server/src/main/java/org/apache/storm/scheduler/EvenScheduler.java index dfd0ec23b55..877d267ffc8 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/EvenScheduler.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/EvenScheduler.java @@ -45,16 +45,16 @@ public class EvenScheduler implements IScheduler { @VisibleForTesting public static List sortSlots(List availableSlots) { - //For example, we have a three nodes(supervisor1, supervisor2, supervisor3) cluster: - //slots before sort: - //supervisor1:6700, supervisor1:6701, - //supervisor2:6700, supervisor2:6701, supervisor2:6702, - //supervisor3:6700, supervisor3:6703, supervisor3:6702, supervisor3:6701 - //slots after sort: - //supervisor3:6700, supervisor2:6700, supervisor1:6700, - //supervisor3:6701, supervisor2:6701, supervisor1:6701, - //supervisor3:6702, supervisor2:6702, - //supervisor3:6703 + // For example, we have a three nodes(supervisor1, supervisor2, supervisor3) cluster: + // slots before sort: + // supervisor1:6700, supervisor1:6701, + // supervisor2:6700, supervisor2:6701, supervisor2:6702, + // supervisor3:6700, supervisor3:6703, supervisor3:6702, supervisor3:6701 + // slots after sort: + // supervisor3:6700, supervisor2:6700, supervisor1:6700, + // supervisor3:6701, supervisor2:6701, supervisor1:6701, + // supervisor3:6702, supervisor2:6702, + // supervisor3:6703 if (availableSlots != null && availableSlots.size() > 0) { // group by node @@ -107,35 +107,50 @@ public static Map> getAliveAssignedWorkerSlotE } /** - * Round-robin relocation of currently-assigned workers onto fully-idle supervisors. Each round-robin iteration moves - * at most one worker per topology, so multiple topologies share the idle slots and a single returning supervisor ends - * up hosting workers from several topologies — preserving the per-supervisor workload diversity that a fresh + * Round-robin relocation of currently-assigned workers onto fully-idle supervisors. Each + * round-robin iteration moves + * at most one worker per topology, so multiple topologies share the idle slots and a single + * returning supervisor ends + * up hosting workers from several topologies — preserving the per-supervisor workload diversity + * that a fresh * cluster has after submission. * *

      Per-topology cap in one scheduling round is * {@code idleSupervisorCount * floor(numWorkers / nonBlacklistedSupervisorCount)}, further tightened by - * {@link DaemonConfig#NIMBUS_EVEN_REBALANCE_MAX_FREE_PER_TOPOLOGY} when set to a positive value. Topologies whose + * {@link DaemonConfig#NIMBUS_EVEN_REBALANCE_MAX_FREE_PER_TOPOLOGY} when set to a positive + * value. Topologies whose * computed cap is zero (typically {@code numWorkers < numSupervisors}) are skipped entirely. The trigger remains - * binary — only fires when at least one supervisor has zero used slots — so a near-balanced cluster sees no + * binary — only fires when at least one supervisor has zero used slots — so a near-balanced + * cluster sees no * movement. {@code numWorkers} here is the topology's declared worker count, so the cap is an upper bound, - * not a guarantee: for an under-assigned topology the donor guard in {@link #relocateOneWorkerOntoIdleSlot} — which - * never drains a source supervisor below one worker — can keep the actual number of relocations below it. + * not a guarantee: for an under-assigned topology the donor guard in {@link + * #relocateOneWorkerOntoIdleSlot} — which + * never drains a source supervisor below one worker — can keep the actual number of relocations + * below it. * - *

      Workers are always pulled from the supervisor where this topology has the most workers, and only when that - * supervisor would still hold at least one worker afterward. Each pulled worker's executors are placed directly - * onto an idle slot, so the subsequent sortSlots / interleave pass cannot drop them back into the just-vacated - * slots. Ties between equally loaded source supervisors are resolved by supervisor id, lexicographically. + *

      Workers are always pulled from the supervisor where this topology has the most workers, + * and only when that + * supervisor would still hold at least one worker afterward. Each pulled worker's executors are + * placed directly + * onto an idle slot, so the subsequent sortSlots / interleave pass cannot drop them back into + * the just-vacated + * slots. Ties between equally loaded source supervisors are resolved by supervisor id, + * lexicographically. * - *

      Gated by {@link DaemonConfig#NIMBUS_EVEN_REBALANCE_ON_IDLE_SUPERVISOR_ENABLED}: when disabled (the default) the - * method returns before scanning any supervisor, so a cluster that has not opted in pays no per-scheduling-round cost. + *

      Gated by {@link DaemonConfig#NIMBUS_EVEN_REBALANCE_ON_IDLE_SUPERVISOR_ENABLED}: when + * disabled (the default) the + * method returns before scanning any supervisor, so a cluster that has not opted in pays no + * per-scheduling-round cost. * *

      This is the package-private entry point reached by the {@code scheduleTopologiesEvenly} overloads (when * {@code redistributeOntoIdle} is true) and called directly by {@link DefaultScheduler#defaultSchedule(Topologies, - * Cluster)}; its visibility is dictated by those callers, not by tests (which also reach it from the same package). + * Cluster)}; its visibility is dictated by those callers, not by tests (which also reach it + * from the same package). */ static void redistributeOntoIdleSupervisors(Topologies topologies, Cluster cluster) { if (!ObjectReader.getBoolean( - cluster.getConf().get(DaemonConfig.NIMBUS_EVEN_REBALANCE_ON_IDLE_SUPERVISOR_ENABLED), false)) { + cluster.getConf() + .get(DaemonConfig.NIMBUS_EVEN_REBALANCE_ON_IDLE_SUPERVISOR_ENABLED), false)) { return; } int nonBlacklistedSupervisorCount = 0; @@ -162,7 +177,8 @@ static void redistributeOntoIdleSupervisors(Topologies topologies, Cluster clust } } } - if (idleTargets.isEmpty() || nonBlacklistedSupervisorCount == 0 || idleSupervisorCount == 0) { + if (idleTargets.isEmpty() || nonBlacklistedSupervisorCount == 0 + || idleSupervisorCount == 0) { return; } @@ -172,12 +188,15 @@ static void redistributeOntoIdleSupervisors(Topologies topologies, Cluster clust List orderedTopos = new ArrayList<>(); Map remainingBudget = new HashMap<>(); for (TopologyDetails topo : topologies.getTopologies()) { - // Skip topologies already present on every idle supervisor -- relocating gains them no workload diversity. - // Reuses the idle-supervisor set computed above instead of re-scanning every supervisor for each topology. + // Skip topologies already present on every idle supervisor -- relocating gains them no + // workload diversity. + // Reuses the idle-supervisor set computed above instead of re-scanning every supervisor + // for each topology. if (!topologyCanReuseIdleSupervisor(cluster, topo, idleSupervisorIds)) { continue; } - int target = (topo.getNumWorkers() / nonBlacklistedSupervisorCount) * idleSupervisorCount; + int target = (topo + .getNumWorkers() / nonBlacklistedSupervisorCount) * idleSupervisorCount; if (target <= 0) { continue; } @@ -216,16 +235,19 @@ static void redistributeOntoIdleSupervisors(Topologies topologies, Cluster clust } } if (totalRelocated > 0) { - LOG.info("EvenScheduler: relocated {} worker(s) onto idle supervisor(s) round-robin across {} topologies.", + LOG.info("EvenScheduler: relocated {} worker(s) onto idle supervisor(s) round-robin " + + "across {} topologies.", totalRelocated, orderedTopos.size()); } } /** * Returns true when at least one of the already-identified idle supervisors does not currently host {@code topology} - * -- i.e. the topology can gain workload diversity by relocating onto it. This is the per-topology half of the binary + * -- i.e. the topology can gain workload diversity by relocating onto it. This is the + * per-topology half of the binary * idle-rebalance trigger; it operates on the pre-computed {@code idleSupervisorIds} (each already vetted by - * {@link Cluster#isIdleSupervisorAvailableForEvenRebalance(SupervisorDetails)}) so the per-topology loop avoids a full + * {@link Cluster#isIdleSupervisorAvailableForEvenRebalance(SupervisorDetails)}) so the + * per-topology loop avoids a full * supervisor rescan. */ private static boolean topologyCanReuseIdleSupervisor(Cluster cluster, TopologyDetails topology, @@ -245,7 +267,8 @@ private static boolean topologyCanReuseIdleSupervisor(Cluster cluster, TopologyD /** * Pulls a single worker from the supervisor where {@code topology} currently has the most workers and reassigns its * executors onto the next idle slot from {@code idleTargets}. Returns false (without consuming an idle target) if - * the topology has no eligible source supervisor — namely all of its supervisors host at most one of its workers, + * the topology has no eligible source supervisor — namely all of its supervisors host at most + * one of its workers, * which would otherwise drain that supervisor to zero and turn it into the next round's idle. */ private static boolean relocateOneWorkerOntoIdleSlot(TopologyDetails topology, Cluster cluster, @@ -256,7 +279,8 @@ private static boolean relocateOneWorkerOntoIdleSlot(TopologyDetails topology, C for (WorkerSlot slot : slotToExecutors.keySet()) { nodeToSlots.computeIfAbsent(slot.getNodeId(), k -> new ArrayList<>()).add(slot); } - List>> candidates = new ArrayList<>(nodeToSlots.entrySet()); + List>> candidates = new ArrayList<>(nodeToSlots + .entrySet()); candidates.removeIf(e -> e.getValue().size() < 2); candidates.sort(Comparator .>>comparingInt(e -> e.getValue().size()) @@ -276,21 +300,28 @@ private static boolean relocateOneWorkerOntoIdleSlot(TopologyDetails topology, C return false; } WorkerSlot target = idleTargets.poll(); - // freeSlot-then-assign is intentionally ordered and non-atomic. target is a pre-verified fully-idle slot -- - // idleTargets only holds ports from supervisors that passed Cluster#isIdleSupervisorAvailableForEvenRebalance, - // which requires getUsedPorts to be empty -- so assign cannot hit its slot-occupied path and no rollback is - // needed. In the near-impossible event assign threw here, the freed executors are picked up by the regular + // freeSlot-then-assign is intentionally ordered and non-atomic. target is a pre-verified + // fully-idle slot -- + // idleTargets only holds ports from supervisors that passed + // Cluster#isIdleSupervisorAvailableForEvenRebalance, + // which requires getUsedPorts to be empty -- so assign cannot hit its slot-occupied path + // and no rollback is + // needed. In the near-impossible event assign threw here, the freed executors are picked up + // by the regular // scheduling pass on the same round. cluster.freeSlot(victim); cluster.assign(target, topology.getId(), execs); return true; } - private static Map scheduleTopology(TopologyDetails topology, Cluster cluster) { + private static Map scheduleTopology(TopologyDetails topology, + Cluster cluster) { List availableSlots = cluster.getAvailableSlots(); Set allExecutors = topology.getExecutors(); - Map> aliveAssigned = getAliveAssignedWorkerSlotExecutors(cluster, topology.getId()); - int totalSlotsToUse = Math.min(topology.getNumWorkers(), availableSlots.size() + aliveAssigned.size()); + Map> aliveAssigned = + getAliveAssignedWorkerSlotExecutors(cluster, topology.getId()); + int totalSlotsToUse = Math.min(topology.getNumWorkers(), availableSlots.size() + + aliveAssigned.size()); List sortedList = sortSlots(availableSlots); if (sortedList == null) { @@ -298,9 +329,10 @@ private static Map scheduleTopology(TopologyDetails return new HashMap(); } - //allow requesting slots number bigger than available slots + // allow requesting slots number bigger than available slots int toIndex = (totalSlotsToUse - aliveAssigned.size()) - > sortedList.size() ? sortedList.size() : (totalSlotsToUse - aliveAssigned.size()); + > sortedList.size() ? sortedList.size() : (totalSlotsToUse - aliveAssigned + .size()); List reassignSlots = sortedList.subList(0, toIndex); Set aliveExecutors = new HashSet(); @@ -341,32 +373,44 @@ public static void scheduleTopologiesEvenly(Topologies topologies, Cluster clust * *

      {@code redistributeOntoIdle} exists to keep the per-topology {@code max.free.per.topology} cap applied once per * scheduling round. The {@link EvenScheduler#schedule(Topologies, Cluster)} entry point passes {@code true}: it runs - * here once over the full topology set, so the redistribute is the single full-set round-robin pass. + * here once over the full topology set, so the redistribute is the single full-set round-robin + * pass. * {@link DefaultScheduler#defaultSchedule(Topologies, Cluster)} instead calls - * {@link #redistributeOntoIdleSupervisors(Topologies, Cluster)} itself, once, over the full set, and then delegates + * {@link #redistributeOntoIdleSupervisors(Topologies, Cluster)} itself, once, over the full + * set, and then delegates * here once per leftover topology with a single-topology {@link Topologies}; it passes {@code false} so the cap is not - * re-applied per topology. Without that guard a returning supervisor left idle by the full-set pass would be filled a - * second time in the same round, letting an under-assigned topology move up to twice the cap (apache/storm#8778 + * re-applied per topology. Without that guard a returning supervisor left idle by the full-set + * pass would be filled a + * second time in the same round, letting an under-assigned topology move up to twice the cap + * (apache/storm#8778 * follow-up). */ - static void scheduleTopologiesEvenly(Topologies topologies, Cluster cluster, boolean redistributeOntoIdle) { + static void scheduleTopologiesEvenly(Topologies topologies, Cluster cluster, + boolean redistributeOntoIdle) { if (redistributeOntoIdle) { redistributeOntoIdleSupervisors(topologies, cluster); } for (TopologyDetails topology : cluster.needsSchedulingTopologies()) { - // needsSchedulingTopologies() returns the cluster's full topology set, but this run is scoped to the - // topologies passed in: EvenScheduler.schedule passes the full set (so the guard is a no-op), while - // DefaultScheduler.defaultSchedule calls us once per leftover topology with a single-topology Topologies. - // The redistribute pass (when run) acted only on that passed-in set too. Skip topologies outside it so - // the leftover path never schedules one the caller excluded -- e.g. a down isolated topology on a reserved host. + // needsSchedulingTopologies() returns the cluster's full topology set, but this run is + // scoped to the + // topologies passed in: EvenScheduler.schedule passes the full set (so the guard is a + // no-op), while + // DefaultScheduler.defaultSchedule calls us once per leftover topology with a + // single-topology Topologies. + // The redistribute pass (when run) acted only on that passed-in set too. Skip + // topologies outside it so + // the leftover path never schedules one the caller excluded -- e.g. a down isolated + // topology on a reserved host. if (topologies.getById(topology.getId()) == null) { continue; } String topologyId = topology.getId(); Map newAssignment = scheduleTopology(topology, cluster); - Map> nodePortToExecutors = Utils.reverseMap(newAssignment); + Map> nodePortToExecutors = Utils + .reverseMap(newAssignment); - for (Map.Entry> entry : nodePortToExecutors.entrySet()) { + for (Map.Entry> entry : nodePortToExecutors + .entrySet()) { WorkerSlot nodePort = entry.getKey(); List executors = entry.getValue(); cluster.assign(nodePort, topologyId, executors); @@ -376,7 +420,7 @@ static void scheduleTopologiesEvenly(Topologies topologies, Cluster cluster, boo @Override public void prepare(Map conf, StormMetricsRegistry metricsRegistry) { - //noop + // noop } @Override diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/INimbus.java b/storm-server/src/main/java/org/apache/storm/scheduler/INimbus.java index 4e26708e475..046c3fb430f 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/INimbus.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/INimbus.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -20,7 +26,8 @@ public interface INimbus { void prepare(Map topoConf, String schedulerLocalDir); /** - * Returns all slots that are available for the next round of scheduling. A slot is available for scheduling + * Returns all slots that are available for the next round of scheduling. A slot is available + * for scheduling * if it is free and can be assigned to, or if it is used and can be reassigned. */ Collection allSlotsAvailableForScheduling(Collection existingSupervisors, @@ -28,12 +35,13 @@ Collection allSlotsAvailableForScheduling(Collection topologiesMissingAssignments); /** - * this is called after the assignment is changed in ZK. + * This is called after the assignment is changed in ZK. */ - void assignSlots(Topologies topologies, Map> newSlotsByTopologyId); + void assignSlots(Topologies topologies, Map> newSlotsByTopologyId); /** - * map from node id to supervisor details. + * Map from node id to supervisor details. */ String getHostName(Map existingSupervisors, String nodeId); diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/INodeAssignmentSentCallBack.java b/storm-server/src/main/java/org/apache/storm/scheduler/INodeAssignmentSentCallBack.java index 9c7cd5f12bd..7c31bf0c457 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/INodeAssignmentSentCallBack.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/INodeAssignmentSentCallBack.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/IScheduler.java b/storm-server/src/main/java/org/apache/storm/scheduler/IScheduler.java index e7888c822ab..29f81d87f2c 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/IScheduler.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/IScheduler.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -23,12 +29,16 @@ public interface IScheduler extends INodeAssignmentSentCallBack { * Set assignments for the topologies which needs scheduling. The new assignments is available * through `cluster.getAssignments()` * - *@param topologies all the topologies in the cluster, some of them need schedule. Topologies object here - * only contain static information about topologies. Information like assignments, slots are all in + * @param topologies all the topologies in the cluster, some of them need schedule. Topologies + * object here + * only contain static information about topologies. Information like assignments, slots are + * all in * the `cluster` object. *@param cluster the cluster these topologies are running in. `cluster` contains everything user - * need to develop a new scheduling logic. e.g. supervisors information, available slots, current - * assignments for all the topologies etc. User can set the new assignment for topologies using + * need to develop a new scheduling logic. e.g. supervisors information, available slots, + * current + * assignments for all the topologies etc. User can set the new assignment for topologies + * using * cluster.setAssignmentById()` */ void schedule(Topologies topologies, Cluster cluster); @@ -41,7 +51,7 @@ public interface IScheduler extends INodeAssignmentSentCallBack { Map config(); /** - * called once when the system is shutting down, should be idempotent. + * Called once when the system is shutting down, should be idempotent. */ default void cleanup() { } diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/ISchedulingState.java b/storm-server/src/main/java/org/apache/storm/scheduler/ISchedulingState.java index e530a45e651..88f9864542f 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/ISchedulingState.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/ISchedulingState.java @@ -24,7 +24,6 @@ import java.util.List; import java.util.Map; import java.util.Set; - import org.apache.storm.daemon.nimbus.TopologyResources; import org.apache.storm.generated.WorkerResources; import org.apache.storm.networktopography.DNSToSwitchMapping; @@ -65,7 +64,8 @@ public interface ISchedulingState { boolean needsScheduling(TopologyDetails topology); /** - * Like {@link #needsScheduling(TopologyDetails)} but does not take into account the number of workers requested. This is because the + * Like {@link #needsScheduling(TopologyDetails)} but does not take into account the number of + * workers requested. This is because the * number of workers is ignored in RAS * * @param topology the topology to check @@ -105,7 +105,7 @@ public interface ISchedulingState { String getHost(String supervisorId); /** - * get the unassigned executors of the topology. + * Get the unassigned executors of the topology. * * @param topology the topology to check * @return the unassigned executors of the topology. @@ -159,7 +159,9 @@ Map> getNeedsSchedulingComponentToExecutors( /** * Get all the available worker slots in the cluster, that are not blacklisted. - * @param blacklistedSupervisorIds list of supervisor ids that should also be considered blacklisted. + * + * @param blacklistedSupervisorIds list of supervisor ids that should also be considered + * blacklisted. */ List getNonBlacklistedAvailableSlots(List blacklistedSupervisorIds); @@ -199,13 +201,15 @@ Map> getNeedsSchedulingComponentToExecutors( /** * Get the resources on the supervisor that are available to be scheduled. + * * @param sd the supervisor. * @return the resources available to be scheduled. */ NormalizedResourceOffer getAvailableResources(SupervisorDetails sd); /** - * Would scheduling exec on ws fit? With a heap <= maxHeap total memory added <= memoryAvailable and cpu added <= cpuAvailable. + * Would scheduling exec on ws fit? With a heap <= maxHeap total memory added <= memoryAvailable + * and cpu added <= cpuAvailable. * * @param ws the slot to put it in * @param exec the executor to investigate @@ -222,12 +226,12 @@ boolean wouldFit( double maxHeap); /** - * get the current assignment for the topology. + * Get the current assignment for the topology. */ SchedulerAssignment getAssignmentById(String topologyId); /** - * get slots used by a topology. + * Get slots used by a topology. */ Collection getUsedSlotsByTopologyId(String topologyId); @@ -261,6 +265,7 @@ boolean wouldFit( /** * Get the resources in the cluster that are available for scheduling. + * * @param blacklistedSupervisorIds other ids that are tentatively blacklisted. */ NormalizedResourceOffer getNonBlacklistedClusterAvailableResources(Collection blacklistedSupervisorIds); @@ -303,18 +308,23 @@ default Map getHostToRack() { Map getStatusMap(); /** - * Get the amount of resources used by topologies. Used for displaying resource information on the UI. + * Get the amount of resources used by topologies. Used for displaying resource information on + * the UI. * - * @return a map that contains multiple topologies and the resources the topology requested and assigned. Key: topology id Value: an - * array that describes the resources the topology requested and assigned in the following format: {requestedMemOnHeap, + * @return a map that contains multiple topologies and the resources the topology requested and + * assigned. Key: topology id Value: an + * array that describes the resources the topology requested and assigned in the following + * format: {requestedMemOnHeap, * requestedMemOffHeap, requestedCpu, assignedMemOnHeap, assignedMemOffHeap, assignedCpu} */ Map getTopologyResourcesMap(); /** - * Get the amount of used and free resources on a supervisor. Used for displaying resource information on the UI + * Get the amount of used and free resources on a supervisor. Used for displaying resource + * information on the UI * - * @return a map where the key is the supervisor id and the value is a map that represents resource usage for a supervisor in the + * @return a map where the key is the supervisor id and the value is a map that represents + * resource usage for a supervisor in the * following format: {totalMem, totalCpu, usedMem, usedCpu} */ Map getSupervisorsResourcesMap(); @@ -356,7 +366,8 @@ default Map getHostToRack() { Map getConf(); /** - * Determine the list of racks on which topologyIds have been assigned. Note that the returned set + * Determine the list of racks on which topologyIds have been assigned. Note that the returned + * set * may contain {@link DNSToSwitchMapping#DEFAULT_RACK} if {@link #getHostToRack()} is null or * does not contain the assigned host. * @@ -375,7 +386,8 @@ default Set getAssignedRacks(String... topologyIds) { String nodeId = slot.getNodeId(); SupervisorDetails supervisorDetails = getSupervisorById(nodeId); String hostId = supervisorDetails.getHost(); - String rackId = networkTopographyInverted.getOrDefault(hostId, DNSToSwitchMapping.DEFAULT_RACK); + String rackId = networkTopographyInverted.getOrDefault(hostId, + DNSToSwitchMapping.DEFAULT_RACK); ret.add(rackId); } } diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/ISupervisor.java b/storm-server/src/main/java/org/apache/storm/scheduler/ISupervisor.java index 7cb1dd67c8e..a1b4876dfb4 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/ISupervisor.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/ISupervisor.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/IsolationScheduler.java b/storm-server/src/main/java/org/apache/storm/scheduler/IsolationScheduler.java index 0e2c283a6a4..d7cf99c2ea4 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/IsolationScheduler.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/IsolationScheduler.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -32,7 +38,8 @@ import org.slf4j.LoggerFactory; // for each isolated topology: -// compute even distribution of executors -> workers on the number of workers specified for the topology +// compute even distribution of executors -> workers on the number of workers specified for the +// topology // compute distribution of workers to machines // determine host -> list of [slot, topology id, executors] // iterate through hosts and: a machine is good if: @@ -48,7 +55,8 @@ public class IsolationScheduler implements IScheduler { @Override public void prepare(Map conf, StormMetricsRegistry metricsRegistry) { - this.isoMachines = (Map) conf.get(DaemonConfig.ISOLATION_SCHEDULER_MACHINES); + this.isoMachines = (Map) conf + .get(DaemonConfig.ISOLATION_SCHEDULER_MACHINES); Validate.notEmpty(isoMachines); } @@ -57,20 +65,26 @@ public Map> config() { return Collections.emptyMap(); } - // get host -> all assignable worker slots for non-blacklisted machines (assigned or not assigned) - // will then have a list of machines that need to be assigned (machine -> [topology, list of list of executors]) - // match each spec to a machine (who has the right number of workers), free everything else on that machine and assign those slots + // get host -> all assignable worker slots for non-blacklisted machines (assigned or not + // assigned) + // will then have a list of machines that need to be assigned (machine -> [topology, list of + // list of executors]) + // match each spec to a machine (who has the right number of workers), free everything else on + // that machine and assign those slots // (do one topology at a time) // blacklist all machines who had production slots defined // log isolated topologies who weren't able to get enough slots / machines - // run default scheduler on isolated topologies that didn't have enough slots + non-isolated topologies on remaining machines + // run default scheduler on isolated topologies that didn't have enough slots + non-isolated + // topologies on remaining machines // set blacklist to what it was initially @Override public void schedule(Topologies topologies, Cluster cluster) { List isoTopologies = isolatedTopologies(topologies.getTopologies()); Set isoIds = extractTopologyIds(isoTopologies); - Map>> topologyWorkerSpecs = topologyWorkerSpecs(isoTopologies); - Map> topologyMachineDistributions = topologyMachineDistributions(isoTopologies); + Map>> topologyWorkerSpecs = + topologyWorkerSpecs(isoTopologies); + Map> topologyMachineDistributions = + topologyMachineDistributions(isoTopologies); Map> hostAssignments = hostAssignments(cluster); for (Map.Entry> entry : hostAssignments.entrySet()) { @@ -103,7 +117,8 @@ && checkAssignmentWorkerSpecs(assignments, workerSpecs)) { for (Map.Entry>> entry : topologyWorkerSpecs.entrySet()) { String topologyId = entry.getKey(); Set> executorSet = entry.getValue(); - List workerNum = distributionToSortedAmounts(topologyMachineDistributions.get(topologyId)); + List workerNum = distributionToSortedAmounts(topologyMachineDistributions + .get(topologyId)); for (Integer num : workerNum) { HostAssignableSlots hostSlots = hss.peek(); List slot = hostSlots != null ? hostSlots.getWorkerSlots() : null; @@ -123,7 +138,8 @@ && checkAssignmentWorkerSpecs(assignments, workerSpecs)) { List failedTopologyIds = extractFailedTopologyIds(topologyWorkerSpecs); if (failedTopologyIds.size() > 0) { LOG.warn("Unable to isolate topologies " + failedTopologyIds - + ". No machine had enough worker slots to run the remaining workers for these topologies. " + + ". No machine had enough worker slots to run the remaining workers for " + + "these topologies. " + "Clearing all other resources and will wait for enough resources for " + "isolated topologies before allocating any other resources."); // clear workers off all hosts that are not blacklisted @@ -171,9 +187,12 @@ private Set extractTopologyIds(List topologies) { return ids; } - private List extractFailedTopologyIds(Map>> isoTopologyWorkerSpecs) { + private List extractFailedTopologyIds(Map>> isoTopologyWorkerSpecs) { List failedTopologyIds = new ArrayList(); - for (Map.Entry>> topoWorkerSpecsEntry : isoTopologyWorkerSpecs.entrySet()) { + for (Map.Entry>> topoWorkerSpecsEntry : isoTopologyWorkerSpecs + .entrySet()) { Set> workerSpecs = topoWorkerSpecsEntry.getValue(); if (workerSpecs != null && !workerSpecs.isEmpty()) { failedTopologyIds.add(topoWorkerSpecsEntry.getKey()); @@ -184,7 +203,8 @@ private List extractFailedTopologyIds(Map set of sets of executors private Map>> topologyWorkerSpecs(List topologies) { - Map>> workerSpecs = new HashMap>>(); + Map>> workerSpecs = + new HashMap>>(); for (TopologyDetails topology : topologies) { workerSpecs.put(topology.getId(), computeWorkerSpecs(topology)); } @@ -193,17 +213,20 @@ private Map>> topologyWorkerSpecs(List> hostAssignments(Cluster cluster) { Collection assignmentValues = cluster.getAssignments().values(); - Map> hostAssignments = new HashMap>(); + Map> hostAssignments = + new HashMap>(); for (SchedulerAssignment sa : assignmentValues) { - Map> slotExecutors = Utils.reverseMap(sa.getExecutorToSlot()); + Map> slotExecutors = Utils.reverseMap(sa + .getExecutorToSlot()); Set>> entries = slotExecutors.entrySet(); for (Map.Entry> entry : entries) { WorkerSlot slot = entry.getKey(); List executors = entry.getValue(); String host = cluster.getHost(slot.getNodeId()); - AssignmentInfo ass = new AssignmentInfo(slot, sa.getTopologyId(), new HashSet(executors)); + AssignmentInfo ass = new AssignmentInfo(slot, sa.getTopologyId(), + new HashSet(executors)); List executorList = hostAssignments.get(host); if (executorList == null) { executorList = new ArrayList(); @@ -216,7 +239,8 @@ private Map> hostAssignments(Cluster cluster) { } private Set> computeWorkerSpecs(TopologyDetails topology) { - Map> compExecutors = Utils.reverseMap(topology.getExecutorToComponent()); + Map> compExecutors = Utils.reverseMap(topology + .getExecutorToComponent()); List allExecutors = new ArrayList(); Collection> values = compExecutors.values(); @@ -226,7 +250,8 @@ private Set> computeWorkerSpecs(TopologyDetails topology) { int numWorkers = topology.getNumWorkers(); int bucketIndex = 0; - Map> bucketExecutors = new HashMap>(numWorkers); + Map> bucketExecutors = + new HashMap>(numWorkers); for (ExecutorDetails executor : allExecutors) { Set executors = bucketExecutors.get(bucketIndex); if (executors == null) { @@ -241,7 +266,8 @@ private Set> computeWorkerSpecs(TopologyDetails topology) { } private Map> topologyMachineDistributions(List isoTopologies) { - Map> machineDistributions = new HashMap>(); + Map> machineDistributions = + new HashMap>(); for (TopologyDetails topology : isoTopologies) { machineDistributions.put(topology.getId(), machineDistribution(topology)); } @@ -268,7 +294,8 @@ private boolean checkAssignmentTopology(List assignments, String return true; } - private boolean checkAssignmentWorkerSpecs(List assigments, Set> workerSpecs) { + private boolean checkAssignmentWorkerSpecs(List assigments, + Set> workerSpecs) { for (AssignmentInfo ass : assigments) { if (!workerSpecs.contains(ass.getExecutors())) { return false; @@ -373,9 +400,11 @@ public int compare(Integer o1, Integer o2) { return sorts; } - private Set allocatedTopologies(Map>> topologyToWorkerSpecs) { + private Set allocatedTopologies(Map>> topologyToWorkerSpecs) { Set allocatedTopologies = new HashSet(); - Set>>> entries = topologyToWorkerSpecs.entrySet(); + Set>>> entries = topologyToWorkerSpecs + .entrySet(); for (Map.Entry>> entry : entries) { if (entry.getValue().isEmpty()) { allocatedTopologies.add(entry.getKey()); diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/SchedulerAssignmentImpl.java b/storm-server/src/main/java/org/apache/storm/scheduler/SchedulerAssignmentImpl.java index bcb0b920736..334071a1e99 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/SchedulerAssignmentImpl.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/SchedulerAssignmentImpl.java @@ -31,7 +31,8 @@ public class SchedulerAssignmentImpl implements SchedulerAssignment { private static final Logger LOG = LoggerFactory.getLogger(SchedulerAssignmentImpl.class); - private static Function> MAKE_LIST = (k) -> new LinkedList<>(); + private static Function> MAKE_LIST = + (k) -> new LinkedList<>(); /** * topology-id this assignment is for. @@ -39,7 +40,7 @@ public class SchedulerAssignmentImpl implements SchedulerAssignment { private final String topologyId; /** - * assignment detail, a mapping from executor to WorkerSlot. + * Assignment detail, a mapping from executor to WorkerSlot. */ private final Map executorToSlot = new HashMap<>(); private final Map resources = new HashMap<>(); @@ -50,16 +51,22 @@ public class SchedulerAssignmentImpl implements SchedulerAssignment { * Create a new assignment. * * @param topologyId the id of the topology the assignment is for. - * @param executorToSlot the executor to slot mapping for the assignment. Can be null and set through other methods later. - * @param resources the resources for the current assignments. Can be null and set through other methods later. - * @param nodeIdToTotalSharedOffHeap the shared memory for this assignment can be null and set through other methods later. + * @param executorToSlot the executor to slot mapping for the assignment. Can be null and set + * through other methods later. + * @param resources the resources for the current assignments. Can be null and set through other + * methods later. + * @param nodeIdToTotalSharedOffHeap the shared memory for this assignment can be null and set + * through other methods later. */ - public SchedulerAssignmentImpl(String topologyId, Map executorToSlot, + public SchedulerAssignmentImpl(String topologyId, Map executorToSlot, Map resources, Map nodeIdToTotalSharedOffHeap) { this.topologyId = topologyId; if (executorToSlot != null) { - if (executorToSlot.entrySet().stream().anyMatch((entry) -> entry.getKey() == null || entry.getValue() == null)) { - throw new RuntimeException("Cannot create a scheduling with a null in it " + executorToSlot); + if (executorToSlot.entrySet().stream().anyMatch((entry) -> entry.getKey() == null + || entry.getValue() == null)) { + throw new RuntimeException("Cannot create a scheduling with a null in it " + + executorToSlot); } this.executorToSlot.putAll(executorToSlot); for (Map.Entry entry : executorToSlot.entrySet()) { @@ -67,14 +74,18 @@ public SchedulerAssignmentImpl(String topologyId, Map entry.getKey() == null || entry.getValue() == null)) { - throw new RuntimeException("Cannot create resources with a null in it " + resources); + if (resources.entrySet().stream().anyMatch((entry) -> entry.getKey() == null || entry + .getValue() == null)) { + throw new RuntimeException("Cannot create resources with a null in it " + + resources); } this.resources.putAll(resources); } if (nodeIdToTotalSharedOffHeap != null) { - if (nodeIdToTotalSharedOffHeap.entrySet().stream().anyMatch((entry) -> entry.getKey() == null || entry.getValue() == null)) { - throw new RuntimeException("Cannot create off heap with a null in it " + nodeIdToTotalSharedOffHeap); + if (nodeIdToTotalSharedOffHeap.entrySet().stream().anyMatch((entry) -> entry + .getKey() == null || entry.getValue() == null)) { + throw new RuntimeException("Cannot create off heap with a null in it " + + nodeIdToTotalSharedOffHeap); } this.nodeIdToTotalSharedOffHeapNode.putAll(nodeIdToTotalSharedOffHeap); } @@ -86,12 +97,14 @@ public SchedulerAssignmentImpl(String topologyId) { public SchedulerAssignmentImpl(SchedulerAssignment assignment) { this(assignment.getTopologyId(), assignment.getExecutorToSlot(), - assignment.getScheduledResources(), assignment.getNodeIdToTotalSharedOffHeapNodeMemory()); + assignment.getScheduledResources(), assignment + .getNodeIdToTotalSharedOffHeapNodeMemory()); } @Override public String toString() { - return this.getClass().getSimpleName() + " topo: " + topologyId + " execToSlots: " + executorToSlot; + return this.getClass().getSimpleName() + " topo: " + topologyId + " execToSlots: " + + executorToSlot; } /** @@ -146,7 +159,8 @@ public void assign(WorkerSlot slot, Collection executors) { /** * Assign the slot to executors. */ - public void assign(WorkerSlot slot, Collection executors, WorkerResources slotResources) { + public void assign(WorkerSlot slot, Collection executors, + WorkerResources slotResources) { if (slot == null) { throw new AssertionError("WorkerSlot parameter is null"); } diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/SingleTopologyCluster.java b/storm-server/src/main/java/org/apache/storm/scheduler/SingleTopologyCluster.java index ff020a45902..9e7dccea089 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/SingleTopologyCluster.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/SingleTopologyCluster.java @@ -37,7 +37,7 @@ public SingleTopologyCluster(Cluster other, String topologyId) { @Override protected void assertValidTopologyForModification(String topologyId) { - //AllowedId is null in the constructor, so it can assign what it needs/etc. + // AllowedId is null in the constructor, so it can assign what it needs/etc. if (allowedId != null && !allowedId.equals(topologyId)) { throw new IllegalArgumentException( "Only " + allowedId + " is allowed to be modified at this time."); diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/SupervisorDetails.java b/storm-server/src/main/java/org/apache/storm/scheduler/SupervisorDetails.java index f03e1de5049..1998d164c35 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/SupervisorDetails.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/SupervisorDetails.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -26,16 +32,16 @@ public class SupervisorDetails { private final String id; /** - * thrift server of this supervisor. + * Thrift server of this supervisor. */ private final Integer serverPort; /** - * hostname of this supervisor. + * Hostname of this supervisor. */ private final String host; private final Object meta; /** - * meta data configured for this supervisor. + * Meta data configured for this supervisor. */ private final Object schedulerMeta; /** @@ -43,31 +49,39 @@ public class SupervisorDetails { */ private final NormalizedResourceOffer totalResources; /** - * all the ports of the supervisor. + * All the ports of the supervisor. */ private Set allPorts; private final long uptimeSecs; /** * Create the details of a new supervisor. + * * @param id the ID as reported by the supervisor. * @param serverPort the thrift server for the supervisor. * @param host the host the supervisor is on. - * @param meta meta data reported by the supervisor (should be a collection of the ports on the supervisor). + * @param meta meta data reported by the supervisor (should be a collection of the ports on the + * supervisor). * @param schedulerMeta Not used and can probably be removed. * @param allPorts all of the ports for the supervisor (a better version of meta) * @param totalResources all of the resources for this supervisor. */ - public SupervisorDetails(String id, Integer serverPort, String host, Object meta, Object schedulerMeta, + public SupervisorDetails(String id, Integer serverPort, String host, Object meta, + Object schedulerMeta, Collection allPorts, Map totalResources) { - // Callers that do not supply uptime (tests and every path other than the idle-supervisor rebalance) default to - // Long.MAX_VALUE, i.e. treated as indefinitely stable so the rebalance flap guard never holds them back. This is - // the deliberate opposite of Nimbus#supervisorUptimeSecs, which maps an unset uptime to 0L; production always - // supplies the real uptime through that path, so this default only affects non-feature callers. + // Callers that do not supply uptime (tests and every path other than the idle-supervisor + // rebalance) default to + // Long.MAX_VALUE, i.e. treated as indefinitely stable so the rebalance flap guard never + // holds them back. This is + // the deliberate opposite of Nimbus#supervisorUptimeSecs, which maps an unset uptime to 0L; + // production always + // supplies the real uptime through that path, so this default only affects non-feature + // callers. this(id, serverPort, host, meta, schedulerMeta, allPorts, totalResources, Long.MAX_VALUE); } - public SupervisorDetails(String id, Integer serverPort, String host, Object meta, Object schedulerMeta, + public SupervisorDetails(String id, Integer serverPort, String host, Object meta, + Object schedulerMeta, Collection allPorts, Map totalResources, long uptimeSecs) { this.id = id; @@ -82,7 +96,8 @@ public SupervisorDetails(String id, Integer serverPort, String host, Object meta this.allPorts = new HashSet<>(); } this.totalResources = new NormalizedResourceOffer(totalResources); - LOG.debug("Creating a new supervisor ({}-{}) with resources: {}", this.host, this.id, totalResources); + LOG.debug("Creating a new supervisor ({}-{}) with resources: {}", this.host, this.id, + totalResources); } public SupervisorDetails(String id, Object meta) { @@ -93,7 +108,8 @@ public SupervisorDetails(String id, Object meta, Map totalResour this(id, null, null, meta, null, null, totalResources); } - public SupervisorDetails(String id, Object meta, Map totalResources, long uptimeSecs) { + public SupervisorDetails(String id, Object meta, Map totalResources, + long uptimeSecs) { this(id, null, null, meta, null, null, totalResources, uptimeSecs); } @@ -101,7 +117,8 @@ public SupervisorDetails(String id, Object meta, Collection al this(id, null, null, meta, null, allPorts, null); } - public SupervisorDetails(String id, String host, Object schedulerMeta, Collection allPorts) { + public SupervisorDetails(String id, String host, Object schedulerMeta, + Collection allPorts) { this(id, null, host, null, schedulerMeta, allPorts, null); } diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/SupervisorResources.java b/storm-server/src/main/java/org/apache/storm/scheduler/SupervisorResources.java index 2c578d6bbfb..b6e635c2e2a 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/SupervisorResources.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/SupervisorResources.java @@ -20,7 +20,6 @@ import java.util.HashMap; import java.util.Map; - import org.apache.storm.generated.WorkerResources; import org.apache.storm.scheduler.resource.normalization.NormalizedResourceRequest; @@ -42,14 +41,17 @@ public class SupervisorResources { * @param usedCpu the used CPU on the supervisor * @param usedGenericResources the used generic resources on the supervisor */ - public SupervisorResources(double totalMem, double totalCpu, Map totalGenericResources, + public SupervisorResources(double totalMem, double totalCpu, Map totalGenericResources, double usedMem, double usedCpu, Map usedGenericResources) { this.totalMem = totalMem; this.totalCpu = totalCpu; this.usedMem = usedMem; this.usedCpu = usedCpu; - this.totalGenericResources = totalGenericResources != null ? totalGenericResources : new HashMap<>(); - this.usedGenericResources = usedGenericResources != null ? usedGenericResources : new HashMap<>(); + this.totalGenericResources = totalGenericResources != null + ? totalGenericResources : new HashMap<>(); + this.usedGenericResources = usedGenericResources != null + ? usedGenericResources : new HashMap<>(); } public double getUsedMem() { @@ -85,7 +87,8 @@ public Map getUsedGenericResources() { } public SupervisorResources add(WorkerResources wr) { - usedGenericResources = NormalizedResourceRequest.addResourceMap(usedGenericResources, wr.get_resources()); + usedGenericResources = NormalizedResourceRequest.addResourceMap(usedGenericResources, wr + .get_resources()); NormalizedResourceRequest.removeNonGenericResources(usedGenericResources); return new SupervisorResources( diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/Topologies.java b/storm-server/src/main/java/org/apache/storm/scheduler/Topologies.java index d135afba8b8..493f298fd3a 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/Topologies.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/Topologies.java @@ -91,7 +91,8 @@ public TopologyDetails getById(String topologyId) { } /** - * Get a topology given a topology name. Nimbus prevents multiple topologies from having the same name, so this assumes it is true. + * Get a topology given a topology name. Nimbus prevents multiple topologies from having the + * same name, so this assumes it is true. * * @param topologyName the name of the topology to look for * @return the a topology with the given name. diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/TopologyDetails.java b/storm-server/src/main/java/org/apache/storm/scheduler/TopologyDetails.java index a750c189ff9..61b39482f45 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/TopologyDetails.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/TopologyDetails.java @@ -47,29 +47,33 @@ public class TopologyDetails { private final StormTopology topology; private final Map executorToComponent; private final int numWorkers; - //when topology was launched + // when topology was launched private final int launchTime; private final String owner; private final String topoName; - //>> + // >> private Map resourceList; - //Max heap size for a worker used by topology + // Max heap size for a worker used by topology private Double topologyWorkerMaxHeapSize; - //topology priority + // topology priority private Integer topologyPriority; // Only contains user topology specific executors private Map userTopologyComponentsMap; - public TopologyDetails(String topologyId, Map topologyConf, StormTopology topology, int numWorkers, String owner) { + public TopologyDetails(String topologyId, Map topologyConf, + StormTopology topology, int numWorkers, String owner) { this(topologyId, topologyConf, topology, numWorkers, null, 0, owner); } - public TopologyDetails(String topologyId, Map topologyConf, StormTopology topology, + public TopologyDetails(String topologyId, Map topologyConf, + StormTopology topology, int numWorkers, Map executorToComponents, String owner) { this(topologyId, topologyConf, topology, numWorkers, executorToComponents, 0, owner); } - public TopologyDetails(String topologyId, Map topologyConf, StormTopology topology, int numWorkers, + public TopologyDetails(String topologyId, Map topologyConf, + StormTopology topology, int numWorkers, Map executorToComponents, int launchTime, String owner) { this.owner = owner; this.topologyId = topologyId; @@ -129,7 +133,8 @@ public Map> getComponentToExecutors() { Map> ret = new HashMap<>(); Map execToComp = getExecutorToComponent(); if (execToComp != null) { - execToComp.forEach((exec, comp) -> ret.computeIfAbsent(comp, (k) -> new HashSet<>()).add(exec)); + execToComp.forEach((exec, comp) -> ret.computeIfAbsent(comp, (k) -> new HashSet<>()) + .add(exec)); } return ret; } @@ -143,8 +148,9 @@ private void initResourceList() { // Extract bolt resource info if (topology.get_bolts() != null) { for (Map.Entry bolt : topology.get_bolts().entrySet()) { - //the json_conf is populated by TopologyBuilder (e.g. boltDeclarer.setMemoryLoad) - NormalizedResourceRequest topologyResources = new NormalizedResourceRequest(bolt.getValue().get_common(), + // the json_conf is populated by TopologyBuilder (e.g. boltDeclarer.setMemoryLoad) + NormalizedResourceRequest topologyResources = new NormalizedResourceRequest(bolt + .getValue().get_common(), topologyConf, bolt.getKey()); for (Map.Entry anExecutorToComponent : executorToComponent.entrySet()) { @@ -157,7 +163,8 @@ private void initResourceList() { // Extract spout resource info if (topology.get_spouts() != null) { for (Map.Entry spout : topology.get_spouts().entrySet()) { - NormalizedResourceRequest topologyResources = new NormalizedResourceRequest(spout.getValue().get_common(), + NormalizedResourceRequest topologyResources = new NormalizedResourceRequest(spout + .getValue().get_common(), topologyConf, spout.getKey()); for (Map.Entry anExecutorToComponent : executorToComponent.entrySet()) { @@ -169,7 +176,7 @@ private void initResourceList() { } else { LOG.warn("Topology " + topologyId + " does not seem to have any spouts!"); } - //schedule tasks that are not part of components returned from topology.get_spout or + // schedule tasks that are not part of components returned from topology.get_spout or // topology.getbolt (AKA sys tasks most specifically __acker tasks) for (ExecutorDetails exec : getExecutors()) { if (!resourceList.containsKey(exec)) { @@ -204,7 +211,8 @@ private Set getInputsTo(ComponentCommon comp) { } /** - * Returns a representation of the non-system components of the topology graph. Each Component object in the returning map is populated + * Returns a representation of the non-system components of the topology graph. Each Component + * object in the returning map is populated * with the list of its parents, children and execs assigned to that component. * * @return a map of components @@ -233,7 +241,8 @@ private Map computeComponentMap(StormTopology topology) { String compId = entry.getKey(); SpoutSpec spout = entry.getValue(); if (!Utils.isSystemId(compId)) { - Component comp = new Component(ComponentType.SPOUT, compId, componentToExecs(compId), spout.get_common().get_inputs()); + Component comp = new Component(ComponentType.SPOUT, compId, + componentToExecs(compId), spout.get_common().get_inputs()); ret.put(compId, comp); } } @@ -243,13 +252,14 @@ private Map computeComponentMap(StormTopology topology) { String compId = entry.getKey(); Bolt bolt = entry.getValue(); if (!Utils.isSystemId(compId)) { - Component comp = new Component(ComponentType.BOLT, compId, componentToExecs(compId), bolt.get_common().get_inputs()); + Component comp = new Component(ComponentType.BOLT, compId, + componentToExecs(compId), bolt.get_common().get_inputs()); ret.put(compId, comp); } } } - //Link the components together + // Link the components together if (spouts != null) { for (Map.Entry entry : spouts.entrySet()) { Component spout = ret.get(entry.getKey()); @@ -355,7 +365,7 @@ public Set getSharedMemoryRequests( } Set ret = new HashSet<>(); if (topology != null) { - //topology being null is used for tests We probably should fix that at some point, + // topology being null is used for tests We probably should fix that at some point, // but it is not trivial to do... Map> compToSharedName = topology.get_component_to_shared_memory(); if (compToSharedName != null) { @@ -387,6 +397,7 @@ public NormalizedResourceRequest getTotalResources(ExecutorDetails exec) { /** * Get an approximate total resources needed for this topology. ignores shared memory. + * * @return the approximate total resources needed for this topology. */ public NormalizedResourceRequest getApproximateTotalResources() { @@ -424,7 +435,8 @@ public Double getTotalCpuReqTask(ExecutorDetails exec) { } /** - * Note: The public API relevant to resource aware scheduling is unstable as of May 2015. We reserve the right to change them. + * Note: The public API relevant to resource aware scheduling is unstable as of May 2015. We + * reserve the right to change them. * * @return the total on-heap memory requested for this topology */ @@ -454,7 +466,8 @@ public double getRequestedNonSharedOnHeap() { } /** - * Note: The public API relevant to resource aware scheduling is unstable as of May 2015. We reserve the right to change them. + * Note: The public API relevant to resource aware scheduling is unstable as of May 2015. We + * reserve the right to change them. * * @return the total off-heap memory requested for this topology */ @@ -484,7 +497,8 @@ public double getRequestedSharedOffHeap() { } /** - * Note: The public API relevant to resource aware scheduling is unstable as of May 2015. We reserve the right to change them. + * Note: The public API relevant to resource aware scheduling is unstable as of May 2015. We + * reserve the right to change them. * * @return the total cpu requested for this topology */ @@ -506,7 +520,7 @@ public Map getTotalRequestedGenericResources() { } /** - * get the resources requirements for a executor. + * Get the resources requirements for a executor. * * @param exec executor details * @return a map containing the resource requirements for this exec @@ -528,11 +542,12 @@ public boolean hasExecInTopo(ExecutorDetails exec) { } /** - * add resource requirements for a executor. + * Add resource requirements for a executor. */ public void addResourcesForExec(ExecutorDetails exec, NormalizedResourceRequest resourceList) { if (hasExecInTopo(exec)) { - LOG.warn("Executor {} already exists...ResourceList: {}", exec, getTaskResourceReqList(exec)); + LOG.warn("Executor {} already exists...ResourceList: {}", exec, + getTaskResourceReqList(exec)); return; } this.resourceList.put(exec, resourceList); @@ -547,7 +562,7 @@ private void addDefaultResforExec(ExecutorDetails exec) { } /** - * initializes member variables. + * Initializes member variables. */ private void initConfigs() { this.topologyWorkerMaxHeapSize = @@ -557,12 +572,12 @@ private void initConfigs() { ObjectReader.getInt(topologyConf.get(Config.TOPOLOGY_PRIORITY), null); // Fails in storm-core: org.apache.storm.scheduler-test / testname: test-cluster - //if (this.topologyWorkerMaxHeapSize == null) { + // if (this.topologyWorkerMaxHeapSize == null) { // throw new AssertionError("topologyWorkerMaxHeapSize is null"); - //} - //if (this.topologyPriority == null) { + // } + // if (this.topologyPriority == null) { // throw new AssertionError("topologyPriority is null"); - //} + // } } /** @@ -582,7 +597,7 @@ public String getTopologySubmitter() { } /** - * get the priority of this topology. + * Get the priority of this topology. */ public int getTopologyPriority() { return topologyPriority; diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java b/storm-server/src/main/java/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java index 2f9aca767c6..6289568816a 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -36,7 +42,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; - public class BlacklistScheduler implements IScheduler { public static final int DEFAULT_BLACKLIST_SCHEDULER_RESUME_TIME = 1800; public static final int DEFAULT_BLACKLIST_SCHEDULER_TOLERANCE_COUNT = 3; @@ -72,24 +77,31 @@ public void prepare(Map conf, StormMetricsRegistry metricsRegist this.conf = conf; this.metricsRegistry = metricsRegistry; - toleranceTime = ObjectReader.getInt(this.conf.get(DaemonConfig.BLACKLIST_SCHEDULER_TOLERANCE_TIME), + toleranceTime = ObjectReader.getInt(this.conf + .get(DaemonConfig.BLACKLIST_SCHEDULER_TOLERANCE_TIME), DEFAULT_BLACKLIST_SCHEDULER_TOLERANCE_TIME); - toleranceCount = ObjectReader.getInt(this.conf.get(DaemonConfig.BLACKLIST_SCHEDULER_TOLERANCE_COUNT), + toleranceCount = ObjectReader.getInt(this.conf + .get(DaemonConfig.BLACKLIST_SCHEDULER_TOLERANCE_COUNT), DEFAULT_BLACKLIST_SCHEDULER_TOLERANCE_COUNT); - resumeTime = ObjectReader.getInt(this.conf.get(DaemonConfig.BLACKLIST_SCHEDULER_RESUME_TIME), + resumeTime = ObjectReader.getInt(this.conf + .get(DaemonConfig.BLACKLIST_SCHEDULER_RESUME_TIME), DEFAULT_BLACKLIST_SCHEDULER_RESUME_TIME); blacklistSendAssignentFailures = ObjectReader.getBoolean(this.conf.get( DaemonConfig.BLACKLIST_SCHEDULER_ENABLE_SEND_ASSIGNMENT_FAILURES), false); - String reporterClassName = ObjectReader.getString(this.conf.get(DaemonConfig.BLACKLIST_SCHEDULER_REPORTER), + String reporterClassName = ObjectReader.getString(this.conf + .get(DaemonConfig.BLACKLIST_SCHEDULER_REPORTER), LogReporter.class.getName()); reporter = (IReporter) initializeInstance(reporterClassName, "blacklist reporter"); - String strategyClassName = ObjectReader.getString(this.conf.get(DaemonConfig.BLACKLIST_SCHEDULER_STRATEGY), + String strategyClassName = ObjectReader.getString(this.conf + .get(DaemonConfig.BLACKLIST_SCHEDULER_STRATEGY), DefaultBlacklistStrategy.class.getName()); - blacklistStrategy = (IBlacklistStrategy) initializeInstance(strategyClassName, "blacklist strategy"); + blacklistStrategy = (IBlacklistStrategy) initializeInstance(strategyClassName, + "blacklist strategy"); - nimbusMonitorFreqSecs = ObjectReader.getInt(this.conf.get(DaemonConfig.NIMBUS_MONITOR_FREQ_SECS)); + nimbusMonitorFreqSecs = ObjectReader.getInt(this.conf + .get(DaemonConfig.NIMBUS_MONITOR_FREQ_SECS)); blacklistStrategy.prepare(this.conf); windowSize = toleranceTime / nimbusMonitorFreqSecs; @@ -98,11 +110,13 @@ public void prepare(Map conf, StormMetricsRegistry metricsRegist cachedSupervisors = new HashMap<>(); blacklistedSupervisorIds = new HashSet<>(); blacklistOnBadSlots = ObjectReader.getBoolean( - this.conf.get(DaemonConfig.BLACKLIST_SCHEDULER_ASSUME_SUPERVISOR_BAD_BASED_ON_BAD_SLOT), + this.conf + .get(DaemonConfig.BLACKLIST_SCHEDULER_ASSUME_SUPERVISOR_BAD_BASED_ON_BAD_SLOT), true); - //nimbus:num-blacklisted-supervisor + non-blacklisted supervisor = nimbus:num-supervisors - metricsRegistry.registerGauge("nimbus:num-blacklisted-supervisor", () -> blacklistedSupervisorIds.size()); + // nimbus:num-blacklisted-supervisor + non-blacklisted supervisor = nimbus:num-supervisors + metricsRegistry.registerGauge("nimbus:num-blacklisted-supervisor", + () -> blacklistedSupervisorIds.size()); } @Override @@ -140,7 +154,8 @@ private void trackMissedHeartbeats(Map supervisors) { Set cachedSupervisorsKeySet = cachedSupervisors.keySet(); Set supervisorsKeySet = supervisors.keySet(); - Set badSupervisorKeys = Sets.difference(cachedSupervisorsKeySet, supervisorsKeySet); //cached supervisor doesn't show up + Set badSupervisorKeys = Sets.difference(cachedSupervisorsKeySet, + supervisorsKeySet); // cached supervisor doesn't show up HashMap> badSupervisors = new HashMap<>(); for (String key : badSupervisorKeys) { badSupervisors.put(key, cachedSupervisors.get(key)); @@ -151,12 +166,13 @@ private void trackMissedHeartbeats(Map supervisors) { if (cachedSupervisors.containsKey(key)) { if (blacklistOnBadSlots) { Set badSlots = badSlots(supervisorDetails, key); - if (badSlots.size() > 0) { //supervisor contains bad slots + if (badSlots.size() > 0) { // supervisor contains bad slots badSupervisors.put(key, badSlots); } } } else { - cachedSupervisors.put(key, supervisorDetails.getAllPorts()); //new supervisor to cache + cachedSupervisors.put(key, supervisorDetails + .getAllPorts()); // new supervisor to cache } } badSupervisorsToleranceSlidingWindow.add(badSupervisors); @@ -180,7 +196,8 @@ private Set badSlots(SupervisorDetails supervisor, String supervisorKey Set newPorts = Sets.difference(supervisorPorts, cachedSupervisorPorts); if (newPorts.size() > 0) { - // add new ports to cached supervisor. We need a modifiable set to allow removing ports later. + // add new ports to cached supervisor. We need a modifiable set to allow removing ports + // later. Set allPorts = new HashSet<>(newPorts); allPorts.addAll(cachedSupervisorPorts); cachedSupervisors.put(supervisorKey, allPorts); @@ -209,14 +226,15 @@ private Set getBlacklistHosts(Cluster cluster, Set blacklistIds) if (host != null) { blacklistHostSet.add(host); } else { - LOG.info("supervisor {} is not alive, do not need to add to blacklist.", supervisor); + LOG.info("supervisor {} is not alive, do not need to add to blacklist.", + supervisor); } } return blacklistHostSet; } /** - * supervisor or port never exits once in tolerance time will be removed from cache. + * Supervisor or port never exits once in tolerance time will be removed from cache. */ private void removeLongTimeDisappearFromCache() { @@ -247,7 +265,8 @@ private void removeLongTimeDisappearFromCache() { int value = entry.getValue(); if (value == windowSize) { // supervisor which was never back to normal in tolerance period will be removed from cache cachedSupervisors.remove(key); - LOG.info("Supervisor {} was never back to normal during tolerance period, probably dead. Will remove from cache.", key); + LOG.info("Supervisor {} was never back to normal during tolerance period, " + + "probably dead. Will remove from cache.", key); } } @@ -262,7 +281,8 @@ private void removeLongTimeDisappearFromCache() { slots.remove(slot); cachedSupervisors.put(supervisorKey, slots); } - LOG.info("Worker slot {} was never back to normal during tolerance period, probably dead. Will be removed from cache.", + LOG.info("Worker slot {} was never back to normal during tolerance period, " + + "probably dead. Will be removed from cache.", workerSlot); } } @@ -281,7 +301,8 @@ private Object initializeInstance(String className, String representation) { } else if (cause instanceof IllegalAccessException) { LOG.error("Throw IllegalAccessException {} for name {}", representation, className); } else { - LOG.error("Throw unexpected exception {} {} for name {}", cause, representation, className); + LOG.error("Throw unexpected exception {} {} for name {}", cause, representation, + className); } throw e; diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/blacklist/reporters/IReporter.java b/storm-server/src/main/java/org/apache/storm/scheduler/blacklist/reporters/IReporter.java index b1bb933b2fc..1b834039689 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/blacklist/reporters/IReporter.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/blacklist/reporters/IReporter.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -17,7 +23,7 @@ import java.util.Set; /** - * report blacklist to alert system. + * Report blacklist to alert system. */ public interface IReporter { void report(String message); diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/blacklist/reporters/LogReporter.java b/storm-server/src/main/java/org/apache/storm/scheduler/blacklist/reporters/LogReporter.java index 4d3eded41ff..bdaf3aaa342 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/blacklist/reporters/LogReporter.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/blacklist/reporters/LogReporter.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -27,7 +33,8 @@ public void report(String message) { } @Override - public void reportBlacklist(String supervisor, List>> toleranceBuffer) { + public void reportBlacklist(String supervisor, List>> toleranceBuffer) { LOG.warn("add supervisor {} to blacklist. The bad slot history of supervisors is : {}", supervisor, toleranceBuffer); } diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/blacklist/strategies/DefaultBlacklistStrategy.java b/storm-server/src/main/java/org/apache/storm/scheduler/blacklist/strategies/DefaultBlacklistStrategy.java index 675e636edcf..03811b70ee8 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/blacklist/strategies/DefaultBlacklistStrategy.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/blacklist/strategies/DefaultBlacklistStrategy.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -48,15 +54,19 @@ public class DefaultBlacklistStrategy implements IBlacklistStrategy { @Override public void prepare(Map conf) { - toleranceCount = ObjectReader.getInt(conf.get(DaemonConfig.BLACKLIST_SCHEDULER_TOLERANCE_COUNT), + toleranceCount = ObjectReader.getInt(conf + .get(DaemonConfig.BLACKLIST_SCHEDULER_TOLERANCE_COUNT), DEFAULT_BLACKLIST_SCHEDULER_TOLERANCE_COUNT); - resumeTime = ObjectReader.getInt(conf.get(DaemonConfig.BLACKLIST_SCHEDULER_RESUME_TIME), DEFAULT_BLACKLIST_SCHEDULER_RESUME_TIME); + resumeTime = ObjectReader.getInt(conf.get(DaemonConfig.BLACKLIST_SCHEDULER_RESUME_TIME), + DEFAULT_BLACKLIST_SCHEDULER_RESUME_TIME); - String reporterClassName = ObjectReader.getString(conf.get(DaemonConfig.BLACKLIST_SCHEDULER_REPORTER), + String reporterClassName = ObjectReader.getString(conf + .get(DaemonConfig.BLACKLIST_SCHEDULER_REPORTER), LogReporter.class.getName()); reporter = (IReporter) initializeInstance(reporterClassName, "blacklist reporter"); - nimbusMonitorFreqSecs = ObjectReader.getInt(conf.get(DaemonConfig.NIMBUS_MONITOR_FREQ_SECS)); + nimbusMonitorFreqSecs = ObjectReader.getInt(conf + .get(DaemonConfig.NIMBUS_MONITOR_FREQ_SECS)); blacklist = new TreeMap<>(); } @@ -78,7 +88,8 @@ public Set getBlacklist(List>> supervisorsWithF for (Map item : sendAssignmentFailureCount) { for (Map.Entry entry : item.entrySet()) { String supervisorNode = entry.getKey(); - int sendAssignmentFailures = entry.getValue() + countMap.getOrDefault(supervisorNode, 0); + int sendAssignmentFailures = entry.getValue() + countMap + .getOrDefault(supervisorNode, 0); countMap.put(supervisorNode, sendAssignmentFailures); } } @@ -87,7 +98,8 @@ public Set getBlacklist(List>> supervisorsWithF String supervisor = entry.getKey(); int count = entry.getValue(); if (count >= toleranceCount) { - if (!blacklist.containsKey(supervisor)) { // if not in blacklist then add it and set the resume time according to config + if (!blacklist + .containsKey(supervisor)) { // if not in blacklist then add it and set the resume time according to config LOG.debug("Added supervisor {} to blacklist", supervisor); LOG.debug("supervisorsWithFailures : {}", supervisorsWithFailures); LOG.debug("sendAssignmentFailureCount: {}", sendAssignmentFailureCount); @@ -96,7 +108,8 @@ public Set getBlacklist(List>> supervisorsWithF } } } - Set toRelease = releaseBlacklistWhenNeeded(cluster, new ArrayList<>(blacklist.keySet())); + Set toRelease = releaseBlacklistWhenNeeded(cluster, new ArrayList<>(blacklist + .keySet())); // After having computed the final blacklist, // the nodes which are released due to resource shortage will be put to the "greylist". if (toRelease != null) { @@ -123,17 +136,20 @@ public void resumeFromBlacklist() { } for (String key : readyToRemove) { blacklist.remove(key); - LOG.info("Supervisor {} has been blacklisted more than resume period. Removed from blacklist.", key); + LOG.info("Supervisor {} has been blacklisted more than resume period. Removed from " + + "blacklist.", key); } } /** * Decide when/if to release blacklisted hosts. + * * @param cluster the current state of the cluster. * @param blacklistedNodeIds the current set of blacklisted node ids sorted by earliest * @return the set of nodes to be released. */ - protected Set releaseBlacklistWhenNeeded(Cluster cluster, final List blacklistedNodeIds) { + protected Set releaseBlacklistWhenNeeded(Cluster cluster, + final List blacklistedNodeIds) { Set readyToRemove = new HashSet<>(); if (blacklistedNodeIds.size() > 0) { int availableSlots = cluster.getNonBlacklistedAvailableSlots(blacklistedNodeIds).size(); @@ -146,7 +162,7 @@ protected Set releaseBlacklistWhenNeeded(Cluster cluster, final List availableSupervisors = cluster.getSupervisors(); int shortageSlots = neededSlots - availableSlots; LOG.debug("Need {} slots.", neededSlots); @@ -154,10 +170,13 @@ protected Set releaseBlacklistWhenNeeded(Cluster cluster, final List 0) { - LOG.info("Need {} slots more. Releasing some blacklisted nodes to cover it.", shortageSlots); + LOG.info("Need {} slots more. Releasing some blacklisted nodes to cover it.", + shortageSlots); - //release earliest blacklist - but release all supervisors on a given blacklisted host. - Map> hostToSupervisorIds = createHostToSupervisorMap(blacklistedNodeIds, cluster); + // release earliest blacklist - but release all supervisors on a given blacklisted + // host. + Map> hostToSupervisorIds = + createHostToSupervisorMap(blacklistedNodeIds, cluster); for (Set supervisorIds : hostToSupervisorIds.values()) { for (String supervisorId : supervisorIds) { SupervisorDetails sd = availableSupervisors.get(supervisorId); @@ -165,7 +184,8 @@ protected Set releaseBlacklistWhenNeeded(Cluster cluster, final Listhttp://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -27,12 +33,17 @@ public interface IBlacklistStrategy { * * @param badSupervisorsToleranceSlidingWindow bad supervisors buffered in sliding window * @param sendAssignmentFailureCount supervisors with failed assignment calls in sliding window - * @param cluster the cluster these topologies are running in. `cluster` contains everything user - * need to develop a new scheduling logic. e.g. supervisors information, available slots, current - * assignments for all the topologies etc. User can set the new assignment for topologies using + * @param cluster the cluster these topologies are running in. `cluster` contains everything + * user + * need to develop a new scheduling logic. e.g. supervisors information, available slots, + * current + * assignments for all the topologies etc. User can set the new assignment for topologies + * using * cluster.setAssignmentById()` - * @param topologies all the topologies in the cluster, some of them need schedule. Topologies object here - * only contain static information about topologies. Information like assignments, slots are all in + * @param topologies all the topologies in the cluster, some of them need schedule. Topologies + * object here + * only contain static information about topologies. Information like assignments, slots are + * all in * the `cluster` object. * @return blacklisted supervisors' id set */ @@ -41,7 +52,7 @@ Set getBlacklist(List>> badSupervisorsTolerance Cluster cluster, Topologies topologies); /** - * resume supervisors form blacklist. Blacklist is just a temporary list for supervisors, + * Resume supervisors form blacklist. Blacklist is just a temporary list for supervisors, * or there will be less and less available resources. * This will be called every time before getBlacklist() and schedule. */ diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/blacklist/strategies/RasBlacklistStrategy.java b/storm-server/src/main/java/org/apache/storm/scheduler/blacklist/strategies/RasBlacklistStrategy.java index 4273f79aadd..2170aade4e3 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/blacklist/strategies/RasBlacklistStrategy.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/blacklist/strategies/RasBlacklistStrategy.java @@ -22,7 +22,6 @@ import java.util.List; import java.util.Map; import java.util.Set; - import org.apache.storm.generated.InvalidTopologyException; import org.apache.storm.scheduler.Cluster; import org.apache.storm.scheduler.SupervisorDetails; @@ -35,28 +34,33 @@ import org.slf4j.LoggerFactory; /** - * Blacklisting strategy just like the default one, but specifically setup for use with the resource aware scheduler. + * Blacklisting strategy just like the default one, but specifically setup for use with the resource + * aware scheduler. */ public class RasBlacklistStrategy extends DefaultBlacklistStrategy { private static final Logger LOG = LoggerFactory.getLogger(RasBlacklistStrategy.class); @Override - protected Set releaseBlacklistWhenNeeded(Cluster cluster, final List blacklistedNodeIds) { + protected Set releaseBlacklistWhenNeeded(Cluster cluster, + final List blacklistedNodeIds) { LOG.info("RAS We have {} nodes blacklisted...", blacklistedNodeIds.size()); Set readyToRemove = new HashSet<>(); if (blacklistedNodeIds.size() > 0) { int availableSlots = cluster.getNonBlacklistedAvailableSlots(blacklistedNodeIds).size(); int neededSlots = 0; - NormalizedResourceOffer available = cluster.getNonBlacklistedClusterAvailableResources(blacklistedNodeIds); + NormalizedResourceOffer available = cluster + .getNonBlacklistedClusterAvailableResources(blacklistedNodeIds); NormalizedResourceOffer needed = new NormalizedResourceOffer(); for (TopologyDetails td : cluster.getTopologies()) { if (cluster.needsSchedulingRas(td)) { int slots = 0; try { - slots = ServerUtils.getEstimatedWorkerCountForRasTopo(td.getConf(), td.getTopology()); + slots = ServerUtils.getEstimatedWorkerCountForRasTopo(td.getConf(), td + .getTopology()); } catch (InvalidTopologyException e) { - LOG.warn("Could not guess the number of slots needed for {}", td.getName(), e); + LOG.warn("Could not guess the number of slots needed for {}", td.getName(), + e); } int assignedSlots = cluster.getAssignedNumWorkers(td); int tdSlotsNeeded = slots - assignedSlots; @@ -65,11 +69,12 @@ protected Set releaseBlacklistWhenNeeded(Cluster cluster, final List availableSupervisors = cluster.getSupervisors(); NormalizedResourceOffer shortage = new NormalizedResourceOffer(needed); shortage.remove(available, cluster.getResourceMetrics()); @@ -79,20 +84,25 @@ protected Set releaseBlacklistWhenNeeded(Cluster cluster, final List 0) { - LOG.info("Need {} and {} slots more. Releasing some blacklisted nodes to cover it.", shortage, shortageSlots); + LOG.info("Need {} and {} slots more. Releasing some blacklisted nodes to cover it.", + shortage, shortageSlots); - //release earliest blacklist - but release all supervisors on a given blacklisted host. - Map> hostToSupervisorIds = createHostToSupervisorMap(blacklistedNodeIds, cluster); + // release earliest blacklist - but release all supervisors on a given blacklisted + // host. + Map> hostToSupervisorIds = + createHostToSupervisorMap(blacklistedNodeIds, cluster); for (Set supervisorIds : hostToSupervisorIds.values()) { for (String supervisorId : supervisorIds) { SupervisorDetails sd = availableSupervisors.get(supervisorId); if (sd != null) { - NormalizedResourcesWithMemory sdAvailable = cluster.getAvailableResources(sd); + NormalizedResourcesWithMemory sdAvailable = cluster + .getAvailableResources(sd); int sdAvailableSlots = cluster.getAvailablePorts(sd).size(); readyToRemove.add(supervisorId); shortage.remove(sdAvailable, cluster.getResourceMetrics()); shortageSlots -= sdAvailableSlots; - LOG.info("Releasing {} with {} and {} slots leaving {} and {} slots to go", supervisorId, + LOG.info("Releasing {} with {} and {} slots leaving {} and {} slots " + + "to go", supervisorId, sdAvailable, sdAvailableSlots, shortage, shortageSlots); } } diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/multitenant/DefaultPool.java b/storm-server/src/main/java/org/apache/storm/scheduler/multitenant/DefaultPool.java index 05ccfb7929c..902c4a8c191 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/multitenant/DefaultPool.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/multitenant/DefaultPool.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -135,18 +141,20 @@ public void scheduleAsNeeded(NodePool... lesserPools) { int slotsRequested = Math.min(totalTasks, origRequest); int slotsUsed = Node.countSlotsUsed(topId, nodes); int slotsFree = Node.countFreeSlotsAlive(nodes); - //Check to see if we have enough slots before trying to get them + // Check to see if we have enough slots before trying to get them int slotsAvailable = 0; if (slotsRequested > slotsFree) { slotsAvailable = NodePool.slotsAvailable(lesserPools); } int slotsToUse = Math.min(slotsRequested - slotsUsed, slotsFree + slotsAvailable); int executorsNotRunning = cluster.getUnassignedExecutors(td).size(); - LOG.debug("Slots... requested {} used {} free {} available {} to be used {}, executors not running {}", + LOG.debug("Slots... requested {} used {} free {} available {} to be used {}, " + + "executors not running {}", slotsRequested, slotsUsed, slotsFree, slotsAvailable, slotsToUse, executorsNotRunning); if (slotsToUse <= 0) { if (executorsNotRunning > 0) { - cluster.setStatus(topId, "Not fully scheduled (No free slots in default pool) " + cluster.setStatus(topId, + "Not fully scheduled (No free slots in default pool) " + executorsNotRunning + " executors not scheduled"); } else { @@ -154,7 +162,7 @@ public void scheduleAsNeeded(NodePool... lesserPools) { cluster.setStatus(topId, "Running with fewer slots than requested (" + slotsUsed + "/" + origRequest + ")"); - } else { //slotsUsed < origRequest + } else { // slotsUsed < origRequest cluster.setStatus(topId, "Fully Scheduled (requested " + origRequest + " slots, but could only use " + slotsUsed + ")"); @@ -169,7 +177,7 @@ public void scheduleAsNeeded(NodePool... lesserPools) { } if (executorsNotRunning <= 0) { - //There are free slots that we can take advantage of now. + // There are free slots that we can take advantage of now. for (Node n : nodes) { n.freeTopology(topId, cluster); } diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/multitenant/FreePool.java b/storm-server/src/main/java/org/apache/storm/scheduler/multitenant/FreePool.java index 867baa4a032..584c0f02a29 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/multitenant/FreePool.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/multitenant/FreePool.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -107,7 +113,7 @@ public NodeAndSlotCounts getNodeAndSlotCountIfSlotsWereTaken(int slotsNeeded) { @Override public void scheduleAsNeeded(NodePool... lesserPools) { - //No topologies running so NOOP + // No topologies running so NOOP } @Override diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/multitenant/IsolatedPool.java b/storm-server/src/main/java/org/apache/storm/scheduler/multitenant/IsolatedPool.java index 77f9a3d3990..879cfadc12e 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/multitenant/IsolatedPool.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/multitenant/IsolatedPool.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -16,8 +22,8 @@ import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; -import java.util.Map; import java.util.Map.Entry; +import java.util.Map; import java.util.Set; import org.apache.storm.Config; import org.apache.storm.scheduler.SchedulerAssignment; @@ -64,7 +70,7 @@ public void addTopology(TopologyDetails td) { @Override public boolean canAdd(TopologyDetails td) { - //Only add topologies that are not sharing nodes with other topologies + // Only add topologies that are not sharing nodes with other topologies String topId = td.getId(); SchedulerAssignment assignment = cluster.getAssignmentById(topId); if (assignment != null) { @@ -90,7 +96,8 @@ public void scheduleAsNeeded(NodePool... lesserPools) { +nodesRequested.intValue()); } if (cluster.needsScheduling(td) - || (effectiveNodesRequested != null && allNodes.size() != effectiveNodesRequested)) { + || (effectiveNodesRequested != null && allNodes + .size() != effectiveNodesRequested)) { LOG.debug("Scheduling topology {}", topId); int slotsToUse = 0; if (effectiveNodesRequested == null) { @@ -99,7 +106,7 @@ public void scheduleAsNeeded(NodePool... lesserPools) { slotsToUse = getNodesForIsolatedTop(td, allNodes, lesserPools, effectiveNodesRequested); } - //No slots to schedule for some reason, so skip it. + // No slots to schedule for some reason, so skip it. if (slotsToUse <= 0) { continue; } @@ -149,6 +156,7 @@ private Node findBestNode(Collection nodes) { /** * Get the nodes needed to schedule an isolated topology. + * * @param td the topology to be scheduled * @param allNodes the nodes already scheduled for this topology. * This will be updated to include new nodes if needed. @@ -181,7 +189,7 @@ private int getNodesForIsolatedTop(TopologyDetails td, Set allNodes, return 0; } - //In order to avoid going over maxNodes I may need to steal from + // In order to avoid going over maxNodes I may need to steal from // myself even though other pools have free nodes. so figure out how // much each group should provide int nodesNeededFromOthers = Math.min(Math.min(maxNodes - usedNodes, @@ -195,7 +203,7 @@ private int getNodesForIsolatedTop(TopologyDetails td, Set allNodes, return 0; } - //Get the nodes + // Get the nodes Collection found = NodePool.takeNodes(nodesNeededFromOthers, lesserPools); usedNodes += found.size(); allNodes.addAll(found); @@ -224,8 +232,11 @@ private int getNodesForIsolatedTop(TopologyDetails td, Set allNodes, + " total slots"); } else { // if # of workers requested is less than we took - // then we know some workers we track died, since we have more workers than we are supposed to have - cluster.setStatus(topId, "Node has partially crashed, if this situation persists rebalance the topology."); + // then we know some workers we track died, since we have more workers than we are + // supposed to have + cluster.setStatus(topId, + "Node has partially crashed, if this situation persists rebalance the " + + "topology."); } } return slotsToUse; @@ -233,6 +244,7 @@ private int getNodesForIsolatedTop(TopologyDetails td, Set allNodes, /** * Get the nodes needed to schedule a non-isolated topology. + * * @param td the topology to be scheduled * @param allNodes the nodes already scheduled for this topology. * This will be updated to include new nodes if needed. @@ -248,7 +260,7 @@ private int getNodesForNotIsolatedTop(TopologyDetails td, Set allNodes, int slotsRequested = Math.min(totalTasks, origRequest); int slotsUsed = Node.countSlotsUsed(topId, allNodes); int slotsFree = Node.countFreeSlotsAlive(allNodes); - //Check to see if we have enough slots before trying to get them + // Check to see if we have enough slots before trying to get them int slotsAvailable = 0; if (slotsRequested > slotsFree) { slotsAvailable = NodePool.slotsAvailable(lesserPools); diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/multitenant/MultitenantScheduler.java b/storm-server/src/main/java/org/apache/storm/scheduler/multitenant/MultitenantScheduler.java index 2121d0c17e3..4cbdfb43f07 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/multitenant/MultitenantScheduler.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/multitenant/MultitenantScheduler.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -44,7 +50,9 @@ public void prepare(Map conf, StormMetricsRegistry metricsRegist /** * Load from configLoaders first; if no config available, read from multitenant-scheduler.yaml; - * if no config available from multitenant-scheduler.yaml, get configs from conf. Only one will be used. + * if no config available from multitenant-scheduler.yaml, get configs from conf. Only one will + * be used. + * * @return User pool configs. */ private Map loadConfig() { @@ -52,11 +60,13 @@ private Map loadConfig() { // Try the loader plugin, if configured if (configLoader != null) { - ret = (Map) configLoader.load(DaemonConfig.MULTITENANT_SCHEDULER_USER_POOLS); + ret = (Map) configLoader + .load(DaemonConfig.MULTITENANT_SCHEDULER_USER_POOLS); if (ret != null) { return ret; } else { - LOG.warn("Config loader returned null. Will try to read from multitenant-scheduler.yaml"); + LOG.warn("Config loader returned null. Will try to read from " + + "multitenant-scheduler.yaml"); } } @@ -66,7 +76,8 @@ private Map loadConfig() { if (ret != null) { return ret; } else { - LOG.warn("Reading from multitenant-scheduler.yaml returned null. This could because the file is not available. " + LOG.warn("Reading from multitenant-scheduler.yaml returned null. This could because " + + "the file is not available. " + "Will load configs from storm configuration"); } @@ -87,7 +98,7 @@ public Map config() { @Override public void schedule(Topologies topologies, Cluster cluster) { LOG.debug("Rerunning scheduling..."); - //refresh the config every time before scheduling + // refresh the config every time before scheduling schedulerConfigCache.refresh(); Map nodeIdToNode = Node.getAllNodesFrom(cluster); @@ -117,7 +128,7 @@ public void schedule(Topologies topologies, Cluster cluster) { pool.addTopology(td); } - //Now schedule all of the topologies that need to be scheduled + // Now schedule all of the topologies that need to be scheduled for (IsolatedPool pool : userPools.values()) { pool.scheduleAsNeeded(freePool, defaultPool); } diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/multitenant/Node.java b/storm-server/src/main/java/org/apache/storm/scheduler/multitenant/Node.java index 0c2ba56df5b..c0ff732f085 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/multitenant/Node.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/multitenant/Node.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -16,8 +22,8 @@ import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; -import java.util.Map; import java.util.Map.Entry; +import java.util.Map; import java.util.Set; import org.apache.storm.scheduler.Cluster; import org.apache.storm.scheduler.ExecutorDetails; @@ -96,7 +102,7 @@ public static int countTotalSlotsAlive(Collection nodes) { public static Map getAllNodesFrom(Cluster cluster) { Map nodeIdToNode = new HashMap<>(); for (SupervisorDetails sup : cluster.getSupervisors().values()) { - //Node ID and supervisor ID are the same. + // Node ID and supervisor ID are the same. String id = sup.getId(); boolean isAlive = !cluster.isBlackListed(id); LOG.debug("Found a {} Node {} {}", @@ -115,7 +121,8 @@ public static Map getAllNodesFrom(Cluster cluster) { nodeIdToNode.put(id, node); } if (!node.isAlive()) { - //The supervisor on the node down so add an orphaned slot to hold the unsupervised worker + // The supervisor on the node down so add an orphaned slot to hold the + // unsupervised worker node.addOrphanedSlot(ws); } if (node.assignInternal(ws, topId, true)) { @@ -142,6 +149,7 @@ public boolean isAlive() { /** * Get running topologies. + * * @return a collection of the topology ids currently running on this node */ public Collection getRunningTopologies() { @@ -244,6 +252,7 @@ boolean assignInternal(WorkerSlot ws, String topId, boolean dontThrow) { /** * Free all slots on this node. This will update the Cluster too. + * * @param cluster the cluster to be updated */ public void freeAllSlots(Cluster cluster) { @@ -261,6 +270,7 @@ public void freeAllSlots(Cluster cluster) { /** * Frees a single slot in this node. + * * @param ws the slot to free * @param cluster the cluster to update */ @@ -294,6 +304,7 @@ public void free(WorkerSlot ws, Cluster cluster, boolean forceFree) { /** * Frees all the slots for a topology. + * * @param topId the topology to free slots for * @param cluster the cluster to update */ @@ -314,6 +325,7 @@ public void freeTopology(String topId, Cluster cluster) { /** * Assign a free slot on the node to the following topology and executors. * This will update the cluster too. + * * @param topId the topology to assign a free slot to. * @param executors the executors to run in that slot. * @param cluster the cluster to be updated diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/multitenant/NodePool.java b/storm-server/src/main/java/org/apache/storm/scheduler/multitenant/NodePool.java index 477f2430965..e67f9806b34 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/multitenant/NodePool.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/multitenant/NodePool.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -18,8 +24,8 @@ import java.util.HashSet; import java.util.LinkedList; import java.util.List; -import java.util.Map; import java.util.Map.Entry; +import java.util.Map; import java.util.Set; import org.apache.storm.Config; import org.apache.storm.scheduler.Cluster; @@ -48,6 +54,7 @@ public static int slotsAvailable(NodePool[] pools) { /** * Get number of available slots. + * * @return the number of slots that are available to be taken */ public abstract int slotsAvailable(); @@ -62,6 +69,7 @@ public static int nodesAvailable(NodePool[] pools) { /** * Get the number of available nodes. + * * @return the number of nodes that are available to be taken */ public abstract int nodesAvailable(); @@ -113,6 +121,7 @@ public static int getNodeCountIfSlotsWereTaken(int slots, NodePool[] pools) { /** * Initialize the pool. + * * @param cluster the cluster * @param nodeIdToNode the mapping of node id to nodes */ @@ -123,12 +132,14 @@ public void init(Cluster cluster, Map nodeIdToNode) { /** * Add a topology to the pool. + * * @param td the topology to add */ public abstract void addTopology(TopologyDetails td); /** * Check if this topology can be added to this pool. + * * @param td the topology * @return true if it can else false */ @@ -136,6 +147,7 @@ public void init(Cluster cluster, Map nodeIdToNode) { /** * Take nodes from this pool that can fulfill possibly up to the slotsNeeded. + * * @param slotsNeeded the number of slots that are needed. * @return a Collection of nodes with the removed nodes in it. * This may be empty, but should not be null. @@ -144,6 +156,7 @@ public void init(Cluster cluster, Map nodeIdToNode) { /** * Get the number of nodes and slots this would provide to get the slots needed. + * * @param slots the number of slots needed * @return the number of nodes and slots that would be returned. */ @@ -151,16 +164,18 @@ public void init(Cluster cluster, Map nodeIdToNode) { /** * Take up to nodesNeeded from this pool. + * * @param nodesNeeded the number of nodes that are needed. * @return a Collection of nodes with the removed nodes in it. * This may be empty, but should not be null. */ @SuppressWarnings("checkstyle:OverloadMethodsDeclarationOrder") - //simply suppress until https://github.com/checkstyle/checkstyle/issues/3770 is resolved + // simply suppress until https://github.com/checkstyle/checkstyle/issues/3770 is resolved public abstract Collection takeNodes(int nodesNeeded); /** * Reschedule any topologies as needed. + * * @param lesserPools pools that may be used to steal nodes from. */ public abstract void scheduleAsNeeded(NodePool... lesserPools); @@ -189,6 +204,7 @@ public static class RoundRobinSlotScheduler { /** * Create a new scheduler for a given topology. + * * @param td the topology to schedule * @param slotsToUse the number of slots to use for the executors left to * schedule. @@ -218,7 +234,8 @@ public RoundRobinSlotScheduler(TopologyDetails td, int slotsToUse, } spreadToSchedule = new HashMap<>(); - List spreadComps = (List) td.getConf().get(Config.TOPOLOGY_SPREAD_COMPONENTS); + List spreadComps = (List) td.getConf() + .get(Config.TOPOLOGY_SPREAD_COMPONENTS); if (spreadComps != null) { for (String comp : spreadComps) { spreadToSchedule.put(comp, new ArrayList()); @@ -231,7 +248,8 @@ public RoundRobinSlotScheduler(TopologyDetails td, int slotsToUse, } int at = 0; - for (Entry> entry : this.cluster.getNeedsSchedulingComponentToExecutors(td).entrySet()) { + for (Entry> entry : this.cluster + .getNeedsSchedulingComponentToExecutors(td).entrySet()) { LOG.debug("Scheduling for {}", entry.getKey()); if (spreadToSchedule.containsKey(entry.getKey())) { LOG.debug("Saving {} for spread...", entry.getKey()); @@ -252,6 +270,7 @@ public RoundRobinSlotScheduler(TopologyDetails td, int slotsToUse, /** * Assign a slot to the given node. + * * @param n the node to assign a slot to. * @return true if there are more slots to assign else false. */ @@ -261,7 +280,7 @@ public boolean assignSlotTo(Node n) { } Set slot = slots.pop(); if (slot == lastSlot) { - //The last slot fill it up + // The last slot fill it up for (Entry> entry : spreadToSchedule.entrySet()) { if (entry.getValue().size() > 0) { slot.addAll(entry.getValue()); diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/resource/RasNode.java b/storm-server/src/main/java/org/apache/storm/scheduler/resource/RasNode.java index afdb9b288a7..32b320ff7d7 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/resource/RasNode.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/resource/RasNode.java @@ -23,10 +23,9 @@ import java.util.HashMap; import java.util.HashSet; import java.util.LinkedList; -import java.util.Map; import java.util.Map.Entry; +import java.util.Map; import java.util.Set; - import org.apache.storm.Config; import org.apache.storm.scheduler.Cluster; import org.apache.storm.scheduler.ExecutorDetails; @@ -46,12 +45,14 @@ public class RasNode implements Comparable { private final String nodeId; private final Cluster cluster; private final Set originallyFreeSlots; - //A map consisting of all workers on the node. - //The key of the map is the worker id and the value is the corresponding workerslot object + // A map consisting of all workers on the node. + // The key of the map is the worker id and the value is the corresponding workerslot object private Map slots = new HashMap<>(); - // A map describing which topologies are using which slots on this node. The format of the map is the following: + // A map describing which topologies are using which slots on this node. The format of the map + // is the following: // {TopologyId -> {WorkerId -> {Executors}}} - private Map>> topIdToUsedSlots = new HashMap<>(); + private Map>> topIdToUsedSlots = + new HashMap<>(); private String hostname; private boolean isAlive; private SupervisorDetails sup; @@ -59,6 +60,7 @@ public class RasNode implements Comparable { /** * Create a new node. + * * @param nodeId the id of the node. * @param sup the supervisor this is for. * @param cluster the cluster this is a part of. @@ -71,7 +73,7 @@ public RasNode( Cluster cluster, Map workerIdToWorker, Map>> assignmentMap) { - //Node ID and supervisor ID are the same. + // Node ID and supervisor ID are the same. this.nodeId = nodeId; if (sup == null) { isAlive = false; @@ -86,12 +88,12 @@ public RasNode( slots = workerIdToWorker; } - //initialize assignment map + // initialize assignment map if (assignmentMap != null) { topIdToUsedSlots = assignmentMap; } - //check if node is alive + // check if node is alive if (isAlive && sup != null) { hostname = sup.getHost(); this.sup = sup; @@ -129,6 +131,7 @@ private Collection workerIdsToWorkers(Collection workerIds) /** * Get the IDs of all free slots on this node. + * * @return the ids of the free slots. */ public Collection getFreeSlotsId() { @@ -162,6 +165,7 @@ public Collection getUsedSlots() { /** * Get slots used by the given topology. + * * @param topId the id of the topology to get. * @return the slots currently assigned to that topology on this node. */ @@ -212,12 +216,12 @@ public void freeAllSlots() { LOG.warn("Freeing all slots on a dead node {} ", nodeId); } cluster.freeSlots(slots.values()); - //clearing assignments + // clearing assignments topIdToUsedSlots.clear(); } /** - * frees a single executor. + * Frees a single executor. * * @param exec is the executor to free * @param topo the topology the executor is a part of @@ -262,12 +266,13 @@ public void free(WorkerSlot ws) { TopologyDetails topo = findTopologyUsingWorker(ws); if (topo == null) { - throw new IllegalArgumentException("Tried to free a slot " + ws + " that was already free!"); + throw new IllegalArgumentException("Tried to free a slot " + ws + + " that was already free!"); } - //free slot + // free slot cluster.freeSlot(ws); - //cleanup internal assignments + // cleanup internal assignments topIdToUsedSlots.get(topo.getId()).remove(ws.getId()); } @@ -297,7 +302,8 @@ private TopologyDetails findTopologyUsingWorker(WorkerSlot ws) { * @param td the topology the executors are from * @param executors executors to assign to the specified worker slot */ - public void assign(WorkerSlot target, TopologyDetails td, Collection executors) { + public void assign(WorkerSlot target, TopologyDetails td, + Collection executors) { if (!isAlive) { throw new IllegalStateException("Trying to adding to a dead node " + nodeId); } @@ -306,7 +312,8 @@ public void assign(WorkerSlot target, TopologyDetails td, Collection new HashMap<>()) .computeIfAbsent(target.getId(), (tid) -> new LinkedList<>()) .addAll(executors); @@ -327,6 +334,7 @@ public void assign(WorkerSlot target, TopologyDetails td, Collection components = new HashSet<>(); - Map> topologyExecutors = topIdToUsedSlots.get(td.getId()); + Map> topologyExecutors = topIdToUsedSlots.get(td + .getId()); if (topologyExecutors != null) { Collection slotExecs = topologyExecutors.get(ws.getId()); if (slotExecs != null) { @@ -405,9 +416,11 @@ public boolean wouldFit(WorkerSlot ws, ExecutorDetails exec, TopologyDetails td) /** * Is there any possibility that exec could ever fit on this node. + * * @param exec the executor to schedule * @param td the topology the executor is a part of - * @return true if there is the possibility it might fit, no guarantee that it will, or false if there is no + * @return true if there is the possibility it might fit, no guarantee that it will, or false if + * there is no * way it would ever fit. */ public boolean couldEverFit(ExecutorDetails exec, TopologyDetails td) { @@ -478,10 +491,13 @@ public NormalizedResourceOffer getTotalResources() { */ public NormalizedResourceOffer getTotalAvailableResources() { if (sup != null) { - NormalizedResourceOffer availableResources = new NormalizedResourceOffer(sup.getTotalResources()); - if (availableResources.remove(cluster.getAllScheduledResourcesForNode(sup.getId()), cluster.getResourceMetrics())) { + NormalizedResourceOffer availableResources = new NormalizedResourceOffer(sup + .getTotalResources()); + if (availableResources.remove(cluster.getAllScheduledResourcesForNode(sup.getId()), + cluster.getResourceMetrics())) { if (!loggedUnderageUsage) { - LOG.error("Resources on {} became negative and was clamped to 0 {}.", hostname, availableResources); + LOG.error("Resources on {} became negative and was clamped to 0 {}.", hostname, + availableResources); loggedUnderageUsage = true; } } diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/resource/RasNodes.java b/storm-server/src/main/java/org/apache/storm/scheduler/resource/RasNodes.java index 645c4e066b3..dda49126cb1 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/resource/RasNodes.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/resource/RasNodes.java @@ -43,10 +43,10 @@ public RasNodes(Cluster cluster) { public static Map getAllNodesFrom(Cluster cluster) { - //A map of node ids to node objects + // A map of node ids to node objects Map nodeIdToNode = new HashMap<>(); - //A map of assignments organized by node with the following format: - //{nodeId -> {topologyId -> {workerId -> {execs}}}} + // A map of assignments organized by node with the following format: + // {nodeId -> {topologyId -> {workerId -> {execs}}}} Map>>> assignmentRelationshipMap = new HashMap<>(); Map> workerIdToWorker = new HashMap<>(); @@ -80,7 +80,7 @@ public static Map getAllNodesFrom(Cluster cluster) { } for (SupervisorDetails sup : cluster.getSupervisors().values()) { - //Initialize a worker slot for every port even if there is no assignment to it + // Initialize a worker slot for every port even if there is no assignment to it for (int port : sup.getAllPorts()) { WorkerSlot worker = new WorkerSlot(sup.getId(), port); if (!workerIdToWorker.containsKey(sup.getId())) { @@ -100,7 +100,7 @@ public static Map getAllNodesFrom(Cluster cluster) { assignmentRelationshipMap.get(sup.getId()))); } - //Add in supervisors that might have crashed but workers are still alive + // Add in supervisors that might have crashed but workers are still alive for (Map.Entry>>> entry : assignmentRelationshipMap.entrySet()) { String nodeId = entry.getKey(); @@ -111,14 +111,15 @@ public static Map getAllNodesFrom(Cluster cluster) { nodeId, assignments); nodeIdToNode.put( - nodeId, new RasNode(nodeId, null, cluster, workerIdToWorker.get(nodeId), assignments)); + nodeId, new RasNode(nodeId, null, cluster, workerIdToWorker.get(nodeId), + assignments)); } } return nodeIdToNode; } /** - * get node object from nodeId. + * Get node object from nodeId. */ public RasNode getNodeById(String nodeId) { return this.nodeMap.get(nodeId); @@ -152,7 +153,8 @@ public Collection getNodes() { public Map> getHostnameToNodes() { Map> hostnameToNodes = new HashMap<>(); nodeMap.values() - .forEach(node -> hostnameToNodes.computeIfAbsent(node.getHostname(), (hn) -> new ArrayList<>()).add(node)); + .forEach(node -> hostnameToNodes.computeIfAbsent(node.getHostname(), + (hn) -> new ArrayList<>()).add(node)); return hostnameToNodes; } diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/resource/ResourceAwareScheduler.java b/storm-server/src/main/java/org/apache/storm/scheduler/resource/ResourceAwareScheduler.java index a113a425db0..f12069f8fc0 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/resource/ResourceAwareScheduler.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/resource/ResourceAwareScheduler.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

      - * http://www.apache.org/licenses/LICENSE-2.0 - *

      - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

      http://www.apache.org/licenses/LICENSE-2.0 + * + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -69,7 +75,8 @@ private static void markFailedTopology(User u, Cluster c, TopologyDetails td, St markFailedTopology(u, c, td, message, null); } - private static void markFailedTopology(User u, Cluster c, TopologyDetails td, String message, Throwable t) { + private static void markFailedTopology(User u, Cluster c, TopologyDetails td, String message, + Throwable t) { c.setStatus(td, message); String realMessage = td.getId() + " " + message; if (t != null) { @@ -112,24 +119,27 @@ public Map> config() { @Override public void schedule(Topologies topologies, Cluster cluster) { - //refresh the config every time before scheduling + // refresh the config every time before scheduling schedulerConfigCache.refresh(); Map userMap = getUsers(cluster); - List orderedTopologies = new ArrayList<>(schedulingPriorityStrategy.getOrderedTopologies(cluster, userMap)); + List orderedTopologies = new ArrayList<>(schedulingPriorityStrategy + .getOrderedTopologies(cluster, userMap)); if (LOG.isDebugEnabled()) { - LOG.debug("Ordered list of topologies is: {}", orderedTopologies.stream().map((t) -> t.getId()).collect(Collectors.toList())); + LOG.debug("Ordered list of topologies is: {}", orderedTopologies.stream().map((t) -> t + .getId()).collect(Collectors.toList())); } // clear tmpEvictedTopologiesMap at the beginning of each round of scheduling // move it to evictedTopologiesMap at the end of this round of scheduling Map> tmpEvictedTopologiesMap = new HashMap<>(); for (TopologyDetails td : orderedTopologies) { if (!cluster.needsSchedulingRas(td)) { - //cluster forgets about its previous status, so if it is scheduled just leave it. + // cluster forgets about its previous status, so if it is scheduled just leave it. cluster.setStatusIfAbsent(td.getId(), "Fully Scheduled"); } else { User submitter = userMap.get(td.getTopologySubmitter()); - scheduleTopology(td, cluster, submitter, orderedTopologies, tmpEvictedTopologiesMap); + scheduleTopology(td, cluster, submitter, orderedTopologies, + tmpEvictedTopologiesMap); } } evictedTopologiesMap = tmpEvictedTopologiesMap; @@ -137,7 +147,8 @@ public void schedule(Topologies topologies, Cluster cluster) { private void scheduleTopology(TopologyDetails td, Cluster cluster, final User topologySubmitter, List orderedTopologies, Map> tmpEvictedTopologiesMap) { - //A copy of cluster that we can modify, but does not get committed back to cluster unless scheduling succeeds + // A copy of cluster that we can modify, but does not get committed back to cluster unless + // scheduling succeeds Cluster workingState = new Cluster(cluster); RasNodes nodes = new RasNodes(workingState); IStrategy rasStrategy = null; @@ -146,9 +157,11 @@ private void scheduleTopology(TopologyDetails td, Cluster cluster, final User to String strategy = (String) td.getConf().get(Config.TOPOLOGY_SCHEDULER_STRATEGY); if (strategy.startsWith("backtype.storm")) { // Storm support to launch workers of older version. - // If the config of TOPOLOGY_SCHEDULER_STRATEGY comes from the older version, replace the package name. + // If the config of TOPOLOGY_SCHEDULER_STRATEGY comes from the older version, + // replace the package name. strategy = strategy.replace("backtype.storm", "org.apache.storm"); - LOG.debug("Replaced backtype.storm with org.apache.storm for Config.TOPOLOGY_SCHEDULER_STRATEGY"); + LOG.debug("Replaced backtype.storm with org.apache.storm for " + + "Config.TOPOLOGY_SCHEDULER_STRATEGY"); } rasStrategy = ReflectionUtils.newSchedulerStrategyInstance(strategy, conf); rasStrategy.prepare(conf); @@ -160,26 +173,31 @@ private void scheduleTopology(TopologyDetails td, Cluster cluster, final User to + " config is one of the allowed strategies: " + e.getAllowedStrategies() + ", or ask your administrator to add " + e.getAttemptedClass() - + " to the nimbus config " + Config.NIMBUS_SCHEDULER_STRATEGY_CLASS_WHITELIST, e); + + " to the nimbus config " + + Config.NIMBUS_SCHEDULER_STRATEGY_CLASS_WHITELIST, e); return; } catch (RuntimeException e) { markFailedTopology(topologySubmitter, cluster, td, - "Unsuccessful in scheduling - failed to create instance of topology strategy " + "Unsuccessful in scheduling - failed to create instance of " + + "topology strategy " + strategyConf + ". Please check logs for details", e); return; } // Log warning here to avoid duplicating / spamming in strategy / scheduling code. - boolean oneExecutorPerWorker = (Boolean) td.getConf().get(Config.TOPOLOGY_RAS_ONE_EXECUTOR_PER_WORKER); - boolean oneComponentPerWorker = (Boolean) td.getConf().get(Config.TOPOLOGY_RAS_ONE_COMPONENT_PER_WORKER); + boolean oneExecutorPerWorker = (Boolean) td.getConf() + .get(Config.TOPOLOGY_RAS_ONE_EXECUTOR_PER_WORKER); + boolean oneComponentPerWorker = (Boolean) td.getConf() + .get(Config.TOPOLOGY_RAS_ONE_COMPONENT_PER_WORKER); if (oneExecutorPerWorker && oneComponentPerWorker) { LOG.warn("Conflicting options: {} and {} are both set! Ignoring {} option.", Config.TOPOLOGY_RAS_ONE_EXECUTOR_PER_WORKER, Config.TOPOLOGY_RAS_ONE_COMPONENT_PER_WORKER, Config.TOPOLOGY_RAS_ONE_COMPONENT_PER_WORKER); } - TopologySchedulingResources topologySchedulingResources = new TopologySchedulingResources(workingState, td); + TopologySchedulingResources topologySchedulingResources = + new TopologySchedulingResources(workingState, td); final IStrategy finalRasStrategy = rasStrategy; for (int i = 0; i < maxSchedulingAttempts; i++) { SingleTopologyCluster toSchedule = new SingleTopologyCluster(workingState, td.getId()); @@ -192,8 +210,10 @@ private void scheduleTopology(TopologyDetails td, Cluster cluster, final User to try { result = schedulingFuture.get(schedulingTimeoutSeconds, TimeUnit.SECONDS); } catch (TimeoutException te) { - markFailedTopology(topologySubmitter, cluster, td, "Scheduling took too long for " - + td.getId() + " using strategy " + rasStrategy.getClass().getName() + " timeout after " + markFailedTopology(topologySubmitter, cluster, td, + "Scheduling took too long for " + + td.getId() + " using strategy " + rasStrategy.getClass() + .getName() + " timeout after " + schedulingTimeoutSeconds + " seconds using config " + DaemonConfig.SCHEDULING_TIMEOUT_SECONDS_PER_TOPOLOGY + "."); schedulingTimeoutMeter.mark(); @@ -201,7 +221,8 @@ private void scheduleTopology(TopologyDetails td, Cluster cluster, final User to return; } } else { - result = SchedulingResult.failure(SchedulingStatus.FAIL_NOT_ENOUGH_RESOURCES, ""); + result = SchedulingResult.failure(SchedulingStatus.FAIL_NOT_ENOUGH_RESOURCES, + ""); } LOG.debug("scheduling result: {}", result); if (result == null) { @@ -211,44 +232,55 @@ private void scheduleTopology(TopologyDetails td, Cluster cluster, final User to if (result.isSuccess()) { cluster.updateFrom(toSchedule); cluster.setStatus(td.getId(), "Running - " + result.getMessage()); - //DONE + // DONE return; } else if (result.getStatus() == SchedulingStatus.FAIL_NOT_ENOUGH_RESOURCES) { LOG.debug("Not enough resources to schedule {}", td.getName()); - List reversedList = ImmutableList.copyOf(orderedTopologies).reverse(); - LOG.debug("Attempting to make space for topo {} from user {}", td.getName(), td.getTopologySubmitter()); + List reversedList = ImmutableList.copyOf(orderedTopologies) + .reverse(); + LOG.debug("Attempting to make space for topo {} from user {}", td.getName(), + td.getTopologySubmitter()); int tdIndex = reversedList.indexOf(td); topologySchedulingResources.setRemainingRequiredResources(toSchedule, td); Set tmpEvictedTopos = new HashSet<>(); for (int index = 0; index < tdIndex; index++) { TopologyDetails topologyEvict = reversedList.get(index); - SchedulerAssignment evictAssignemnt = workingState.getAssignmentById(topologyEvict.getId()); + SchedulerAssignment evictAssignemnt = workingState + .getAssignmentById(topologyEvict.getId()); if (evictAssignemnt != null && !evictAssignemnt.getSlots().isEmpty()) { - topologySchedulingResources.adjustResourcesForEvictedTopology(toSchedule, topologyEvict); + topologySchedulingResources + .adjustResourcesForEvictedTopology(toSchedule, + topologyEvict); tmpEvictedTopos.add(topologyEvict.getId()); - Collection workersToEvict = workingState.getUsedSlotsByTopologyId(topologyEvict.getId()); + Collection workersToEvict = workingState + .getUsedSlotsByTopologyId(topologyEvict.getId()); nodes.freeSlots(workersToEvict); if (topologySchedulingResources.canSchedule()) { - //We evicted enough topologies to have a hope of scheduling, so try it now, and don't evict more + // We evicted enough topologies to have a hope of scheduling, so + // try it now, and don't evict more // than is needed break; } } } if (!tmpEvictedTopos.isEmpty()) { - LOG.warn("Evicted Topologies {} when scheduling topology: {}", tmpEvictedTopos, td.getId()); - tmpEvictedTopologiesMap.computeIfAbsent(td.getId(), k -> new HashSet<>()).addAll(tmpEvictedTopos); + LOG.warn("Evicted Topologies {} when scheduling topology: {}", + tmpEvictedTopos, td.getId()); + tmpEvictedTopologiesMap.computeIfAbsent(td.getId(), + k -> new HashSet<>()).addAll(tmpEvictedTopos); } else { StringBuilder message = new StringBuilder(); - message.append("Not enough resources to schedule after evicting lower priority topologies. "); - message.append(topologySchedulingResources.getRemainingRequiredResourcesMessage()); + message.append("Not enough resources to schedule after evicting lower " + + "priority topologies. "); + message.append(topologySchedulingResources + .getRemainingRequiredResourcesMessage()); message.append(result.getErrorMessage()); markFailedTopology(topologySubmitter, cluster, td, message.toString()); return; } - //Only place we fall though to do the loop over again... - } else { //Any other failure result + // Only place we fall though to do the loop over again... + } else { // Any other failure result topologySubmitter.markTopoUnsuccess(td, cluster, result.toString()); return; } @@ -256,15 +288,18 @@ private void scheduleTopology(TopologyDetails td, Cluster cluster, final User to } catch (Exception ex) { internalErrorMeter.mark(); markFailedTopology(topologySubmitter, cluster, td, - "Internal Error - Exception thrown when scheduling. Please check logs for details", ex); + "Internal Error - Exception thrown when scheduling. Please check logs for " + + "details", ex); return; } } - // We can only reach here when we failed to free enough space by evicting current topologies after {maxSchedulingAttempts} + // We can only reach here when we failed to free enough space by evicting current topologies + // after {maxSchedulingAttempts} // while that scheduler did evict something at each attempt. markFailedTopology(topologySubmitter, cluster, td, "Failed to make enough resources for " + td.getId() - + " by evicting lower priority topologies within " + maxSchedulingAttempts + " attempts. " + + " by evicting lower priority topologies within " + maxSchedulingAttempts + + " attempts. " + topologySchedulingResources.getRemainingRequiredResourcesMessage()); } @@ -272,26 +307,30 @@ private void scheduleTopology(TopologyDetails td, Cluster cluster, final User to * Return eviction information as map {scheduled topo : evicted topos} * NOTE this method returns the map of a completed scheduling round. * If scheduling is going on, this method will return a map of last scheduling round - *

      - * TODO: This method is only used for testing . It's subject to change if we plan to use this info elsewhere. + * + *

      TODO: This method is only used for testing . It's subject to change if we plan to use this + * info elsewhere. *

      + * * @return a MAP of scheduled (topo : evicted) topos of most recent completed scheduling round */ public Map> getEvictedTopologiesMap() { return Collections.unmodifiableMap(evictedTopologiesMap); } - /* * Class for tracking resources for scheduling a topology. * - * Ideally we would simply track NormalizedResources, but shared topology memory complicates things. - * Topologies with shared memory may use more than the SharedMemoryLowerBound, and topologyRequiredResources + * Ideally we would simply track NormalizedResources, but shared topology memory complicates + * things. + * Topologies with shared memory may use more than the SharedMemoryLowerBound, and + * topologyRequiredResources * ignores shared memory. * * Resources are tracked in two ways: * 1) AvailableResources. Track cluster available resources and required topology resources. - * 2) RemainingRequiredResources. Start with required topology resources, and deduct for partially scheduled and evicted topologies. + * 2) RemainingRequiredResources. Start with required topology resources, and deduct for + * partially scheduled and evicted topologies. */ private class TopologySchedulingResources { boolean remainingResourcesAreSet; @@ -312,12 +351,15 @@ private class TopologySchedulingResources { remainingResourcesAreSet = false; // available resources (lower bound since blacklisted supervisors do not contribute) - clusterAvailableResources = cluster.getNonBlacklistedClusterAvailableResources(Collections.emptyList()); + clusterAvailableResources = cluster + .getNonBlacklistedClusterAvailableResources(Collections.emptyList()); clusterAvailableMemory = clusterAvailableResources.getTotalMemoryMb(); // required resources topologyRequiredResources = td.getApproximateTotalResources(); - topologyRequiredNonSharedMemory = td.getRequestedNonSharedOffHeap() + td.getRequestedNonSharedOnHeap(); - topologySharedMemoryLowerBound = td.getRequestedSharedOffHeap() + td.getRequestedSharedOnHeap(); + topologyRequiredNonSharedMemory = td.getRequestedNonSharedOffHeap() + td + .getRequestedNonSharedOnHeap(); + topologySharedMemoryLowerBound = td.getRequestedSharedOffHeap() + td + .getRequestedSharedOnHeap(); // partially scheduled topology resources setScheduledTopologyResources(cluster, td); } @@ -338,7 +380,8 @@ boolean canSchedule() { } boolean canScheduleAvailable() { - NormalizedResourceOffer availableResources = new NormalizedResourceOffer(clusterAvailableResources); + NormalizedResourceOffer availableResources = + new NormalizedResourceOffer(clusterAvailableResources); availableResources.add(topologyScheduledResources); boolean insufficientResources = availableResources.remove(topologyRequiredResources); if (insufficientResources) { @@ -346,7 +389,8 @@ boolean canScheduleAvailable() { } double availableMemory = clusterAvailableMemory + topologyScheduledMemory; - double totalRequiredTopologyMemory = topologyRequiredNonSharedMemory + topologySharedMemoryLowerBound; + double totalRequiredTopologyMemory = topologyRequiredNonSharedMemory + + topologySharedMemoryLowerBound; return (availableMemory >= totalRequiredTopologyMemory); } @@ -354,7 +398,8 @@ boolean canScheduleRemainingRequired() { if (!remainingResourcesAreSet) { return true; } - if (remainingRequiredTopologyResources.areAnyOverZero() || (remainingRequiredTopologyMemory > 0)) { + if (remainingRequiredTopologyResources.areAnyOverZero() + || (remainingRequiredTopologyMemory > 0)) { return false; } @@ -370,7 +415,8 @@ void setRemainingRequiredResources(Cluster cluster, TopologyDetails td) { remainingRequiredTopologyResources.add(topologyRequiredResources); remainingRequiredTopologyResources.remove(topologyScheduledResources); - remainingRequiredTopologyMemory = (topologyRequiredNonSharedMemory + topologySharedMemoryLowerBound) + remainingRequiredTopologyMemory = (topologyRequiredNonSharedMemory + + topologySharedMemoryLowerBound) - (topologyScheduledMemory); } @@ -378,7 +424,8 @@ void setRemainingRequiredResources(Cluster cluster, TopologyDetails td) { void adjustResourcesForEvictedTopology(Cluster cluster, TopologyDetails evict) { SchedulerAssignment assignment = cluster.getAssignmentById(evict.getId()); if (assignment != null) { - NormalizedResourceRequest evictResources = evict.getApproximateResources(assignment.getExecutors()); + NormalizedResourceRequest evictResources = evict.getApproximateResources(assignment + .getExecutors()); double topologyScheduledMemory = computeScheduledTopologyMemory(cluster, evict); clusterAvailableResources.add(evictResources); @@ -417,7 +464,8 @@ private double computeScheduledTopologyMemory(Cluster cluster, TopologyDetails t String getRemainingRequiredResourcesMessage() { StringBuilder message = new StringBuilder(); - NormalizedResourceOffer clusterRemainingAvailableResources = new NormalizedResourceOffer(); + NormalizedResourceOffer clusterRemainingAvailableResources = + new NormalizedResourceOffer(); clusterRemainingAvailableResources.add(clusterAvailableResources); clusterRemainingAvailableResources.remove(topologyScheduledResources); @@ -425,17 +473,21 @@ String getRemainingRequiredResourcesMessage() { double cpuNeeded = remainingRequiredTopologyResources.getTotalCpu(); if (memoryNeeded > 0) { message.append("Additional Memory Required: ").append(memoryNeeded).append(" MB "); - message.append("(Available: ").append(clusterRemainingAvailableResources.getTotalMemoryMb()).append(" MB). "); + message.append("(Available: ").append(clusterRemainingAvailableResources + .getTotalMemoryMb()).append(" MB). "); } if (cpuNeeded > 0) { message.append("Additional CPU Required: ").append(cpuNeeded).append("% CPU "); - message.append("(Available: ").append(clusterRemainingAvailableResources.getTotalCpu()).append(" % CPU)."); + message.append("(Available: ").append(clusterRemainingAvailableResources + .getTotalCpu()).append(" % CPU)."); } if (remainingRequiredTopologyResources.getNormalizedResources().anyNonCpuOverZero()) { message.append(" Additional Topology Required Resources: "); - message.append(remainingRequiredTopologyResources.getNormalizedResources().toString()); + message.append(remainingRequiredTopologyResources.getNormalizedResources() + .toString()); message.append(" Cluster Available Resources: "); - message.append(clusterRemainingAvailableResources.getNormalizedResources().toString()); + message.append(clusterRemainingAvailableResources.getNormalizedResources() + .toString()); message.append(". "); } return message.toString(); @@ -454,13 +506,16 @@ private Map getUsers(Cluster cluster) { for (TopologyDetails td : cluster.getTopologies()) { String topologySubmitter = td.getTopologySubmitter(); - //additional safety check to make sure that topologySubmitter is going to be a valid value + // additional safety check to make sure that topologySubmitter is going to be a valid + // value if (topologySubmitter == null || topologySubmitter.equals("")) { - LOG.error("Cannot determine user for topology {}. Will skip scheduling this topology", td.getName()); + LOG.error("Cannot determine user for topology {}. Will skip scheduling this " + + "topology", td.getName()); continue; } if (!userMap.containsKey(topologySubmitter)) { - userMap.put(topologySubmitter, new User(topologySubmitter, userResourcePools.get(topologySubmitter))); + userMap.put(topologySubmitter, new User(topologySubmitter, userResourcePools + .get(topologySubmitter))); } } return userMap; @@ -473,8 +528,10 @@ private Map> convertToDouble(Map> userPoolEntry : raw.entrySet()) { String user = userPoolEntry.getKey(); ret.put(user, new HashMap<>()); - for (Map.Entry resourceEntry : userPoolEntry.getValue().entrySet()) { - ret.get(user).put(resourceEntry.getKey(), resourceEntry.getValue().doubleValue()); + for (Map.Entry resourceEntry : userPoolEntry.getValue() + .entrySet()) { + ret.get(user).put(resourceEntry.getKey(), resourceEntry.getValue() + .doubleValue()); } } } @@ -485,7 +542,9 @@ private Map> convertToDouble(Map{resourceType->amountGuaranteed}} */ @@ -494,26 +553,32 @@ private Map> loadConfig() { // Try the loader plugin, if configured if (configLoader != null) { - raw = (Map>) configLoader.load(DaemonConfig.RESOURCE_AWARE_SCHEDULER_USER_POOLS); + raw = (Map>) configLoader + .load(DaemonConfig.RESOURCE_AWARE_SCHEDULER_USER_POOLS); if (raw != null) { return convertToDouble(raw); } else { - LOG.warn("Config loader returned null. Will try to read from user-resource-pools.yaml"); + LOG.warn("Config loader returned null. Will try to read from " + + "user-resource-pools.yaml"); } } // if no configs from loader, try to read from user-resource-pools.yaml - Map fromFile = Utils.findAndReadConfigFile("user-resource-pools.yaml", false); - raw = (Map>) fromFile.get(DaemonConfig.RESOURCE_AWARE_SCHEDULER_USER_POOLS); + Map fromFile = Utils.findAndReadConfigFile("user-resource-pools.yaml", + false); + raw = (Map>) fromFile + .get(DaemonConfig.RESOURCE_AWARE_SCHEDULER_USER_POOLS); if (raw != null) { return convertToDouble(raw); } else { - LOG.warn("Reading from user-resource-pools.yaml returned null. This could because the file is not available. " + LOG.warn("Reading from user-resource-pools.yaml returned null. This could because the " + + "file is not available. " + "Will load configs from storm configuration"); } // if no configs from user-resource-pools.yaml, get configs from conf - raw = (Map>) conf.get(DaemonConfig.RESOURCE_AWARE_SCHEDULER_USER_POOLS); + raw = (Map>) conf + .get(DaemonConfig.RESOURCE_AWARE_SCHEDULER_USER_POOLS); return convertToDouble(raw); } diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/resource/ResourceUtils.java b/storm-server/src/main/java/org/apache/storm/scheduler/resource/ResourceUtils.java index e4a523ca62f..6514c7abead 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/resource/ResourceUtils.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/resource/ResourceUtils.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

      - * http://www.apache.org/licenses/LICENSE-2.0 - *

      - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

      http://www.apache.org/licenses/LICENSE-2.0 + * + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -30,7 +36,8 @@ public class ResourceUtils { private static final Logger LOG = LoggerFactory.getLogger(ResourceUtils.class); - public static NormalizedResourceRequest getBoltResources(StormTopology topology, Map topologyConf, + public static NormalizedResourceRequest getBoltResources(StormTopology topology, Map topologyConf, String componentId) { if (topology.get_bolts() != null) { Bolt bolt = topology.get_bolts().get(componentId); @@ -39,11 +46,13 @@ public static NormalizedResourceRequest getBoltResources(StormTopology topology, return null; } - public static Map getBoltsResources(StormTopology topology, Map topologyConf) { + public static Map getBoltsResources(StormTopology topology, + Map topologyConf) { Map boltResources = new HashMap<>(); if (topology.get_bolts() != null) { for (Map.Entry bolt : topology.get_bolts().entrySet()) { - NormalizedResourceRequest topologyResources = new NormalizedResourceRequest(bolt.getValue().get_common(), + NormalizedResourceRequest topologyResources = new NormalizedResourceRequest(bolt + .getValue().get_common(), topologyConf, bolt.getKey()); if (LOG.isTraceEnabled()) { LOG.trace("Turned component {} into {}", bolt.getKey(), topologyResources); @@ -68,7 +77,8 @@ public static Map getSpoutsResources(StormTop Map spoutResources = new HashMap<>(); if (topology.get_spouts() != null) { for (Map.Entry spout : topology.get_spouts().entrySet()) { - NormalizedResourceRequest topologyResources = new NormalizedResourceRequest(spout.getValue().get_common(), + NormalizedResourceRequest topologyResources = new NormalizedResourceRequest(spout + .getValue().get_common(), topologyConf, spout.getKey()); if (LOG.isTraceEnabled()) { LOG.trace("Turned component {} into {}", spout.getKey(), topologyResources); @@ -79,7 +89,8 @@ public static Map getSpoutsResources(StormTop return spoutResources; } - public static void updateStormTopologyResources(StormTopology topology, Map> resourceUpdatesMap) { + public static void updateStormTopologyResources(StormTopology topology, Map> resourceUpdatesMap) { Map> componentsUpdated = new HashMap<>(); if (topology.get_spouts() != null) { for (Map.Entry spout : topology.get_spouts().entrySet()) { @@ -89,8 +100,10 @@ public static void updateStormTopologyResources(StormTopology topology, Map resourcesUpdate = NormalizedResources - .RESOURCE_NAME_NORMALIZER.normalizedResourceMap(resourceUpdatesMap.get(spoutName)); - String newJsonConf = getJsonWithUpdatedResources(spoutCommon.get_json_conf(), resourcesUpdate); + .RESOURCE_NAME_NORMALIZER.normalizedResourceMap(resourceUpdatesMap + .get(spoutName)); + String newJsonConf = getJsonWithUpdatedResources(spoutCommon.get_json_conf(), + resourcesUpdate); spoutCommon.set_json_conf(newJsonConf); componentsUpdated.put(spoutName, resourcesUpdate); } @@ -105,8 +118,10 @@ public static void updateStormTopologyResources(StormTopology topology, Map resourcesUpdate = NormalizedResources - .RESOURCE_NAME_NORMALIZER.normalizedResourceMap(resourceUpdatesMap.get(boltName)); - String newJsonConf = getJsonWithUpdatedResources(boltCommon.get_json_conf(), resourceUpdatesMap.get(boltName)); + .RESOURCE_NAME_NORMALIZER.normalizedResourceMap(resourceUpdatesMap + .get(boltName)); + String newJsonConf = getJsonWithUpdatedResources(boltCommon.get_json_conf(), + resourceUpdatesMap.get(boltName)); boltCommon.set_json_conf(newJsonConf); componentsUpdated.put(boltName, resourcesUpdate); } @@ -123,7 +138,8 @@ public static void updateStormTopologyResources(StormTopology topology, Map entry : NormalizedResources.RESOURCE_NAME_NORMALIZER.getResourceNameMapping().entrySet()) { + for (Map.Entry entry : NormalizedResources.RESOURCE_NAME_NORMALIZER + .getResourceNameMapping().entrySet()) { if (entry.getValue().equals(normalizedResourceName)) { return entry.getKey(); } @@ -131,7 +147,8 @@ public static String getCorrespondingLegacyResourceName(String normalizedResourc return normalizedResourceName; } - public static String getJsonWithUpdatedResources(String jsonConf, Map resourceUpdates) { + public static String getJsonWithUpdatedResources(String jsonConf, Map resourceUpdates) { try { JSONParser parser = new JSONParser(); Object obj = parser.parse(jsonConf); @@ -143,18 +160,24 @@ public static String getJsonWithUpdatedResources(String jsonConf, Map resourceUpdateEntry : resourceUpdates.entrySet()) { - if (NormalizedResources.RESOURCE_NAME_NORMALIZER.getResourceNameMapping().containsValue(resourceUpdateEntry.getKey())) { + if (NormalizedResources.RESOURCE_NAME_NORMALIZER.getResourceNameMapping() + .containsValue(resourceUpdateEntry.getKey())) { // if there will be legacy values they will be in the outer conf - jsonObject.remove(getCorrespondingLegacyResourceName(resourceUpdateEntry.getKey())); - componentResourceMap.remove(getCorrespondingLegacyResourceName(resourceUpdateEntry.getKey())); + jsonObject.remove(getCorrespondingLegacyResourceName(resourceUpdateEntry + .getKey())); + componentResourceMap + .remove(getCorrespondingLegacyResourceName(resourceUpdateEntry + .getKey())); } - componentResourceMap.put(resourceUpdateEntry.getKey(), resourceUpdateEntry.getValue()); + componentResourceMap.put(resourceUpdateEntry.getKey(), resourceUpdateEntry + .getValue()); } jsonObject.put(Config.TOPOLOGY_COMPONENT_RESOURCES_MAP, componentResourceMap); return jsonObject.toJSONString(); } catch (ParseException ex) { - throw new RuntimeException("Failed to parse component resources with json: " + jsonConf); + throw new RuntimeException("Failed to parse component resources with json: " + + jsonConf); } } diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/resource/SchedulingResult.java b/storm-server/src/main/java/org/apache/storm/scheduler/resource/SchedulingResult.java index 882c6be8794..02a1ec5dced 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/resource/SchedulingResult.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/resource/SchedulingResult.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

      - * http://www.apache.org/licenses/LICENSE-2.0 - *

      - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

      http://www.apache.org/licenses/LICENSE-2.0 + * + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -16,19 +22,20 @@ import org.slf4j.LoggerFactory; /** - * This class serves as a mechanism to return results and messages from a scheduling strategy to the Resource Aware + * This class serves as a mechanism to return results and messages from a scheduling strategy to the + * Resource Aware * Scheduler. */ public class SchedulingResult { private static final Logger LOG = LoggerFactory.getLogger(SchedulingResult.class); - //status of scheduling the topology e.g. success or fail? + // status of scheduling the topology e.g. success or fail? private final SchedulingStatus status; - //arbitrary message to be returned when scheduling is done + // arbitrary message to be returned when scheduling is done private final String message; - //error message returned is something went wrong + // error message returned is something went wrong private final String errorMessage; private SchedulingResult(SchedulingStatus status, String message, String errorMessage) { diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/resource/SchedulingStatus.java b/storm-server/src/main/java/org/apache/storm/scheduler/resource/SchedulingStatus.java index 47c05410d73..b8bc38ae7af 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/resource/SchedulingStatus.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/resource/SchedulingStatus.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

      - * http://www.apache.org/licenses/LICENSE-2.0 - *

      - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

      http://www.apache.org/licenses/LICENSE-2.0 + * + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -21,7 +27,8 @@ public enum SchedulingStatus { FAIL_OTHER; public static EnumSet success = EnumSet.of(SUCCESS); - public static EnumSet failure = EnumSet.of(FAIL_INVALID_TOPOLOGY, FAIL_NOT_ENOUGH_RESOURCES, + public static EnumSet failure = EnumSet.of(FAIL_INVALID_TOPOLOGY, + FAIL_NOT_ENOUGH_RESOURCES, FAIL_OTHER); public static boolean isStatusSuccess(SchedulingStatus status) { diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/resource/User.java b/storm-server/src/main/java/org/apache/storm/scheduler/resource/User.java index 3bda9227271..a71332d44bf 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/resource/User.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/resource/User.java @@ -25,7 +25,6 @@ import java.util.Map; import java.util.Set; import java.util.TreeSet; - import org.apache.storm.daemon.nimbus.TopologyResources; import org.apache.storm.scheduler.Cluster; import org.apache.storm.scheduler.ISchedulingState; @@ -33,7 +32,7 @@ import org.apache.storm.scheduler.TopologyDetails; public class User { - //Topologies that were deemed to be invalid + // Topologies that were deemed to be invalid private final Set unsuccess = new HashSet<>(); private final double cpuGuarantee; private final double memoryGuarantee; @@ -49,10 +48,12 @@ public User(String userId, Map resourcePool) { userId, resourcePool == null ? 0.0 : resourcePool.getOrDefault("cpu", 0.0), resourcePool == null ? 0.0 : resourcePool.getOrDefault("memory", 0.0), - resourcePool == null ? Collections.emptyMap() : extractGenericResourceEntries(resourcePool)); + resourcePool == null ? Collections + .emptyMap() : extractGenericResourceEntries(resourcePool)); } - private User(String userId, double cpuGuarantee, double memoryGuarantee, Map genericGuarantee) { + private User(String userId, double cpuGuarantee, double memoryGuarantee, Map genericGuarantee) { this.userId = userId; this.cpuGuarantee = cpuGuarantee; this.memoryGuarantee = memoryGuarantee; @@ -103,9 +104,9 @@ public double getResourcePoolAverageUtilization(ISchedulingState cluster) { double cpuResourcePoolUtilization = getCpuResourcePoolUtilization(cluster); double memoryResourcePoolUtilization = getMemoryResourcePoolUtilization(cluster); - //cannot be (cpuResourcePoolUtilization + memoryResourcePoolUtilization)/2 - //since memoryResourcePoolUtilization or cpuResourcePoolUtilization can be Double.MAX_VALUE - //Should not return infinity in that case + // cannot be (cpuResourcePoolUtilization + memoryResourcePoolUtilization)/2 + // since memoryResourcePoolUtilization or cpuResourcePoolUtilization can be Double.MAX_VALUE + // Should not return infinity in that case return ((cpuResourcePoolUtilization) / 2.0) + ((memoryResourcePoolUtilization) / 2.0); } @@ -125,7 +126,8 @@ public double getMemoryResourcePoolUtilization(ISchedulingState cluster) { public double getMemoryResourceRequest(ISchedulingState cluster) { double sum = 0.0; - Set topologyDetailsSet = new HashSet<>(cluster.getTopologies().getTopologiesOwnedBy(userId)); + Set topologyDetailsSet = new HashSet<>(cluster.getTopologies() + .getTopologiesOwnedBy(userId)); for (TopologyDetails topo : topologyDetailsSet) { sum += topo.getTotalRequestedMemOnHeap() + topo.getTotalRequestedMemOffHeap(); } @@ -134,7 +136,8 @@ public double getMemoryResourceRequest(ISchedulingState cluster) { public double getCpuResourceRequest(ISchedulingState cluster) { double sum = 0.0; - Set topologyDetailsSet = new HashSet<>(cluster.getTopologies().getTopologiesOwnedBy(userId)); + Set topologyDetailsSet = new HashSet<>(cluster.getTopologies() + .getTopologiesOwnedBy(userId)); for (TopologyDetails topo : topologyDetailsSet) { sum += topo.getTotalRequestedCpu(); } @@ -196,7 +199,8 @@ public TopologyDetails getRunningTopologyWithLowestPriority(ISchedulingState clu return queue.last(); } - private static Map extractGenericResourceEntries(Map resourcePool) { + private static Map extractGenericResourceEntries(Map resourcePool) { Map ret = new HashMap<>(); for (Map.Entry entry : resourcePool.entrySet()) { String key = entry.getKey(); @@ -227,7 +231,8 @@ public String toString() { } /** - * Comparator that sorts topologies by priority and then by submission time First sort by Topology Priority, if there is a tie for + * Comparator that sorts topologies by priority and then by submission time First sort by + * Topology Priority, if there is a tie for * topology priority, topology uptime is used to sort. */ static class PQsortByPriorityAndSubmittionTime implements Comparator { diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/resource/normalization/NormalizedResourceOffer.java b/storm-server/src/main/java/org/apache/storm/scheduler/resource/normalization/NormalizedResourceOffer.java index 4eb16f4eefe..6572ad7785e 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/resource/normalization/NormalizedResourceOffer.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/resource/normalization/NormalizedResourceOffer.java @@ -39,8 +39,10 @@ public class NormalizedResourceOffer implements NormalizedResourcesWithMemory { * @param resources the resources to be normalized. */ public NormalizedResourceOffer(Map resources) { - Map normalizedResourceMap = NormalizedResources.RESOURCE_NAME_NORMALIZER.normalizedResourceMap(resources); - totalMemoryMb = normalizedResourceMap.getOrDefault(Constants.COMMON_TOTAL_MEMORY_RESOURCE_NAME, 0.0); + Map normalizedResourceMap = NormalizedResources.RESOURCE_NAME_NORMALIZER + .normalizedResourceMap(resources); + totalMemoryMb = normalizedResourceMap + .getOrDefault(Constants.COMMON_TOTAL_MEMORY_RESOURCE_NAME, 0.0); this.normalizedResources = new NormalizedResources(normalizedResourceMap); } @@ -54,6 +56,7 @@ public NormalizedResourceOffer() { /** * Copy Constructor. + * * @param other what to copy. */ public NormalizedResourceOffer(NormalizedResourceOffer other) { @@ -68,6 +71,7 @@ public double getTotalMemoryMb() { /** * Return these resources as a normalized map. + * * @return the normalized map. */ public Map toNormalizedMap() { @@ -83,12 +87,15 @@ public void add(NormalizedResourcesWithMemory other) { /** * Remove the resources in other from this. + * * @param other the resources to be removed. * @param resourceMetrics The resource related metrics - * @return true if one or more resources in other were larger than available resources in this, else false. + * @return true if one or more resources in other were larger than available resources in this, + * else false. */ public boolean remove(NormalizedResourcesWithMemory other, ResourceMetrics resourceMetrics) { - boolean negativeResources = normalizedResources.remove(other.getNormalizedResources(), resourceMetrics); + boolean negativeResources = normalizedResources.remove(other.getNormalizedResources(), + resourceMetrics); totalMemoryMb -= other.getTotalMemoryMb(); if (totalMemoryMb < 0.0) { negativeResources = true; @@ -106,9 +113,11 @@ public boolean remove(NormalizedResourcesWithMemory other) { /** * Remove the resources in other from this. + * * @param other the resources to be removed. * @param resourceMetrics The resource related metrics - * @return true if one or more resources in other were larger than available resources in this, else false. + * @return true if one or more resources in other were larger than available resources in this, + * else false. */ public boolean remove(WorkerResources other, ResourceMetrics resourceMetrics) { boolean negativeResources = normalizedResources.remove(other); @@ -123,7 +132,9 @@ public boolean remove(WorkerResources other, ResourceMetrics resourceMetrics) { /** * Calculate the average percentage used. - * @see NormalizedResources#calculateAveragePercentageUsedBy(org.apache.storm.scheduler.resource.normalization.NormalizedResources, + * + * @see + * NormalizedResources#calculateAveragePercentageUsedBy(org.apache.storm.scheduler.resource.normalization.NormalizedResources, * double, double) */ public double calculateAveragePercentageUsedBy(NormalizedResourceOffer used) { @@ -133,16 +144,23 @@ public double calculateAveragePercentageUsedBy(NormalizedResourceOffer used) { /** * Calculate the min percentage used of the resource. - * @see NormalizedResources#calculateMinPercentageUsedBy(org.apache.storm.scheduler.resource.normalization.NormalizedResources, double, + * + * @see + * NormalizedResources#calculateMinPercentageUsedBy(org.apache.storm.scheduler.resource.normalization.NormalizedResources, + * double, * double) */ public double calculateMinPercentageUsedBy(NormalizedResourceOffer used) { - return normalizedResources.calculateMinPercentageUsedBy(used.getNormalizedResources(), getTotalMemoryMb(), used.getTotalMemoryMb()); + return normalizedResources.calculateMinPercentageUsedBy(used.getNormalizedResources(), + getTotalMemoryMb(), used.getTotalMemoryMb()); } /** * Check if resources might be able to fit. - * @see NormalizedResources#couldHoldIgnoringSharedMemory(org.apache.storm.scheduler.resource.normalization.NormalizedResources, double, + * + * @see + * NormalizedResources#couldHoldIgnoringSharedMemory(org.apache.storm.scheduler.resource.normalization.NormalizedResources, + * double, * double) */ public boolean couldHoldIgnoringSharedMemory(NormalizedResourcesWithMemory other) { @@ -170,12 +188,15 @@ public String toString() { } /** - * If a node or rack has a kind of resource not in a request, make that resource negative so when sorting that node or rack will + * If a node or rack has a kind of resource not in a request, make that resource negative so + * when sorting that node or rack will * be less likely to be selected. + * * @param requestedResources the requested resources. */ public void updateForRareResourceAffinity(NormalizedResourceRequest requestedResources) { - normalizedResources.updateForRareResourceAffinity(requestedResources.getNormalizedResources()); + normalizedResources.updateForRareResourceAffinity(requestedResources + .getNormalizedResources()); } @Override @@ -191,16 +212,19 @@ public boolean areAnyOverZero() { /** * Is there any possibility that a resource request could ever fit on this. + * * @param minWorkerCpu the configured minimum worker CPU * @param requestedResources the requested resources - * @return true if there is the possibility it might fit, no guarantee that it will, or false if there is no + * @return true if there is the possibility it might fit, no guarantee that it will, or false if + * there is no * way it would ever fit. */ public boolean couldFit(double minWorkerCpu, NormalizedResourceRequest requestedResources) { if (minWorkerCpu < 0.001) { return this.couldHoldIgnoringSharedMemory(requestedResources); } else { - // Assume that there could be a worker already on the node that is under the minWorkerCpu budget. + // Assume that there could be a worker already on the node that is under the + // minWorkerCpu budget. // It's possible we could combine with it. Let's disregard minWorkerCpu from the request // and validate that CPU as a rough fit. double requestedCpu = Math.max(requestedResources.getTotalCpu() - minWorkerCpu, 0.0); diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/resource/normalization/NormalizedResourceRequest.java b/storm-server/src/main/java/org/apache/storm/scheduler/resource/normalization/NormalizedResourceRequest.java index 93756b84414..67f9d0258c9 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/resource/normalization/NormalizedResourceRequest.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/resource/normalization/NormalizedResourceRequest.java @@ -51,14 +51,18 @@ private NormalizedResourceRequest(Map resources, } else { Map normalizedResourceMap = NormalizedResources.RESOURCE_NAME_NORMALIZER .normalizedResourceMap(defaultResources); - normalizedResourceMap.putAll(NormalizedResources.RESOURCE_NAME_NORMALIZER.normalizedResourceMap(resources)); - onHeap = normalizedResourceMap.getOrDefault(Constants.COMMON_ONHEAP_MEMORY_RESOURCE_NAME, 0.0); - offHeap = normalizedResourceMap.getOrDefault(Constants.COMMON_OFFHEAP_MEMORY_RESOURCE_NAME, 0.0); + normalizedResourceMap.putAll(NormalizedResources.RESOURCE_NAME_NORMALIZER + .normalizedResourceMap(resources)); + onHeap = normalizedResourceMap + .getOrDefault(Constants.COMMON_ONHEAP_MEMORY_RESOURCE_NAME, 0.0); + offHeap = normalizedResourceMap + .getOrDefault(Constants.COMMON_OFFHEAP_MEMORY_RESOURCE_NAME, 0.0); normalizedResources = new NormalizedResources(normalizedResourceMap); } } - public NormalizedResourceRequest(ComponentCommon component, Map topoConf, String componentId) { + public NormalizedResourceRequest(ComponentCommon component, Map topoConf, + String componentId) { this(parseResources(component.get_json_conf()), getDefaultResources(topoConf, componentId)); } @@ -70,7 +74,8 @@ public NormalizedResourceRequest() { this((Map) null, null); } - private static void putIfMissing(Map dest, String destKey, Map src, String srcKey) { + private static void putIfMissing(Map dest, String destKey, Map src, String srcKey) { if (!dest.containsKey(destKey)) { Number value = (Number) src.get(srcKey); if (value != null) { @@ -79,9 +84,11 @@ private static void putIfMissing(Map dest, String destKey, Map getDefaultResources(Map topoConf, String componentId) { + private static Map getDefaultResources(Map topoConf, + String componentId) { Map ret = - NormalizedResources.RESOURCE_NAME_NORMALIZER.normalizedResourceMap((Map) topoConf.getOrDefault( + NormalizedResources.RESOURCE_NAME_NORMALIZER.normalizedResourceMap((Map) topoConf.getOrDefault( Config.TOPOLOGY_COMPONENT_RESOURCES_MAP, new HashMap<>())); // Some components might have different resource configs. @@ -89,35 +96,46 @@ private static Map getDefaultResources(Map topoC if (componentId.equals(Acker.ACKER_COMPONENT_ID)) { if (topoConf.containsKey(Config.TOPOLOGY_ACKER_RESOURCES_ONHEAP_MEMORY_MB)) { ret.put(Constants.COMMON_ONHEAP_MEMORY_RESOURCE_NAME, - ObjectReader.getDouble(topoConf.get(Config.TOPOLOGY_ACKER_RESOURCES_ONHEAP_MEMORY_MB))); + ObjectReader.getDouble(topoConf + .get(Config.TOPOLOGY_ACKER_RESOURCES_ONHEAP_MEMORY_MB))); } if (topoConf.containsKey(Config.TOPOLOGY_ACKER_RESOURCES_OFFHEAP_MEMORY_MB)) { ret.put(Constants.COMMON_OFFHEAP_MEMORY_RESOURCE_NAME, - ObjectReader.getDouble(topoConf.get(Config.TOPOLOGY_ACKER_RESOURCES_OFFHEAP_MEMORY_MB))); + ObjectReader.getDouble(topoConf + .get(Config.TOPOLOGY_ACKER_RESOURCES_OFFHEAP_MEMORY_MB))); } if (topoConf.containsKey(Config.TOPOLOGY_ACKER_CPU_PCORE_PERCENT)) { ret.put(Constants.COMMON_CPU_RESOURCE_NAME, - ObjectReader.getDouble(topoConf.get(Config.TOPOLOGY_ACKER_CPU_PCORE_PERCENT))); + ObjectReader.getDouble(topoConf + .get(Config.TOPOLOGY_ACKER_CPU_PCORE_PERCENT))); } } else if (componentId.startsWith(Constants.METRICS_COMPONENT_ID_PREFIX)) { - if (topoConf.containsKey(Config.TOPOLOGY_METRICS_CONSUMER_RESOURCES_ONHEAP_MEMORY_MB)) { + if (topoConf + .containsKey(Config.TOPOLOGY_METRICS_CONSUMER_RESOURCES_ONHEAP_MEMORY_MB)) { ret.put(Constants.COMMON_ONHEAP_MEMORY_RESOURCE_NAME, - ObjectReader.getDouble(topoConf.get(Config.TOPOLOGY_METRICS_CONSUMER_RESOURCES_ONHEAP_MEMORY_MB))); + ObjectReader.getDouble(topoConf + .get(Config.TOPOLOGY_METRICS_CONSUMER_RESOURCES_ONHEAP_MEMORY_MB))); } - if (topoConf.containsKey(Config.TOPOLOGY_METRICS_CONSUMER_RESOURCES_OFFHEAP_MEMORY_MB)) { + if (topoConf + .containsKey(Config.TOPOLOGY_METRICS_CONSUMER_RESOURCES_OFFHEAP_MEMORY_MB)) { ret.put(Constants.COMMON_OFFHEAP_MEMORY_RESOURCE_NAME, - ObjectReader.getDouble(topoConf.get(Config.TOPOLOGY_METRICS_CONSUMER_RESOURCES_OFFHEAP_MEMORY_MB))); + ObjectReader.getDouble(topoConf + .get(Config.TOPOLOGY_METRICS_CONSUMER_RESOURCES_OFFHEAP_MEMORY_MB))); } if (topoConf.containsKey(Config.TOPOLOGY_METRICS_CONSUMER_CPU_PCORE_PERCENT)) { ret.put(Constants.COMMON_CPU_RESOURCE_NAME, - ObjectReader.getDouble(topoConf.get(Config.TOPOLOGY_METRICS_CONSUMER_CPU_PCORE_PERCENT))); + ObjectReader.getDouble(topoConf + .get(Config.TOPOLOGY_METRICS_CONSUMER_CPU_PCORE_PERCENT))); } } } - putIfMissing(ret, Constants.COMMON_CPU_RESOURCE_NAME, topoConf, Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT); - putIfMissing(ret, Constants.COMMON_OFFHEAP_MEMORY_RESOURCE_NAME, topoConf, Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB); - putIfMissing(ret, Constants.COMMON_ONHEAP_MEMORY_RESOURCE_NAME, topoConf, Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB); + putIfMissing(ret, Constants.COMMON_CPU_RESOURCE_NAME, topoConf, + Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT); + putIfMissing(ret, Constants.COMMON_OFFHEAP_MEMORY_RESOURCE_NAME, topoConf, + Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB); + putIfMissing(ret, Constants.COMMON_ONHEAP_MEMORY_RESOURCE_NAME, topoConf, + Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB); return ret; } @@ -132,16 +150,21 @@ private static Map parseResources(String input) { // Legacy resource parsing if (jsonObject.containsKey(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB)) { Double topoMemOnHeap = ObjectReader - .getDouble(jsonObject.get(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB), null); - topologyResources.put(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB, topoMemOnHeap); + .getDouble(jsonObject + .get(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB), null); + topologyResources.put(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB, + topoMemOnHeap); } if (jsonObject.containsKey(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB)) { Double topoMemOffHeap = ObjectReader - .getDouble(jsonObject.get(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB), null); - topologyResources.put(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB, topoMemOffHeap); + .getDouble(jsonObject + .get(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB), null); + topologyResources.put(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB, + topoMemOffHeap); } if (jsonObject.containsKey(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT)) { - Double topoCpu = ObjectReader.getDouble(jsonObject.get(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT), + Double topoCpu = ObjectReader.getDouble(jsonObject + .get(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT), null); topologyResources.put(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT, topoCpu); } @@ -168,6 +191,7 @@ private static Map parseResources(String input) { /** * Convert to a map that is used by configuration and the UI. + * * @return a map with the key as the resource name and the value the resource amount. */ public Map toNormalizedMap() { @@ -190,7 +214,8 @@ public static void removeNonGenericResources(Map map) { /* * return a map that is the sum of resources1 + resources2 */ - public static Map addResourceMap(Map resources1, Map resources2) { + public static Map addResourceMap(Map resources1, Map resources2) { Map sum = new HashMap<>(resources1); if (resources2 != null) { for (Map.Entry me : resources2.entrySet()) { @@ -204,7 +229,8 @@ public static Map addResourceMap(Map resources1, /* * return a map that is the difference of resources1 - resources2 */ - public static Map subtractResourceMap(Map resource1, Map resource2) { + public static Map subtractResourceMap(Map resource1, Map resource2) { if (resource1 == null || resource2 == null) { return new HashMap<>(); } @@ -245,11 +271,12 @@ public void add(NormalizedResourceRequest other) { /** * Add the resources from a worker to those in this. + * * @param value the resources on the worker. */ public void add(WorkerResources value) { this.normalizedResources.add(value); - //The resources are already normalized + // The resources are already normalized Map resources = value.get_resources(); onHeap += resources.getOrDefault(Constants.COMMON_ONHEAP_MEMORY_RESOURCE_NAME, 0.0); offHeap += resources.getOrDefault(Constants.COMMON_OFFHEAP_MEMORY_RESOURCE_NAME, 0.0); diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/resource/normalization/NormalizedResources.java b/storm-server/src/main/java/org/apache/storm/scheduler/resource/normalization/NormalizedResources.java index de097a6b0c7..1ba4099e687 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/resource/normalization/NormalizedResources.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/resource/normalization/NormalizedResources.java @@ -29,7 +29,8 @@ import org.slf4j.LoggerFactory; /** - * Resources that have been normalized. This class is intended as a delegate for more specific types of normalized resource set, since it + * Resources that have been normalized. This class is intended as a delegate for more specific types + * of normalized resource set, since it * does not keep track of memory as a resource. */ public class NormalizedResources { @@ -62,7 +63,8 @@ public NormalizedResources(NormalizedResources other) { } /** - * Create a new normalized set of resources. Note that memory is not managed by this class, as it is not consistent in requests vs + * Create a new normalized set of resources. Note that memory is not managed by this class, as + * it is not consistent in requests vs * offers because of how on heap vs off heap is used. * * @param normalizedResources the normalized resource map @@ -73,7 +75,8 @@ public NormalizedResources(Map normalizedResources) { } /** - * This is for testing only. It allows a test to reset the static state relating to resource names. We reset the mapping because some + * This is for testing only. It allows a test to reset the static state relating to resource + * names. We reset the mapping because some * algorithms sadly have different behavior if a resource exists or not. */ @VisibleForTesting @@ -124,7 +127,8 @@ public void add(WorkerResources value) { } /** - * Remove the other resources from this. This is the same as subtracting the resources in other from this. + * Remove the other resources from this. This is the same as subtracting the resources in other + * from this. * * @param other the resources we want removed. * @param resourceMetrics The resource related metrics @@ -163,7 +167,8 @@ public boolean remove(NormalizedResources other, ResourceMetrics resourceMetrics public boolean remove(WorkerResources value) { Map workerNormalizedResources = value.get_resources(); cpu -= workerNormalizedResources.getOrDefault(Constants.COMMON_CPU_RESOURCE_NAME, 0.0); - return remove(RESOURCE_MAP_ARRAY_BRIDGE.translateToResourceArray(workerNormalizedResources)) || cpu < 0; + return remove(RESOURCE_MAP_ARRAY_BRIDGE + .translateToResourceArray(workerNormalizedResources)) || cpu < 0; } private boolean remove(double[] resourceArray) { @@ -185,11 +190,13 @@ public String toString() { } /** - * Return a Map of the normalized resource name to a double. This should only be used when returning thrift resource requests to the end + * Return a Map of the normalized resource name to a double. This should only be used when + * returning thrift resource requests to the end * user. */ public Map toNormalizedMap() { - Map ret = RESOURCE_MAP_ARRAY_BRIDGE.translateFromResourceArray(otherResources); + Map ret = RESOURCE_MAP_ARRAY_BRIDGE + .translateFromResourceArray(otherResources); ret.put(Constants.COMMON_CPU_RESOURCE_NAME, cpu); return ret; } @@ -202,7 +209,8 @@ private double getResourceAt(int index) { } /** - * A simple sanity check to see if all of the resources in this would be large enough to hold the resources in other ignoring memory. It + * A simple sanity check to see if all of the resources in this would be large enough to hold + * the resources in other ignoring memory. It * does not check memory because with shared memory it is beyond the scope of this. * * @param other the resources that we want to check if they would fit in this. @@ -210,7 +218,8 @@ private double getResourceAt(int index) { * @param otherTotalMemoryMb The total memory in MB of other * @return true if it might fit, else false if it could not possibly fit. */ - public boolean couldHoldIgnoringSharedMemory(NormalizedResources other, double thisTotalMemoryMb, double otherTotalMemoryMb) { + public boolean couldHoldIgnoringSharedMemory(NormalizedResources other, + double thisTotalMemoryMb, double otherTotalMemoryMb) { if (this.cpu < other.getTotalCpu()) { return false; } @@ -218,15 +227,18 @@ public boolean couldHoldIgnoringSharedMemory(NormalizedResources other, double t } /** - * A simple sanity check to see if all of the resources in this would be large enough to hold the resources in other ignoring memory. It - * does not check memory because with shared memory it is beyond the scope of this. It also does not check CPU. + * A simple sanity check to see if all of the resources in this would be large enough to hold + * the resources in other ignoring memory. It + * does not check memory because with shared memory it is beyond the scope of this. It also does + * not check CPU. * * @param other the resources that we want to check if they would fit in this. * @param thisTotalMemoryMb The total memory in MB of this * @param otherTotalMemoryMb The total memory in MB of other * @return true if it might fit, else false if it could not possibly fit. */ - public boolean couldHoldIgnoringSharedMemoryAndCpu(NormalizedResources other, double thisTotalMemoryMb, double otherTotalMemoryMb) { + public boolean couldHoldIgnoringSharedMemoryAndCpu(NormalizedResources other, + double thisTotalMemoryMb, double otherTotalMemoryMb) { int length = Math.max(this.otherResources.length, other.otherResources.length); for (int i = 0; i < length; i++) { if (getResourceAt(i) < other.getResourceAt(i)) { @@ -238,7 +250,8 @@ public boolean couldHoldIgnoringSharedMemoryAndCpu(NormalizedResources other, do } private String getResourceNameForResourceIndex(int resourceIndex) { - for (Map.Entry entry : RESOURCE_MAP_ARRAY_BRIDGE.getResourceNamesToArrayIndex().entrySet()) { + for (Map.Entry entry : RESOURCE_MAP_ARRAY_BRIDGE + .getResourceNamesToArrayIndex().entrySet()) { int index = entry.getValue(); if (index == resourceIndex) { return entry.getKey(); @@ -247,21 +260,28 @@ private String getResourceNameForResourceIndex(int resourceIndex) { return null; } - private void throwBecauseUsedIsNotSubsetOfTotal(NormalizedResources used, double totalMemoryMb, double usedMemoryMb, String info) { - throw new IllegalArgumentException(String.format("The used resources must be a subset of the total resources." - + " Used: '%s', Total: '%s', Used Mem: '%f', Total Mem: '%f', additionalInfo: '%s'", + private void throwBecauseUsedIsNotSubsetOfTotal(NormalizedResources used, double totalMemoryMb, + double usedMemoryMb, String info) { + throw new IllegalArgumentException(String + .format("The used resources must be a subset of the total resources." + + " Used: '%s', Total: '%s', Used Mem: '%f', Total Mem: '%f', " + + "additionalInfo: '%s'", used.toNormalizedMap(), this.toNormalizedMap(), usedMemoryMb, totalMemoryMb, info)); } - private void throwBecauseUsedIsNotSubsetOfTotal(NormalizedResources used, double totalMemoryMb, double usedMemoryMb) { - throw new IllegalArgumentException(String.format("The used resources must be a subset of the total resources." + private void throwBecauseUsedIsNotSubsetOfTotal(NormalizedResources used, double totalMemoryMb, + double usedMemoryMb) { + throw new IllegalArgumentException(String + .format("The used resources must be a subset of the total resources." + " Used: '%s', Total: '%s', Used Mem: '%f', Total Mem: '%f'", used.toNormalizedMap(), this.toNormalizedMap(), usedMemoryMb, totalMemoryMb)); } /** - * Calculate the average resource usage percentage with this being the total resources and used being the amounts used. Used must be a - * subset of the total resources. If a resource in the total has a value of zero, it will be skipped in the calculation to avoid + * Calculate the average resource usage percentage with this being the total resources and used + * being the amounts used. Used must be a + * subset of the total resources. If a resource in the total has a value of zero, it will be + * skipped in the calculation to avoid * division by 0. If all resources are skipped the result is defined to be 100.0. * * @param used the amount of resources used. @@ -269,10 +289,12 @@ private void throwBecauseUsedIsNotSubsetOfTotal(NormalizedResources used, double * @param usedMemoryMb The used memory in MB * @return the average percentage used 0.0 to 100.0. * - * @throws IllegalArgumentException if any resource in used has a greater value than the same resource in the total, or used has generic + * @throws IllegalArgumentException if any resource in used has a greater value than the same + * resource in the total, or used has generic * resources that are not present in the total. */ - public double calculateAveragePercentageUsedBy(NormalizedResources used, double totalMemoryMb, double usedMemoryMb) { + public double calculateAveragePercentageUsedBy(NormalizedResources used, double totalMemoryMb, + double usedMemoryMb) { int skippedResourceTypes = 0; double total = 0.0; if (usedMemoryMb > totalMemoryMb) { @@ -301,7 +323,7 @@ public double calculateAveragePercentageUsedBy(NormalizedResources used, double double totalValue = otherResources[i]; double usedValue; if (i >= used.otherResources.length) { - //Resources missing from used are using none of that resource + // Resources missing from used are using none of that resource usedValue = 0.0; } else { usedValue = used.otherResources[i]; @@ -310,20 +332,25 @@ public double calculateAveragePercentageUsedBy(NormalizedResources used, double throwBecauseUsedIsNotSubsetOfTotal(used, totalMemoryMb, usedMemoryMb); } if (totalValue == 0.0) { - //Skip any resources where the total is 0, the percent used for this resource isn't meaningful. - //We fall back to prioritizing by cpu, memory and any other resources by ignoring this value + // Skip any resources where the total is 0, the percent used for this resource isn't + // meaningful. + // We fall back to prioritizing by cpu, memory and any other resources by ignoring + // this value skippedResourceTypes++; continue; } total += usedValue / totalValue; } - //Adjust the divisor for the average to account for any skipped resources (those where the total was 0) + // Adjust the divisor for the average to account for any skipped resources (those where the + // total was 0) int divisor = 2 + otherResources.length - skippedResourceTypes; if (divisor == 0) { /* - * This is an arbitrary choice to make the result consistent with calculateMin. Any value would be valid here, becase there are - * no (non-zero) resources in the total set of resources, so we're trying to average 0 values. + * This is an arbitrary choice to make the result consistent with calculateMin. Any + * value would be valid here, becase there are + * no (non-zero) resources in the total set of resources, so we're trying to average 0 + * values. */ return 100.0; } else { @@ -332,8 +359,10 @@ public double calculateAveragePercentageUsedBy(NormalizedResources used, double } /** - * Calculate the minimum resource usage percentage with this being the total resources and used being the amounts used. Used must be a - * subset of the total resources. If a resource in the total has a value of zero, it will be skipped in the calculation to avoid + * Calculate the minimum resource usage percentage with this being the total resources and used + * being the amounts used. Used must be a + * subset of the total resources. If a resource in the total has a value of zero, it will be + * skipped in the calculation to avoid * division by 0. If all resources are skipped the result is defined to be 100.0. * * @param used the amount of resources used. @@ -341,10 +370,12 @@ public double calculateAveragePercentageUsedBy(NormalizedResources used, double * @param usedMemoryMb The used memory in MB * @return the minimum percentage used 0.0 to 100.0. * - * @throws IllegalArgumentException if any resource in used has a greater value than the same resource in the total, or used has generic + * @throws IllegalArgumentException if any resource in used has a greater value than the same + * resource in the total, or used has generic * resources that are not present in the total. */ - public double calculateMinPercentageUsedBy(NormalizedResources used, double totalMemoryMb, double usedMemoryMb) { + public double calculateMinPercentageUsedBy(NormalizedResources used, double totalMemoryMb, + double usedMemoryMb) { if (LOG.isTraceEnabled()) { LOG.trace("Calculating min percentage used by. Used Mem: {} Total Mem: {}" + " Used Normalized Resources: {} Total Normalized Resources: {}", totalMemoryMb, usedMemoryMb, @@ -372,16 +403,19 @@ public double calculateMinPercentageUsedBy(NormalizedResources used, double tota for (int i = 0; i < otherResources.length; i++) { if (otherResources[i] == 0.0) { - //Skip any resources where the total is 0, the percent used for this resource isn't meaningful. - //We fall back to prioritizing by cpu, memory and any other resources by ignoring this value + // Skip any resources where the total is 0, the percent used for this resource isn't + // meaningful. + // We fall back to prioritizing by cpu, memory and any other resources by ignoring + // this value continue; } if (i >= used.otherResources.length) { - //Resources missing from used are using none of that resource + // Resources missing from used are using none of that resource return 0; } if (used.otherResources[i] > otherResources[i]) { - String info = String.format("%s, %f > %f", getResourceNameForResourceIndex(i), used.otherResources[i], otherResources[i]); + String info = String.format("%s, %f > %f", getResourceNameForResourceIndex(i), + used.otherResources[i], otherResources[i]); throwBecauseUsedIsNotSubsetOfTotal(used, totalMemoryMb, usedMemoryMb, info); } min = Math.min(min, used.otherResources[i] / otherResources[i]); @@ -390,8 +424,11 @@ public double calculateMinPercentageUsedBy(NormalizedResources used, double tota } /** - * If a node or rack has a kind of resource not in a request, make that resource negative so when sorting that node or rack will - * be less likely to be selected. If the resource is in the request, make that resource positive. + * If a node or rack has a kind of resource not in a request, make that resource negative so + * when sorting that node or rack will + * be less likely to be selected. If the resource is in the request, make that resource + * positive. + * * @param request the requested resources. */ public void updateForRareResourceAffinity(NormalizedResources request) { @@ -430,6 +467,7 @@ private boolean areAnyOverZero(boolean skipCpuCheck) { /** * Are any of the resources positive. + * * @return true of any of the resources are positive. False if they are all <= 0. */ public boolean areAnyOverZero() { @@ -438,6 +476,7 @@ public boolean areAnyOverZero() { /** * Are any of the non cpu resources positive. + * * @return true of any of the non cpu resources are positive. False if they are all <= 0. */ public boolean anyNonCpuOverZero() { diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/resource/normalization/ResourceMapArrayBridge.java b/storm-server/src/main/java/org/apache/storm/scheduler/resource/normalization/ResourceMapArrayBridge.java index 4a4797b6834..cdcc124b2fd 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/resource/normalization/ResourceMapArrayBridge.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/resource/normalization/ResourceMapArrayBridge.java @@ -25,26 +25,31 @@ import org.apache.storm.Constants; /** - * Provides translation between normalized resource maps and resource value arrays. Some operations use resource value arrays instead of the + * Provides translation between normalized resource maps and resource value arrays. Some operations + * use resource value arrays instead of the * full normalized resource map as an optimization. See {@link NormalizedResources}. */ public class ResourceMapArrayBridge { - private final ConcurrentMap resourceNamesToArrayIndex = new ConcurrentHashMap<>(); + private final ConcurrentMap resourceNamesToArrayIndex = + new ConcurrentHashMap<>(); private final AtomicInteger counter = new AtomicInteger(0); /** - * Translates a normalized resource map to an array of resource values. Each resource name will be assigned an index in the array, which - * is guaranteed to be consistent with subsequent invocations of this method. Note that CPU and memory resources are not translated by + * Translates a normalized resource map to an array of resource values. Each resource name will + * be assigned an index in the array, which + * is guaranteed to be consistent with subsequent invocations of this method. Note that CPU and + * memory resources are not translated by * this method, as they are expected to be captured elsewhere. * * @param normalizedResources The resources to translate to an array * @return The array of resource values */ public double[] translateToResourceArray(Map normalizedResources) { - //To avoid locking we will go through the map twice. It should be small so it is probably not a big deal + // To avoid locking we will go through the map twice. It should be small so it is probably + // not a big deal for (String key : normalizedResources.keySet()) { - //We are going to skip over CPU and Memory, because they are captured elsewhere + // We are going to skip over CPU and Memory, because they are captured elsewhere if (!Constants.COMMON_CPU_RESOURCE_NAME.equals(key) && !Constants.COMMON_TOTAL_MEMORY_RESOURCE_NAME.equals(key) && !Constants.COMMON_OFFHEAP_MEMORY_RESOURCE_NAME.equals(key) @@ -52,12 +57,12 @@ public double[] translateToResourceArray(Map normalizedResources resourceNamesToArrayIndex.computeIfAbsent(key, (k) -> counter.getAndIncrement()); } } - //By default all of the values are 0 + // By default all of the values are 0 double[] ret = new double[counter.get()]; for (Map.Entry entry : normalizedResources.entrySet()) { Integer index = resourceNamesToArrayIndex.get(entry.getKey()); if (index != null) { - //index == null if it is memory or CPU + // index == null if it is memory or CPU ret[index] = entry.getValue(); } } @@ -66,6 +71,7 @@ public double[] translateToResourceArray(Map normalizedResources /** * Create an array that has all values 0. + * * @return the empty array. */ public double[] empty() { diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/resource/normalization/ResourceMetrics.java b/storm-server/src/main/java/org/apache/storm/scheduler/resource/normalization/ResourceMetrics.java index 851499b667d..42092874fb2 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/resource/normalization/ResourceMetrics.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/resource/normalization/ResourceMetrics.java @@ -24,7 +24,8 @@ public class ResourceMetrics { private final Meter numNegativeResourceEvents; public ResourceMetrics(StormMetricsRegistry metricsRegistry) { - numNegativeResourceEvents = metricsRegistry.registerMeter("nimbus:num-negative-resource-events"); + numNegativeResourceEvents = metricsRegistry + .registerMeter("nimbus:num-negative-resource-events"); } public Meter getNegativeResourceEventsMeter() { diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/resource/normalization/ResourceNameNormalizer.java b/storm-server/src/main/java/org/apache/storm/scheduler/resource/normalization/ResourceNameNormalizer.java index 57ee34880e6..395860875c3 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/resource/normalization/ResourceNameNormalizer.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/resource/normalization/ResourceNameNormalizer.java @@ -37,14 +37,17 @@ public ResourceNameNormalizer() { Map tmp = new HashMap<>(); tmp.put(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT, Constants.COMMON_CPU_RESOURCE_NAME); tmp.put(Config.SUPERVISOR_CPU_CAPACITY, Constants.COMMON_CPU_RESOURCE_NAME); - tmp.put(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB, Constants.COMMON_ONHEAP_MEMORY_RESOURCE_NAME); - tmp.put(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB, Constants.COMMON_OFFHEAP_MEMORY_RESOURCE_NAME); + tmp.put(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB, + Constants.COMMON_ONHEAP_MEMORY_RESOURCE_NAME); + tmp.put(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB, + Constants.COMMON_OFFHEAP_MEMORY_RESOURCE_NAME); tmp.put(Config.SUPERVISOR_MEMORY_CAPACITY_MB, Constants.COMMON_TOTAL_MEMORY_RESOURCE_NAME); resourceNameMapping = Collections.unmodifiableMap(tmp); } /** - * Normalizes a supervisor resource map or topology details map's keys to universal resource names. + * Normalizes a supervisor resource map or topology details map's keys to universal resource + * names. * * @param resourceMap resource map of either Supervisor or Topology * @return the resource map with common resource names @@ -55,9 +58,10 @@ public Map normalizedResourceMap(Map r } return new HashMap<>(resourceMap.entrySet().stream() .collect(Collectors.toMap( - //Map the key if needed - (e) -> resourceNameMapping.getOrDefault(e.getKey(), e.getKey()), - //Map the value + // Map the key if needed + (e) -> resourceNameMapping.getOrDefault(e.getKey(), e + .getKey()), + // Map the value (e) -> e.getValue().doubleValue()))); } diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/priority/DefaultSchedulingPriorityStrategy.java b/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/priority/DefaultSchedulingPriorityStrategy.java index 0d943eb5057..b8136ddad8a 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/priority/DefaultSchedulingPriorityStrategy.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/priority/DefaultSchedulingPriorityStrategy.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

      - * http://www.apache.org/licenses/LICENSE-2.0 - *

      - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

      http://www.apache.org/licenses/LICENSE-2.0 + * + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -25,14 +31,16 @@ import org.slf4j.LoggerFactory; public class DefaultSchedulingPriorityStrategy implements ISchedulingPriorityStrategy { - private static final Logger LOG = LoggerFactory.getLogger(DefaultSchedulingPriorityStrategy.class); + private static final Logger LOG = LoggerFactory + .getLogger(DefaultSchedulingPriorityStrategy.class); protected SimulatedUser getSimulatedUserFor(User u, ISchedulingState cluster) { return new SimulatedUser(u, cluster); } @Override - public List getOrderedTopologies(ISchedulingState cluster, Map userMap) { + public List getOrderedTopologies(ISchedulingState cluster, Map userMap) { double cpuAvail = cluster.getClusterTotalCpuResource(); double memAvail = cluster.getClusterTotalMemoryResource(); @@ -93,21 +101,25 @@ public TopologyDetails simScheduleNextHighest() { } /** - * Get a score for the simulated user. This is used to sort the users, by their highest priority topology. - * The only requirement is that if the user is over their guarantees, or there are no available resources the + * Get a score for the simulated user. This is used to sort the users, by their highest + * priority topology. + * The only requirement is that if the user is over their guarantees, or there are no + * available resources the * returned score will be > 0. If they are under their guarantee it must be negative. + * * @param availableCpu available CPU on the cluster. * @param availableMemory available memory on the cluster. * @param td the topology we are looking at. * @return the score. */ protected double getScore(double availableCpu, double availableMemory, TopologyDetails td) { - //(Requested + Assigned - Guaranteed)/Available + // (Requested + Assigned - Guaranteed)/Available if (td == null || availableCpu <= 0 || availableMemory <= 0) { return Double.MAX_VALUE; } double wouldBeCpu = assignedCpu + td.getTotalRequestedCpu(); - double wouldBeMem = assignedMemory + td.getTotalRequestedMemOffHeap() + td.getTotalRequestedMemOnHeap(); + double wouldBeMem = assignedMemory + td.getTotalRequestedMemOffHeap() + td + .getTotalRequestedMemOnHeap(); double cpuScore = (wouldBeCpu - guaranteedCpu) / availableCpu; double memScore = (wouldBeMem - guaranteedMemory) / availableMemory; return Math.max(cpuScore, memScore); @@ -138,7 +150,8 @@ public int compare(SimulatedUser o1, SimulatedUser o2) { /** * Comparator that sorts topologies by priority and then by submission time. - * First sort by Topology Priority, if there is a tie for topology priority, topology uptime is used to sort. + * First sort by Topology Priority, if there is a tie for topology priority, topology uptime is + * used to sort. */ private static class TopologyByPriorityAndSubmissionTimeComparator implements Comparator { diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/priority/FIFOSchedulingPriorityStrategy.java b/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/priority/FIFOSchedulingPriorityStrategy.java index 849ffac1d35..df88c47d3d9 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/priority/FIFOSchedulingPriorityStrategy.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/priority/FIFOSchedulingPriorityStrategy.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

      - * http://www.apache.org/licenses/LICENSE-2.0 - *

      - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

      http://www.apache.org/licenses/LICENSE-2.0 + * + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -42,8 +48,8 @@ public double getScore(double availableCpu, double availableMemory) { if (origScore < 0) { return origScore; } - //Not enough guaranteed use the age of the topology instead. - //TODO: need a good way to only do this once... + // Not enough guaranteed use the age of the topology instead. + // TODO: need a good way to only do this once... Collections.sort(tds, new TopologyBySubmissionTimeComparator()); td = getNextHighest(); if (td != null) { diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/priority/GenericResourceAwareSchedulingPriorityStrategy.java b/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/priority/GenericResourceAwareSchedulingPriorityStrategy.java index c2a5eda538b..bd936f0f3b7 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/priority/GenericResourceAwareSchedulingPriorityStrategy.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/priority/GenericResourceAwareSchedulingPriorityStrategy.java @@ -25,16 +25,15 @@ import java.util.List; import java.util.Map; import java.util.Set; - import org.apache.storm.scheduler.ISchedulingState; import org.apache.storm.scheduler.TopologyDetails; import org.apache.storm.scheduler.resource.User; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class GenericResourceAwareSchedulingPriorityStrategy extends DefaultSchedulingPriorityStrategy { - private static final Logger LOG = LoggerFactory.getLogger(GenericResourceAwareSchedulingPriorityStrategy.class); + private static final Logger LOG = LoggerFactory + .getLogger(GenericResourceAwareSchedulingPriorityStrategy.class); @Override protected GrasSimulatedUser getSimulatedUserFor(User u, ISchedulingState cluster) { @@ -42,7 +41,8 @@ protected GrasSimulatedUser getSimulatedUserFor(User u, ISchedulingState cluster } @Override - public List getOrderedTopologies(ISchedulingState cluster, Map userMap) { + public List getOrderedTopologies(ISchedulingState cluster, Map userMap) { double cpuAvail = cluster.getClusterTotalCpuResource(); double memAvail = cluster.getClusterTotalMemoryResource(); Map genericAvail = cluster.getClusterTotalGenericResources(); @@ -55,7 +55,8 @@ public List getOrderedTopologies(ISchedulingState cluster, Map< } while (!users.isEmpty()) { - Collections.sort(users, new GrasSimulatedUserComparator(cpuAvail, memAvail, genericAvail)); + Collections.sort(users, new GrasSimulatedUserComparator(cpuAvail, memAvail, + genericAvail)); GrasSimulatedUser u = users.get(0); TopologyDetails td = u.getNextHighest(); if (td == null) { @@ -66,11 +67,13 @@ public List getOrderedTopologies(ISchedulingState cluster, Map< LOG.info("GRAS SIM Scheduling {} with score of {}", td.getId(), score); cpuAvail -= td.getTotalRequestedCpu(); memAvail -= (td.getTotalRequestedMemOffHeap() + td.getTotalRequestedMemOnHeap()); - for (Map.Entry entry : td.getTotalRequestedGenericResources().entrySet()) { + for (Map.Entry entry : td.getTotalRequestedGenericResources() + .entrySet()) { String resource = entry.getKey(); Double requestedAmount = entry.getValue(); if (!genericAvail.containsKey(resource)) { - LOG.warn("Resource: {} is not supported in this cluster. Ignoring this request.", resource); + LOG.warn("Resource: {} is not supported in this cluster. Ignoring this " + + "request.", resource); } else { genericAvail.put(resource, genericAvail.get(resource) - requestedAmount); } @@ -85,16 +88,19 @@ protected static class GrasSimulatedUser extends SimulatedUser { // Extend support for Generic Resources in addition to CPU and Memory public final Map guaranteedGenericResources; // resource name : guaranteed amount - private Map assignedGenericResources = new HashMap<>(); // resource name : assigned amount + private Map assignedGenericResources = + new HashMap<>(); // resource name : assigned amount public GrasSimulatedUser(User other, ISchedulingState cluster) { super(other, cluster); Map guaranteedGenericResources = new HashMap<>(); // generic resource types that are offered - Set availGenericResourceTypes = cluster.getClusterTotalGenericResources().keySet(); + Set availGenericResourceTypes = cluster.getClusterTotalGenericResources() + .keySet(); for (String resourceType : availGenericResourceTypes) { - Double guaranteedAmount = other.getGenericGuaranteed().getOrDefault(resourceType, 0.0); + Double guaranteedAmount = other.getGenericGuaranteed().getOrDefault(resourceType, + 0.0); guaranteedGenericResources.put(resourceType, guaranteedAmount); } this.guaranteedGenericResources = guaranteedGenericResources; @@ -107,23 +113,29 @@ public TopologyDetails simScheduleNextHighest() { for (Map.Entry entry : tdRequestedGenericResource.entrySet()) { String resource = entry.getKey(); Double requestedAmount = entry.getValue(); - assignedGenericResources.put(resource, assignedGenericResources.getOrDefault(resource, 0.0) + requestedAmount); + assignedGenericResources.put(resource, assignedGenericResources + .getOrDefault(resource, 0.0) + requestedAmount); } return td; } /** - * Get a score for the simulated user. This is used to sort the users, by their highest priority topology. + * Get a score for the simulated user. This is used to sort the users, by their highest + * priority topology. * Only give user guarantees that will not exceed cluster capacity. - * Score of each resource type is calculated as: (Requested + Assigned - Guaranteed)/clusterAvailable + * Score of each resource type is calculated as: (Requested + Assigned - + * Guaranteed)/clusterAvailable * The final score is a max over all resource types. * Topology score will fall into the following intervals if: * User is under quota (guarantee): [(-guarantee)/available : 0] * User is over quota: (0, infinity) - * Unfortunately, score below 0 does not guarantee that the topology will be scheduled due to resources fragmentation. + * Unfortunately, score below 0 does not guarantee that the topology will be scheduled due + * to resources fragmentation. + * * @param availableCpu available CPU on the cluster. * @param availableMemory available memory on the cluster. - * @param availableGenericResources available generic resources (other that cpu and memory) in cluster + * @param availableGenericResources available generic resources (other that cpu and memory) + * in cluster * @param td the topology we are looking at. * @return the score. */ @@ -146,15 +158,18 @@ protected double getScore(double availableCpu, double availableMemory, } Double wouldBeResource = assignedGenericResources.getOrDefault(resource, 0.0) + tdTotalRequestedGeneric.getOrDefault(resource, 0.0); - double thisScore = (wouldBeResource - guaranteedGenericResources.getOrDefault(resource, 0.0)) / available; + double thisScore = (wouldBeResource - guaranteedGenericResources + .getOrDefault(resource, 0.0)) / available; ret = Math.max(ret, thisScore); } return ret; } - protected double getScore(double availableCpu, double availableMemory, Map availableGenericResources) { - return getScore(availableCpu, availableMemory, availableGenericResources, getNextHighest()); + protected double getScore(double availableCpu, double availableMemory, Map availableGenericResources) { + return getScore(availableCpu, availableMemory, availableGenericResources, + getNextHighest()); } } @@ -163,7 +178,8 @@ private static class GrasSimulatedUserComparator implements Comparator genericAvail; - private GrasSimulatedUserComparator(double cpuAvail, double memAvail, Map genericAvail) { + private GrasSimulatedUserComparator(double cpuAvail, double memAvail, Map genericAvail) { this.cpuAvail = cpuAvail; this.memAvail = memAvail; this.genericAvail = genericAvail; @@ -171,7 +187,8 @@ private GrasSimulatedUserComparator(double cpuAvail, double memAvail, Map - * http://www.apache.org/licenses/LICENSE-2.0 - *

      - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

      http://www.apache.org/licenses/LICENSE-2.0 + * + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -22,7 +28,9 @@ public interface ISchedulingPriorityStrategy { /** * Prioritize the list of all topologies in the cluster. + * * @return ordered list of topologies to schedule. */ - List getOrderedTopologies(ISchedulingState schedulingState, Map userMap); + List getOrderedTopologies(ISchedulingState schedulingState, Map userMap); } diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/BaseResourceAwareStrategy.java b/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/BaseResourceAwareStrategy.java index e686e1a093f..aa6460d5e33 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/BaseResourceAwareStrategy.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/BaseResourceAwareStrategy.java @@ -26,7 +26,6 @@ import java.util.List; import java.util.Map; import java.util.Set; - import org.apache.storm.Config; import org.apache.storm.DaemonConfig; import org.apache.storm.daemon.Acker; @@ -56,11 +55,13 @@ public abstract class BaseResourceAwareStrategy implements IStrategy { /** * Different node sorting types available. Two of these are for backward compatibility. * The last one (COMMON) is the new sorting type used across the board. - * Refer to {@link NodeSorter#NodeSorter(Cluster, TopologyDetails, NodeSortType)} for more details. + * Refer to {@link NodeSorter#NodeSorter(Cluster, TopologyDetails, NodeSortType)} for more + * details. */ public enum NodeSortType { /** * Generic Resource Aware Strategy sorting type. + * * @deprecated used by GenericResourceAwareStrategyOld only. Use {link #COMMON} instead. */ @Deprecated @@ -68,6 +69,7 @@ public enum NodeSortType { /** * Default Resource Aware Strategy sorting type. + * * @deprecated used by DefaultResourceAwareStrategyOld only. Use {link #COMMON} instead. */ @Deprecated @@ -116,7 +118,8 @@ public BaseResourceAwareStrategy() { * Initialize for the default implementation of schedule(). * * @param sortNodesForEachExecutor Sort nodes before scheduling each executor. - * @param nodeSortType type of sorting to be applied to object resource collection {@link NodeSortType}. + * @param nodeSortType type of sorting to be applied to object resource collection {@link + * NodeSortType}. */ public BaseResourceAwareStrategy(boolean sortNodesForEachExecutor, NodeSortType nodeSortType) { this.sortNodesForEachExecutor = sortNodesForEachExecutor; @@ -135,13 +138,13 @@ public void prepare(Map config) { *

    • {@link #searcherState}
    • *
    • {@link #execSorter} to sort executors
    • *
    • {@link #nodeSorter} to sort nodes
    • - *

      - * Scheduling consists of three main steps: + * + *

      Scheduling consists of three main steps: *

    • {@link #prepareForScheduling(Cluster, TopologyDetails)}
    • *
    • {@link #checkSchedulingFeasibility()}, and
    • *
    • {@link #scheduleExecutorsOnNodes(List, Iterable)}
    • - *

      - * The executors and nodes are sorted in the order most conducive to scheduling for the strategy. + *

      The executors and nodes are sorted in the order most conducive to scheduling for the + * strategy. * Those interfaces may be overridden by subclasses using mutators: *

    • {@link #setExecSorter(IExecSorter)} and
    • *
    • {@link #setNodeSorter(INodeSorter)}
    • @@ -160,9 +163,10 @@ public SchedulingResult schedule(Cluster cluster, TopologyDetails td) { return earlyResult; } - LOG.debug("Topology {} {} Number of ExecutorsNeedScheduling: {}", topoName, topologyDetails.getId(), unassignedExecutors.size()); + LOG.debug("Topology {} {} Number of ExecutorsNeedScheduling: {}", topoName, topologyDetails + .getId(), unassignedExecutors.size()); - //order executors to be scheduled + // order executors to be scheduled List orderedExecutors = execSorter.sortExecutors(unassignedExecutors); isolateAckersToEnd(orderedExecutors); Iterable sortedNodes = null; @@ -174,7 +178,8 @@ public SchedulingResult schedule(Cluster cluster, TopologyDetails td) { } /** - * Initialize instance variables as the first step in {@link #schedule(Cluster, TopologyDetails)}. + * Initialize instance variables as the first step in {@link #schedule(Cluster, + * TopologyDetails)}. * This method may be extended by subclasses to initialize additional variables as in * {@link ConstraintSolverStrategy#prepareForScheduling(Cluster, TopologyDetails)}. * @@ -200,12 +205,16 @@ protected void prepareForScheduling(Cluster cluster, TopologyDetails topologyDet // From Cluster and TopologyDetails - and cleaned-up // all unassigned executors including system components execs - unassignedExecutors = Collections.unmodifiableSet(new HashSet<>(cluster.getUnassignedExecutors(topologyDetails))); + unassignedExecutors = Collections.unmodifiableSet(new HashSet<>(cluster + .getUnassignedExecutors(topologyDetails))); int confMaxStateSearch = getMaxStateSearchFromTopoConf(topologyDetails.getConf()); - int daemonMaxStateSearch = ObjectReader.getInt(cluster.getConf().get(DaemonConfig.RESOURCE_AWARE_SCHEDULER_MAX_STATE_SEARCH)); + int daemonMaxStateSearch = ObjectReader.getInt(cluster.getConf() + .get(DaemonConfig.RESOURCE_AWARE_SCHEDULER_MAX_STATE_SEARCH)); maxStateSearch = Math.min(daemonMaxStateSearch, confMaxStateSearch); - LOG.debug("The max state search configured by topology {} is {}", topologyDetails.getId(), confMaxStateSearch); - LOG.debug("The max state search that will be used by topology {} is {}", topologyDetails.getId(), maxStateSearch); + LOG.debug("The max state search configured by topology {} is {}", topologyDetails.getId(), + confMaxStateSearch); + LOG.debug("The max state search that will be used by topology {} is {}", topologyDetails + .getId(), maxStateSearch); searcherState = createSearcherState(); setNodeSorter(new NodeSorterHostProximity(cluster, topologyDetails, nodeSortType)); @@ -235,18 +244,24 @@ protected void setNodeSorter(INodeSorter nodeSorter) { } private static long computeMaxSchedulingTimeMs(Map topoConf) { - // expect to be killed by DaemonConfig.SCHEDULING_TIMEOUT_SECONDS_PER_TOPOLOGY seconds, terminate slightly before - int daemonMaxTimeSec = ObjectReader.getInt(topoConf.get(DaemonConfig.SCHEDULING_TIMEOUT_SECONDS_PER_TOPOLOGY), 60); - int confMaxTimeSec = ObjectReader.getInt(topoConf.get(Config.TOPOLOGY_RAS_CONSTRAINT_MAX_TIME_SECS), daemonMaxTimeSec); - return (confMaxTimeSec >= daemonMaxTimeSec) ? daemonMaxTimeSec * 1000L - 200L : confMaxTimeSec * 1000L; + // expect to be killed by DaemonConfig.SCHEDULING_TIMEOUT_SECONDS_PER_TOPOLOGY seconds, + // terminate slightly before + int daemonMaxTimeSec = ObjectReader.getInt(topoConf + .get(DaemonConfig.SCHEDULING_TIMEOUT_SECONDS_PER_TOPOLOGY), 60); + int confMaxTimeSec = ObjectReader.getInt(topoConf + .get(Config.TOPOLOGY_RAS_CONSTRAINT_MAX_TIME_SECS), daemonMaxTimeSec); + return (confMaxTimeSec >= daemonMaxTimeSec) + ? daemonMaxTimeSec * 1000L - 200L : confMaxTimeSec * 1000L; } public static int getMaxStateSearchFromTopoConf(Map topoConf) { int confMaxStateSearch; if (topoConf.containsKey(Config.TOPOLOGY_RAS_CONSTRAINT_MAX_STATE_SEARCH)) { - //this config is always set for topologies of 2.0 or newer versions since it is in defaults.yaml file - //topologies of older versions can also use it if configures it explicitly - confMaxStateSearch = ObjectReader.getInt(topoConf.get(Config.TOPOLOGY_RAS_CONSTRAINT_MAX_STATE_SEARCH)); + // this config is always set for topologies of 2.0 or newer versions since it is in + // defaults.yaml file + // topologies of older versions can also use it if configures it explicitly + confMaxStateSearch = ObjectReader.getInt(topoConf + .get(Config.TOPOLOGY_RAS_CONSTRAINT_MAX_STATE_SEARCH)); } else { // For backwards compatibility confMaxStateSearch = 10_000; @@ -255,12 +270,14 @@ public static int getMaxStateSearchFromTopoConf(Map topoConf) { } public static boolean isOrderByProximity(Map topoConf) { - return ObjectReader.getBoolean(topoConf.get(Config.TOPOLOGY_RAS_ORDER_EXECUTORS_BY_PROXIMITY_NEEDS), false); + return ObjectReader.getBoolean(topoConf + .get(Config.TOPOLOGY_RAS_ORDER_EXECUTORS_BY_PROXIMITY_NEEDS), false); } /** * Create an instance of {@link SchedulingSearcherState}. This method is called by - * {@link #prepareForScheduling(Cluster, TopologyDetails)} and depends on variables initialized therein prior. + * {@link #prepareForScheduling(Cluster, TopologyDetails)} and depends on variables initialized + * therein prior. * * @return a new instance of {@link SchedulingSearcherState}. */ @@ -268,15 +285,16 @@ private SchedulingSearcherState createSearcherState() { Map> workerCompCnts = new HashMap<>(); Map> nodeCompCnts = new HashMap<>(); - //populate with existing assignments + // populate with existing assignments SchedulerAssignment existingAssignment = cluster.getAssignmentById(topologyDetails.getId()); if (existingAssignment != null) { existingAssignment.getExecutorToSlot().forEach((exec, ws) -> { String compId = execToComp.get(exec); RasNode node = nodes.getNodeById(ws.getNodeId()); - Map compCnts = nodeCompCnts.computeIfAbsent(node, (k) -> new HashMap<>()); + Map compCnts = nodeCompCnts.computeIfAbsent(node, + (k) -> new HashMap<>()); compCnts.put(compId, compCnts.getOrDefault(compId, 0) + 1); // increment - //populate worker to comp assignments + // populate worker to comp assignments compCnts = workerCompCnts.computeIfAbsent(ws, (k) -> new HashMap<>()); compCnts.put(compId, compCnts.getOrDefault(compId, 0) + 1); // increment }); @@ -296,17 +314,21 @@ private SchedulingSearcherState createSearcherState() { } /** - * Check scheduling feasibility for a quick failure as the second step in {@link #schedule(Cluster, TopologyDetails)}. + * Check scheduling feasibility for a quick failure as the second step in {@link + * #schedule(Cluster, TopologyDetails)}. * If scheduling is not possible, then return a SchedulingStatus object with a failure status. * If fully scheduled then return a successful SchedulingStatus. - * This method can be extended by subclasses {@link ConstraintSolverStrategy#checkSchedulingFeasibility()} + * This method can be extended by subclasses {@link + * ConstraintSolverStrategy#checkSchedulingFeasibility()} * to check for additional failure conditions. * - * @return A non-null {@link SchedulingResult} to terminate scheduling, otherwise return null to continue scheduling. + * @return A non-null {@link SchedulingResult} to terminate scheduling, otherwise return null to + * continue scheduling. */ protected SchedulingResult checkSchedulingFeasibility() { if (unassignedExecutors.isEmpty()) { - return SchedulingResult.success("Fully Scheduled by " + this.getClass().getSimpleName()); + return SchedulingResult.success("Fully Scheduled by " + this.getClass() + .getSimpleName()); } String err; @@ -324,7 +346,9 @@ protected SchedulingResult checkSchedulingFeasibility() { int execCnt = unassignedExecutors.size(); if (execCnt >= maxStateSearch) { - err = String.format("Unassignerd Executor count (%d) is greater than searchable state count %d", execCnt, maxStateSearch); + err = String + .format("Unassignerd Executor count (%d) is greater than searchable state " + + "count %d", execCnt, maxStateSearch); LOG.error("Topology {}:{}", topoName, err); return SchedulingResult.failure(SchedulingStatus.FAIL_OTHER, err); } @@ -334,19 +358,22 @@ protected SchedulingResult checkSchedulingFeasibility() { /** * Check if the assignment of the executor to the worker is valid. In simple cases, - * this is simply a check of {@link RasNode#wouldFit(WorkerSlot, ExecutorDetails, TopologyDetails)}. + * this is simply a check of {@link RasNode#wouldFit(WorkerSlot, ExecutorDetails, + * TopologyDetails)}. * This method may be extended by subclasses to add additional checks, - * see {@link ConstraintSolverStrategy#isExecAssignmentToWorkerValid(ExecutorDetails, WorkerSlot)}. + * see {@link ConstraintSolverStrategy#isExecAssignmentToWorkerValid(ExecutorDetails, + * WorkerSlot)}. * * @param exec being scheduled. * @param worker on which to schedule. * @return true if executor can be assigned to the worker, false otherwise. */ protected boolean isExecAssignmentToWorkerValid(ExecutorDetails exec, WorkerSlot worker) { - //check resources + // check resources RasNode node = nodes.getNodeById(worker.getNodeId()); if (!node.wouldFit(worker, exec, topologyDetails)) { - LOG.trace("Topology {}, executor {} would not fit in resources available on worker {}", topoName, exec, worker); + LOG.trace("Topology {}, executor {} would not fit in resources available on worker {}", + topoName, exec, worker); return false; } return true; @@ -381,7 +408,7 @@ private void logClusterInfo() { } /** - * hostname to Ids. + * Hostname to Ids. * * @param hostname the hostname. * @return the ids n that node. @@ -419,11 +446,14 @@ private void isolateAckersToEnd(List orderedExecutors) { /** * Try to schedule till successful or till limits (backtrack count or time) have been exceeded. * - * @param orderedExecutors Executors sorted in the preferred order cannot be null - note that ackers are isolated at the end. + * @param orderedExecutors Executors sorted in the preferred order cannot be null - note that + * ackers are isolated at the end. * @param sortedNodesIter Node iterable which may be null. - * @return SchedulingResult with success attribute set to true or false indicting whether ALL executors were assigned. + * @return SchedulingResult with success attribute set to true or false indicting whether ALL + * executors were assigned. */ - protected SchedulingResult scheduleExecutorsOnNodes(List orderedExecutors, Iterable sortedNodesIter) { + protected SchedulingResult scheduleExecutorsOnNodes(List orderedExecutors, + Iterable sortedNodesIter) { long startTimeMilli = Time.currentTimeMillis(); searcherState.setSortedExecs(orderedExecutors); @@ -438,14 +468,17 @@ protected SchedulingResult scheduleExecutorsOnNodes(List ordere for (int i = 0; i < maxExecCnt; i++) { progressIdxForExec[i] = -1; } - LOG.debug("scheduleExecutorsOnNodes: will assign {} executors for topo {}, sortNodesForEachExecutor={}", + LOG.debug("scheduleExecutorsOnNodes: will assign {} executors for topo {}, " + + "sortNodesForEachExecutor={}", maxExecCnt, topoName, sortNodesForEachExecutor); OUTERMOST_LOOP: for (int loopCnt = 0; true; loopCnt++) { - LOG.debug("scheduleExecutorsOnNodes: loopCnt={}, execIndex={}, topo={}", loopCnt, searcherState.getExecIndex(), topoName); + LOG.debug("scheduleExecutorsOnNodes: loopCnt={}, execIndex={}, topo={}", loopCnt, + searcherState.getExecIndex(), topoName); if (searcherState.areSearchLimitsExceeded()) { - LOG.warn("Limits exceeded, backtrackCnt={}, loopCnt={}, topo={}", searcherState.getNumBacktrack(), loopCnt, topoName); + LOG.warn("Limits exceeded, backtrackCnt={}, loopCnt={}, topo={}", searcherState + .getNumBacktrack(), loopCnt, topoName); return searcherState.createSchedulingResult(false, this.getClass().getSimpleName()); } @@ -461,20 +494,23 @@ protected SchedulingResult scheduleExecutorsOnNodes(List ordere // So we skip to the next. if (searcherState.getBoundAckers().contains(exec)) { if (searcherState.areAllExecsScheduled()) { - //Everything is scheduled correctly, so no need to search any more. - LOG.info("scheduleExecutorsOnNodes: Done at loopCnt={} in {}ms, state.elapsedtime={}, backtrackCnt={}, topo={}", + // Everything is scheduled correctly, so no need to search any more. + LOG.info("scheduleExecutorsOnNodes: Done at loopCnt={} in {}ms, " + + "state.elapsedtime={}, backtrackCnt={}, topo={}", loopCnt, Time.currentTimeMillis() - startTimeMilli, Time.currentTimeMillis() - searcherState.startTimeMillis, searcherState.getNumBacktrack(), topoName); - return searcherState.createSchedulingResult(true, this.getClass().getSimpleName()); + return searcherState.createSchedulingResult(true, this.getClass() + .getSimpleName()); } searcherState = searcherState.nextExecutor(); continue OUTERMOST_LOOP; } String comp = execToComp.get(exec); - if (sortedNodesIter == null || (this.sortNodesForEachExecutor && searcherState.isExecCompDifferentFromPrior())) { + if (sortedNodesIter == null || (this.sortNodesForEachExecutor && searcherState + .isExecCompDifferentFromPrior())) { progressIdx = -1; nodeSorter.prepare(exec); sortedNodesIter = nodeSorter.sortAllNodes(); @@ -494,42 +530,51 @@ protected SchedulingResult scheduleExecutorsOnNodes(List ordere if (!isExecAssignmentToWorkerValid(exec, workerSlot)) { // exec can't fit in this workerSlot, try next workerSlot - LOG.trace("Failed to assign exec={}, comp={}, topo={} to worker={} on node=({}, availCpu={}, availMem={}).", + LOG.trace("Failed to assign exec={}, comp={}, topo={} to worker={} on " + + "node=({}, availCpu={}, availMem={}).", exec, comp, topoName, workerSlot, - node.getId(), node.getAvailableCpuResources(), node.getAvailableMemoryResources()); + node.getId(), node.getAvailableCpuResources(), node + .getAvailableMemoryResources()); continue; } searcherState.incStatesSearched(); searcherState.assignCurrentExecutor(execToComp, node, workerSlot); - int numBoundAckerAssigned = assignBoundAckersForNewWorkerSlot(exec, node, workerSlot); + int numBoundAckerAssigned = assignBoundAckersForNewWorkerSlot(exec, node, + workerSlot); if (numBoundAckerAssigned > 0) { - // This exec with some of its bounded ackers have all been successfully assigned + // This exec with some of its bounded ackers have all been successfully + // assigned searcherState.getExecsWithBoundAckers().add(exec); } if (searcherState.areAllExecsScheduled()) { - //Everything is scheduled correctly, so no need to search any more. - LOG.info("scheduleExecutorsOnNodes: Done at loopCnt={} in {}ms, state.elapsedtime={}, backtrackCnt={}, topo={}", + // Everything is scheduled correctly, so no need to search any more. + LOG.info("scheduleExecutorsOnNodes: Done at loopCnt={} in {}ms, " + + "state.elapsedtime={}, backtrackCnt={}, topo={}", loopCnt, Time.currentTimeMillis() - startTimeMilli, Time.currentTimeMillis() - searcherState.startTimeMillis, searcherState.getNumBacktrack(), topoName); - return searcherState.createSchedulingResult(true, this.getClass().getSimpleName()); + return searcherState.createSchedulingResult(true, this.getClass() + .getSimpleName()); } searcherState = searcherState.nextExecutor(); nodeForExec[execIndex] = node; workerSlotForExec[execIndex] = workerSlot; - LOG.debug("scheduleExecutorsOnNodes: Assigned execId={}, comp={} to node={}/cpu={}/mem={}, " + LOG.debug("scheduleExecutorsOnNodes: Assigned execId={}, comp={} to " + + "node={}/cpu={}/mem={}, " + "slot-ordinal={} at loopCnt={}, topo={}", - execIndex, comp, nodeId, node.getAvailableCpuResources(), node.getAvailableMemoryResources(), + execIndex, comp, nodeId, node.getAvailableCpuResources(), node + .getAvailableMemoryResources(), progressIdx, loopCnt, topoName); continue OUTERMOST_LOOP; } } sortedNodesIter = null; // if here, then the executor was not assigned, backtrack; - LOG.debug("scheduleExecutorsOnNodes: Failed to schedule execId={}, comp={} at loopCnt={}, topo={}", + LOG.debug("scheduleExecutorsOnNodes: Failed to schedule execId={}, comp={} at " + + "loopCnt={}, topo={}", execIndex, comp, loopCnt, topoName); if (execIndex == 0) { break; @@ -539,16 +584,17 @@ protected SchedulingResult scheduleExecutorsOnNodes(List ordere } } boolean success = searcherState.areAllExecsScheduled(); - LOG.info("scheduleExecutorsOnNodes: Scheduled={} in {} milliseconds, state.elapsedtime={}, backtrackCnt={}, topo={}", - success, Time.currentTimeMillis() - startTimeMilli, Time.currentTimeMillis() - searcherState.startTimeMillis, + LOG.info("scheduleExecutorsOnNodes: Scheduled={} in {} milliseconds, " + + "state.elapsedtime={}, backtrackCnt={}, topo={}", + success, Time.currentTimeMillis() - startTimeMilli, Time + .currentTimeMillis() - searcherState.startTimeMillis, searcherState.getNumBacktrack(), topoName); return searcherState.createSchedulingResult(success, this.getClass().getSimpleName()); } /** - *

      - * Determine how many bound ackers to put into the given workerSlot. + *

      Determine how many bound ackers to put into the given workerSlot. * Then try to assign the ackers one by one into this workerSlot upto the calculated * maximum required. Return the number of ackers assigned. * @@ -558,24 +604,29 @@ protected SchedulingResult scheduleExecutorsOnNodes(List ordere * 3. No ackers could be assigned because of space or exception. * *

      + * * @param exec being scheduled. * @param node RasNode on which to schedule. * @param workerSlot WorkerSlot on which to schedule. * @return Number of ackers assigned. */ - protected int assignBoundAckersForNewWorkerSlot(ExecutorDetails exec, RasNode node, WorkerSlot workerSlot) { + protected int assignBoundAckersForNewWorkerSlot(ExecutorDetails exec, RasNode node, + WorkerSlot workerSlot) { int numOfAckersToBind = searcherState.getNumOfAckersToBind(exec, workerSlot); if (numOfAckersToBind > 0) { for (int i = 0; i < numOfAckersToBind; i++) { - if (!isExecAssignmentToWorkerValid(searcherState.peekUnassignedAckers(), workerSlot)) { - LOG.debug("Assigned {} of {} ackers on workerSlot={} with the executor={} for topology={}", + if (!isExecAssignmentToWorkerValid(searcherState.peekUnassignedAckers(), + workerSlot)) { + LOG.debug("Assigned {} of {} ackers on workerSlot={} with the executor={} for " + + "topology={}", i, numOfAckersToBind, workerSlot, exec, topoName); return i; } try { searcherState.assignSingleBoundAcker(node, workerSlot); } catch (Exception e) { - LOG.error("Exception happens when assigning {} of {} ackers on workerSlot={} for topology={}", + LOG.error("Exception happens when assigning {} of {} ackers on workerSlot={} " + + "for topology={}", i + 1, numOfAckersToBind, workerSlot, topoName, e); return i; } diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/ConstraintSolverConfig.java b/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/ConstraintSolverConfig.java index 041fc375692..a29fda7b7be 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/ConstraintSolverConfig.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/ConstraintSolverConfig.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

      - * http://www.apache.org/licenses/LICENSE-2.0 - *

      - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

      http://www.apache.org/licenses/LICENSE-2.0 + * + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -19,7 +25,6 @@ import java.util.List; import java.util.Map; import java.util.Set; - import org.apache.storm.Config; import org.apache.storm.scheduler.TopologyDetails; import org.slf4j.Logger; @@ -27,15 +32,19 @@ /** * Component constraint as derived from configuration. - * This is backward compatible and can parse old style Config.TOPOLOGY_RAS_CONSTRAINTS and Config.TOPOLOGY_SPREAD_COMPONENTS. - * New style Config.TOPOLOGY_RAS_CONSTRAINTS is map where each component has a list of other incompatible components + * This is backward compatible and can parse old style Config.TOPOLOGY_RAS_CONSTRAINTS and + * Config.TOPOLOGY_SPREAD_COMPONENTS. + * New style Config.TOPOLOGY_RAS_CONSTRAINTS is map where each component has a list of other + * incompatible components * and an optional number that specifies the maximum co-location count for the component on a node. * - *

      comp-1 cannot exist on same worker as comp-2 or comp-3, and at most "2" comp-1 on same node

      - *

      comp-2 and comp-4 cannot be on same worker (missing comp-1 is implied from comp-1 constraint)

      + *

      comp-1 cannot exist on same worker as comp-2 or comp-3, and at most "2" comp-1 on same + * node

      * - *

      - * { "comp-1": { "maxNodeCoLocationCnt": 2, "incompatibleComponents": ["comp-2", "comp-3" ] }, + *

      comp-2 and comp-4 cannot be on same worker (missing comp-1 is implied from comp-1 + * constraint)

      + * + *

      { "comp-1": { "maxNodeCoLocationCnt": 2, "incompatibleComponents": ["comp-2", "comp-3" ] }, * "comp-2": { "incompatibleComponents": [ "comp-4" ] } * } *

      @@ -46,10 +55,11 @@ public final class ConstraintSolverConfig { public static final String CONSTRAINT_TYPE_MAX_NODE_CO_LOCATION_CNT = "maxNodeCoLocationCnt"; public static final String CONSTRAINT_TYPE_INCOMPATIBLE_COMPONENTS = "incompatibleComponents"; - /** constraint limiting which components cannot co-exist on same worker. + /** + * Constraint limiting which components cannot co-exist on same worker. * ALL components are represented, some with empty set of components. */ private Map> incompatibleComponentSets = new HashMap<>(); - /** constraint limiting executor instances of component on a node. */ + /** Constraint limiting executor instances of component on a node. */ private Map maxNodeCoLocationCnts = new HashMap<>(); private final Map topoConf; @@ -72,7 +82,8 @@ private void computeComponentConstraints() { comps.forEach(k -> incompatibleComponentSets.computeIfAbsent(k, x -> new HashSet<>())); Object rasConstraints = topoConf.get(Config.TOPOLOGY_RAS_CONSTRAINTS); if (rasConstraints == null) { - LOG.warn("TopoId {}: No config supplied for {}", topoId, Config.TOPOLOGY_RAS_CONSTRAINTS); + LOG.warn("TopoId {}: No config supplied for {}", topoId, + Config.TOPOLOGY_RAS_CONSTRAINTS); } else if (rasConstraints instanceof List) { // old style List> constraints = (List>) rasConstraints; @@ -80,18 +91,21 @@ private void computeComponentConstraints() { String comp1 = constraintPair.get(0); String comp2 = constraintPair.get(1); if (!comps.contains(comp1)) { - LOG.warn("TopoId {}: Comp {} declared in constraints is not valid!", topoId, comp1); + LOG.warn("TopoId {}: Comp {} declared in constraints is not valid!", topoId, + comp1); continue; } if (!comps.contains(comp2)) { - LOG.warn("TopoId {}: Comp {} declared in constraints is not valid!", topoId, comp2); + LOG.warn("TopoId {}: Comp {} declared in constraints is not valid!", topoId, + comp2); continue; } incompatibleComponentSets.get(comp1).add(comp2); incompatibleComponentSets.get(comp2).add(comp1); } } else { - Map> constraintMap = (Map>) rasConstraints; + Map> constraintMap = (Map>) rasConstraints; constraintMap.forEach((comp1, v) -> { if (comps.contains(comp1)) { v.forEach((ctype, constraint) -> { @@ -100,28 +114,33 @@ private void computeComponentConstraints() { try { int numValue = Integer.parseInt("" + constraint); if (numValue < 1) { - LOG.warn("TopoId {}: {} {} declared for Comp {} is not valid, expected >= 1", + LOG.warn("TopoId {}: {} {} declared for Comp {} is not " + + "valid, expected >= 1", topoId, ctype, numValue, comp1); } else { maxNodeCoLocationCnts.put(comp1, numValue); } } catch (Exception ex) { - LOG.warn("TopoId {}: {} {} declared for Comp {} for topoId {} is not valid, expected >= 1", + LOG.warn("TopoId {}: {} {} declared for Comp {} for topoId {} " + + "is not valid, expected >= 1", topoId, ctype, constraint, comp1); } break; case CONSTRAINT_TYPE_INCOMPATIBLE_COMPONENTS: if (!(constraint instanceof List || constraint instanceof String)) { - LOG.warn("TopoId {}: {} {} declared for Comp {} is not valid, expecting a list of Comps or 1 Comp", + LOG.warn("TopoId {}: {} {} declared for Comp {} is not valid, " + + "expecting a list of Comps or 1 Comp", topoId, ctype, constraint, comp1); break; } List list; - list = (constraint instanceof String) ? Arrays.asList((String) constraint) : (List) constraint; + list = (constraint instanceof String) ? Arrays + .asList((String) constraint) : (List) constraint; for (String comp2 : list) { if (!comps.contains(comp2)) { - LOG.warn("TopoId {}: {} {} declared for Comp {} is not a valid Comp", topoId, ctype, comp2, comp1); + LOG.warn("TopoId {}: {} {} declared for Comp {} is not a " + + "valid Comp", topoId, ctype, comp2, comp1); continue; } incompatibleComponentSets.get(comp1).add(comp2); @@ -130,7 +149,8 @@ private void computeComponentConstraints() { break; default: - LOG.warn("TopoId {}: ConstraintType={} invalid for Comp={}, valid values are {} and {}, ignoring value={}", + LOG.warn("TopoId {}: ConstraintType={} invalid for Comp={}, valid " + + "values are {} and {}, ignoring value={}", topoId, ctype, comp1, CONSTRAINT_TYPE_MAX_NODE_CO_LOCATION_CNT, CONSTRAINT_TYPE_INCOMPATIBLE_COMPONENTS, constraint); break; @@ -152,18 +172,22 @@ private void computeComponentConstraints() { List spread = (List) obj; for (String comp : spread) { if (!comps.contains(comp)) { - LOG.warn("TopoId {}: Invalid Component {} declared in spread {}", topoId, comp, spread); + LOG.warn("TopoId {}: Invalid Component {} declared in spread {}", topoId, comp, + spread); continue; } if (maxNodeCoLocationCnts.containsKey(comp)) { - LOG.warn("TopoId {}: Component {} maxNodeCoLocationCnt={} already defined in {}, ignoring spread config in {}", topoId, - comp, maxNodeCoLocationCnts.get(comp), Config.TOPOLOGY_RAS_CONSTRAINTS, Config.TOPOLOGY_SPREAD_COMPONENTS); + LOG.warn("TopoId {}: Component {} maxNodeCoLocationCnt={} already defined in " + + "{}, ignoring spread config in {}", topoId, + comp, maxNodeCoLocationCnts + .get(comp), Config.TOPOLOGY_RAS_CONSTRAINTS, Config.TOPOLOGY_SPREAD_COMPONENTS); continue; } maxNodeCoLocationCnts.put(comp, 1); } } else { - LOG.warn("TopoId {}: Ignoring invalid {} config={}", topoId, Config.TOPOLOGY_SPREAD_COMPONENTS, obj); + LOG.warn("TopoId {}: Ignoring invalid {} config={}", topoId, + Config.TOPOLOGY_SPREAD_COMPONENTS, obj); } } @@ -171,7 +195,8 @@ private void computeComponentConstraints() { * Return an object that maps component names to a set of other components * which are incompatible and their executor instances cannot co-exist on the * same worker. - * The map will contain entries only for components that have this {@link #CONSTRAINT_TYPE_INCOMPATIBLE_COMPONENTS} + * The map will contain entries only for components that have this {@link + * #CONSTRAINT_TYPE_INCOMPATIBLE_COMPONENTS} * constraint specified. * * @return a map of component to a set of components that cannot co-exist on the same worker. @@ -183,7 +208,8 @@ public Map> getIncompatibleComponentSets() { /** * Return an object that maps component names to a numeric maximum limit of * executor instances (of that component) that can exist on any node. - * The map will contain entries only for components that have this {@link #CONSTRAINT_TYPE_MAX_NODE_CO_LOCATION_CNT} + * The map will contain entries only for components that have this {@link + * #CONSTRAINT_TYPE_MAX_NODE_CO_LOCATION_CNT} * constraint specified. * * @return a map of component to its maximum limit of executor instances on a node. diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/ConstraintSolverStrategy.java b/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/ConstraintSolverStrategy.java index 0ef2ca49d64..7b038ee4f6f 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/ConstraintSolverStrategy.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/ConstraintSolverStrategy.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

      - * http://www.apache.org/licenses/LICENSE-2.0 - *

      - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

      http://www.apache.org/licenses/LICENSE-2.0 + * + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -37,7 +43,8 @@ public class ConstraintSolverStrategy extends BaseResourceAwareStrategy { private static final Logger LOG = LoggerFactory.getLogger(ConstraintSolverStrategy.class); /** - * Instance variables initialized in first step {@link #prepareForScheduling(Cluster, TopologyDetails)} of + * Instance variables initialized in first step {@link #prepareForScheduling(Cluster, + * TopologyDetails)} of * schedule method {@link #schedule(Cluster, TopologyDetails)}. */ private ConstraintSolverConfig constraintSolverConfig; @@ -58,14 +65,17 @@ protected SchedulingResult checkSchedulingFeasibility() { return res; } if (!isSchedulingFeasible()) { - return SchedulingResult.failure(SchedulingStatus.FAIL_OTHER, "Scheduling not feasible!"); + return SchedulingResult.failure(SchedulingStatus.FAIL_OTHER, + "Scheduling not feasible!"); } return null; } /** * Check if any constraints are violated if exec is scheduled on worker. - * @return true if scheduling exec on worker does not violate any constraints, returns false if it does + * + * @return true if scheduling exec on worker does not violate any constraints, returns false if + * it does */ @Override protected boolean isExecAssignmentToWorkerValid(ExecutorDetails exec, WorkerSlot worker) { @@ -74,14 +84,17 @@ protected boolean isExecAssignmentToWorkerValid(ExecutorDetails exec, WorkerSlot } // check if executor can be on worker based on component exclusions String execComp = execToComp.get(exec); - Map compAssignmentCnts = searcherState.getCompAssignmentCntMapForWorker(worker); + Map compAssignmentCnts = searcherState + .getCompAssignmentCntMapForWorker(worker); Set incompatibleComponents; if (compAssignmentCnts != null - && (incompatibleComponents = constraintSolverConfig.getIncompatibleComponentSets().get(execComp)) != null + && (incompatibleComponents = constraintSolverConfig.getIncompatibleComponentSets() + .get(execComp)) != null && !incompatibleComponents.isEmpty()) { for (String otherComp : compAssignmentCnts.keySet()) { if (incompatibleComponents.contains(otherComp)) { - LOG.debug("Topology {}, exec={} with comp={} has constraint violation with comp={} on worker={}", + LOG.debug("Topology {}, exec={} with comp={} has constraint violation with " + + "comp={} on worker={}", topoName, exec, execComp, otherComp, worker); return false; } @@ -89,13 +102,15 @@ protected boolean isExecAssignmentToWorkerValid(ExecutorDetails exec, WorkerSlot } // check if executor can be on worker based on component node co-location constraint - Map maxNodeCoLocationCnts = constraintSolverConfig.getMaxNodeCoLocationCnts(); + Map maxNodeCoLocationCnts = constraintSolverConfig + .getMaxNodeCoLocationCnts(); if (maxNodeCoLocationCnts.containsKey(execComp)) { int coLocationMaxCnt = maxNodeCoLocationCnts.get(execComp); RasNode node = nodes.getNodeById(worker.getNodeId()); int compCntOnNode = searcherState.getComponentCntOnNode(node, execComp); if (compCntOnNode >= coLocationMaxCnt) { - LOG.debug("Topology {}, exec={} with comp={} has MaxCoLocationCnt violation on node {}, count {} >= colocation count {}", + LOG.debug("Topology {}, exec={} with comp={} has MaxCoLocationCnt violation on " + + "node {}, count {} >= colocation count {}", topoName, exec, execComp, node.getId(), compCntOnNode, coLocationMaxCnt); return false; } @@ -107,17 +122,21 @@ protected boolean isExecAssignmentToWorkerValid(ExecutorDetails exec, WorkerSlot * Determines if a scheduling is valid and all constraints are satisfied (for use in testing). * This is done in three steps. * - *

    • Check if nodeCoLocationCnt-constraints are satisfied. Some components may allow only a certain number of - * executors to exist on the same node {@link ConstraintSolverConfig#getMaxNodeCoLocationCnts()}. + *
    • Check if nodeCoLocationCnt-constraints are satisfied. Some components may allow only a + * certain number of + * executors to exist on the same node {@link + * ConstraintSolverConfig#getMaxNodeCoLocationCnts()}. *
    • * *
    • * Check if incompatibility-constraints are satisfied. Incompatible components - * {@link ConstraintSolverConfig#getIncompatibleComponentSets()} should not be put on the same worker. + * {@link ConstraintSolverConfig#getIncompatibleComponentSets()} should not be put on the same + * worker. *
    • * *
    • - * Check if CPU and Memory resources do not exceed availability on the node and total matches what is expected + * Check if CPU and Memory resources do not exceed availability on the node and total matches + * what is expected * when fully scheduled. *
    • * @@ -138,15 +157,18 @@ public static boolean validateSolution(Cluster cluster, TopologyDetails topo) { // First check NodeCoLocationCnt constraints Map execToComp = topo.getExecutorToComponent(); - Map> nodeCompMap = new HashMap<>(); // this is the critical count + Map> nodeCompMap = + new HashMap<>(); // this is the critical count Map workerToNodes = new HashMap<>(); RasNodes.getAllNodesFrom(cluster) .values() - .forEach(node -> node.getUsedSlots().forEach(workerSlot -> workerToNodes.put(workerSlot, node))); + .forEach(node -> node.getUsedSlots().forEach(workerSlot -> workerToNodes + .put(workerSlot, node))); List errors = new ArrayList<>(); - for (Map.Entry entry : cluster.getAssignmentById(topo.getId()).getExecutorToSlot().entrySet()) { + for (Map.Entry entry : cluster.getAssignmentById(topo.getId()) + .getExecutorToSlot().entrySet()) { ExecutorDetails exec = entry.getKey(); String comp = execToComp.get(exec); WorkerSlot worker = entry.getValue(); @@ -156,11 +178,15 @@ public static boolean validateSolution(Cluster cluster, TopologyDetails topo) { if (!constraintSolverConfig.getMaxNodeCoLocationCnts().containsKey(comp)) { continue; } - int allowedColocationMaxCnt = constraintSolverConfig.getMaxNodeCoLocationCnts().get(comp); - Map oneNodeCompMap = nodeCompMap.computeIfAbsent(nodeId, (k) -> new HashMap<>()); + int allowedColocationMaxCnt = constraintSolverConfig.getMaxNodeCoLocationCnts() + .get(comp); + Map oneNodeCompMap = nodeCompMap.computeIfAbsent(nodeId, + (k) -> new HashMap<>()); oneNodeCompMap.put(comp, oneNodeCompMap.getOrDefault(comp, 0) + 1); if (allowedColocationMaxCnt < oneNodeCompMap.get(comp)) { - String err = String.format("MaxNodeCoLocation: Component %s (exec=%s) on node %s, cnt %d > allowed %d", + String err = String + .format("MaxNodeCoLocation: Component %s (exec=%s) on node %s, cnt %d > " + + "allowed %d", comp, exec, nodeId, oneNodeCompMap.get(comp), allowedColocationMaxCnt); errors.add(err); } @@ -178,9 +204,12 @@ public static boolean validateSolution(Cluster cluster, TopologyDetails topo) { for (String comp1 : comps) { for (String comp2 : comps) { if (!comp1.equals(comp2) - && constraintSolverConfig.getIncompatibleComponentSets().containsKey(comp1) - && constraintSolverConfig.getIncompatibleComponentSets().get(comp1).contains(comp2)) { - String err = String.format("IncompatibleComponents: %s and %s on WorkerSlot: %s", + && constraintSolverConfig.getIncompatibleComponentSets() + .containsKey(comp1) + && constraintSolverConfig.getIncompatibleComponentSets().get(comp1) + .contains(comp2)) { + String err = String + .format("IncompatibleComponents: %s and %s on WorkerSlot: %s", comp1, comp2, entry.getKey()); errors.add(err); } @@ -203,13 +232,17 @@ public static boolean validateSolution(Cluster cluster, TopologyDetails topo) { RasNode node = nodes.get(worker.getNodeId()); if (node.getAvailableMemoryResources() < 0.0) { - String err = String.format("Resource Exhausted: Found node %s with negative available memory %,.2f", + String err = String + .format("Resource Exhausted: Found node %s with negative available memory " + + "%,.2f", node.getId(), node.getAvailableMemoryResources()); errors.add(err); continue; } if (node.getAvailableCpuResources() < 0.0) { - String err = String.format("Resource Exhausted: Found node %s with negative available CPU %,.2f", + String err = String + .format("Resource Exhausted: Found node %s with negative available CPU " + + "%,.2f", node.getId(), node.getAvailableCpuResources()); errors.add(err); continue; @@ -227,21 +260,29 @@ public static boolean validateSolution(Cluster cluster, TopologyDetails topo) { memoryUsed += topo.getTotalMemReqTask(exec); } if (node.getAvailableCpuResources() != (node.getTotalCpuResources() - cpuUsed)) { - String err = String.format("Incorrect CPU Resources: Node %s CPU available is %,.2f, expected %,.2f, " + String err = String + .format("Incorrect CPU Resources: Node %s CPU available is %,.2f, " + + "expected %,.2f, " + "Executors scheduled on node: %s", - node.getId(), node.getAvailableCpuResources(), (node.getTotalCpuResources() - cpuUsed), execs); + node.getId(), node.getAvailableCpuResources(), (node + .getTotalCpuResources() - cpuUsed), execs); errors.add(err); } - if (node.getAvailableMemoryResources() != (node.getTotalMemoryResources() - memoryUsed)) { - String err = String.format("Incorrect Memory Resources: Node %s Memory available is %,.2f, expected %,.2f, " + if (node.getAvailableMemoryResources() != (node + .getTotalMemoryResources() - memoryUsed)) { + String err = String + .format("Incorrect Memory Resources: Node %s Memory available is %,.2f, " + + "expected %,.2f, " + "Executors scheduled on node: %s", - node.getId(), node.getAvailableMemoryResources(), (node.getTotalMemoryResources() - memoryUsed), execs); + node.getId(), node.getAvailableMemoryResources(), (node + .getTotalMemoryResources() - memoryUsed), execs); errors.add(err); } } if (!errors.isEmpty()) { - LOG.error("Topology {} solution is invalid\n\t{}", topo.getName(), String.join("\n\t", errors)); + LOG.error("Topology {} solution is invalid\n\t{}", topo.getName(), String.join("\n\t", + errors)); } return errors.isEmpty(); } @@ -253,12 +294,14 @@ public static boolean validateSolution(Cluster cluster, TopologyDetails topo) { */ private boolean isSchedulingFeasible() { int nodeCnt = nodes.getNodes().size(); - for (Map.Entry entry : constraintSolverConfig.getMaxNodeCoLocationCnts().entrySet()) { + for (Map.Entry entry : constraintSolverConfig.getMaxNodeCoLocationCnts() + .entrySet()) { String comp = entry.getKey(); int maxCoLocationCnt = entry.getValue(); int numExecs = compToExecs.get(comp).size(); if (numExecs > nodeCnt * maxCoLocationCnt) { - LOG.error("Unsatisfiable constraint: Component: {} marked as spread has {} executors which is larger than " + LOG.error("Unsatisfiable constraint: Component: {} marked as spread has {} " + + "executors which is larger than " + "number of nodes * maxCoLocationCnt: {} * {} ", comp, numExecs, nodeCnt, maxCoLocationCnt); return false; } diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/IStrategy.java b/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/IStrategy.java index 9a9ca762c92..7332b173a47 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/IStrategy.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/IStrategy.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

      - * http://www.apache.org/licenses/LICENSE-2.0 - *

      - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

      http://www.apache.org/licenses/LICENSE-2.0 + * + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -18,12 +24,15 @@ import org.apache.storm.scheduler.resource.SchedulingResult; /** - * An interface to for implementing different scheduling strategies for the resource aware scheduling. - * Scheduler should call {@link #prepare(Map)} followed by {@link #schedule(Cluster, TopologyDetails)}. - *

      - * A fully functioning implementation is in the abstract class {@link BaseResourceAwareStrategy}. + * An interface to for implementing different scheduling strategies for the resource aware + * scheduling. + * Scheduler should call {@link #prepare(Map)} followed by {@link #schedule(Cluster, + * TopologyDetails)}. + * + *

      A fully functioning implementation is in the abstract class {@link BaseResourceAwareStrategy}. * Subclasses classes should extend {@link BaseResourceAwareStrategy#BaseResourceAwareStrategy()} - * in their constructors (as in {@link GenericResourceAwareStrategy}, {@link DefaultResourceAwareStrategy} + * in their constructors (as in {@link GenericResourceAwareStrategy}, {@link + * DefaultResourceAwareStrategy} * and {@link ConstraintSolverStrategy}). *

      */ @@ -37,16 +46,21 @@ public interface IStrategy { void prepare(Map config); /** - * This method is invoked to calculate a scheduling for topology td. Cluster will reject any changes that are - * not for the given topology. Any changes made to the cluster will be committed if the scheduling is successful. - *

      - * NOTE: scheduling occurs as a runnable in an interruptable thread. Scheduling should consider being interrupted if + * This method is invoked to calculate a scheduling for topology td. Cluster will reject any + * changes that are + * not for the given topology. Any changes made to the cluster will be committed if the + * scheduling is successful. + * + *

      NOTE: scheduling occurs as a runnable in an interruptable thread. Scheduling should + * consider + * being interrupted if * long running. *

      * * @param schedulingState the current state of the cluster * @param td the topology to schedule for - * @return returns a SchedulingResult object containing SchedulingStatus object to indicate whether scheduling is + * @return returns a SchedulingResult object containing SchedulingStatus object to indicate + * whether scheduling is * successful. */ SchedulingResult schedule(Cluster schedulingState, TopologyDetails td); diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/ObjectResourcesItem.java b/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/ObjectResourcesItem.java index 6b87dbdb206..84555a34d1c 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/ObjectResourcesItem.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/ObjectResourcesItem.java @@ -22,7 +22,7 @@ import org.apache.storm.scheduler.resource.strategies.scheduling.sorter.NodeSorter; /** - * class to keep track of resources on a rack or node. + * Class to keep track of resources on a rack or node. */ public class ObjectResourcesItem { public final String id; @@ -30,11 +30,12 @@ public class ObjectResourcesItem { public NormalizedResourceOffer totalResources; /** - * Amongst all {@link #availableResources}, this is the minimum ratio of resource to the total available in group. - * Note that nodes are grouped into hosts. Hosts into racks. And racks are grouped under the cluster. + * Amongst all {@link #availableResources}, this is the minimum ratio of resource to the total + * available in group. + * Note that nodes are grouped into hosts. Hosts into racks. And racks are grouped under the + * cluster. * - *

      - * An example of this calculation is in {@link NodeSorter} + *

      An example of this calculation is in {@link NodeSorter} * where value is calculated by {@link ObjectResourcesSummary#getAvailableResourcesOverall()} * using {@link NormalizedResourceOffer#calculateMinPercentageUsedBy(NormalizedResourceOffer)}. *

      @@ -42,13 +43,15 @@ public class ObjectResourcesItem { public double minResourcePercent = 0.0; /** - * Amongst all {@link #availableResources}, this is the average ratio of resource to the total available in group. - * Note that nodes are grouped into hosts, hosts into racks, and racks are grouped under the cluster. + * Amongst all {@link #availableResources}, this is the average ratio of resource to the total + * available in group. + * Note that nodes are grouped into hosts, hosts into racks, and racks are grouped under the + * cluster. * - *

      - * An example of this calculation is in {@link NodeSorter} + *

      An example of this calculation is in {@link NodeSorter} * where value is calculated by {@link ObjectResourcesSummary#getAvailableResourcesOverall()} - * using {@link NormalizedResourceOffer#calculateAveragePercentageUsedBy(NormalizedResourceOffer)}. + * using {@link + * NormalizedResourceOffer#calculateAveragePercentageUsedBy(NormalizedResourceOffer)}. *

      */ public double avgResourcePercent = 0.0; @@ -60,10 +63,12 @@ public ObjectResourcesItem(String id) { } public ObjectResourcesItem(ObjectResourcesItem other) { - this(other.id, other.availableResources, other.totalResources, other.minResourcePercent, other.avgResourcePercent); + this(other.id, other.availableResources, other.totalResources, other.minResourcePercent, + other.avgResourcePercent); } - public ObjectResourcesItem(String id, NormalizedResourceOffer availableResources, NormalizedResourceOffer totalResources, + public ObjectResourcesItem(String id, NormalizedResourceOffer availableResources, + NormalizedResourceOffer totalResources, double minResourcePercent, double avgResourcePercent) { this.id = id; this.availableResources = availableResources; diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/ObjectResourcesSummary.java b/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/ObjectResourcesSummary.java index 1e8782c36bd..faa7f9ed83d 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/ObjectResourcesSummary.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/ObjectResourcesSummary.java @@ -24,7 +24,7 @@ import org.apache.storm.scheduler.resource.normalization.NormalizedResourceOffer; /** - * a class to contain individual object resources as well as cumulative stats. + * A class to contain individual object resources as well as cumulative stats. */ public class ObjectResourcesSummary { private List objectResources = new LinkedList<>(); @@ -49,7 +49,8 @@ public ObjectResourcesSummary(ObjectResourcesSummary other) { this.objectResources = objectResourcesList; } - public ObjectResourcesSummary(List objectResources, NormalizedResourceOffer availableResourcesOverall, + public ObjectResourcesSummary(List objectResources, + NormalizedResourceOffer availableResourcesOverall, NormalizedResourceOffer totalResourcesOverall, String identifier) { this.objectResources = objectResources; this.availableResourcesOverall = availableResourcesOverall; diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/RoundRobinResourceAwareStrategy.java b/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/RoundRobinResourceAwareStrategy.java index 1bb5a4bf502..2f2f5767b75 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/RoundRobinResourceAwareStrategy.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/RoundRobinResourceAwareStrategy.java @@ -23,7 +23,6 @@ import java.util.Iterator; import java.util.List; import java.util.Map; - import org.apache.storm.Config; import org.apache.storm.scheduler.ExecutorDetails; import org.apache.storm.scheduler.WorkerSlot; @@ -34,7 +33,8 @@ import org.slf4j.LoggerFactory; public class RoundRobinResourceAwareStrategy extends BaseResourceAwareStrategy { - private static final Logger LOG = LoggerFactory.getLogger(RoundRobinResourceAwareStrategy.class); + private static final Logger LOG = LoggerFactory + .getLogger(RoundRobinResourceAwareStrategy.class); public RoundRobinResourceAwareStrategy() { super(false, NodeSortType.COMMON); @@ -49,14 +49,18 @@ private int getMaxNumberOfNodesRequested() { if (conf.get(Config.TOPOLOGY_ISOLATED_MACHINES) == null) { return Integer.MAX_VALUE; } else { - return ((Number) topologyDetails.getConf().get(Config.TOPOLOGY_ISOLATED_MACHINES)).intValue(); + return ((Number) topologyDetails.getConf().get(Config.TOPOLOGY_ISOLATED_MACHINES)) + .intValue(); } } /** - * If the number of machines is limited, then truncate the node list to this maximum number of nodes - * that have no other topologies running on it. If the current topology is running on it, then it - * is subject to selection in the list. If other topologies are running on it, then it is not selected. + * If the number of machines is limited, then truncate the node list to this maximum number of + * nodes + * that have no other topologies running on it. If the current topology is running on it, then + * it + * is subject to selection in the list. If other topologies are running on it, then it is not + * selected. * * @param sortedNodesIterable Iterable of nodes * @return an ArrayList of nodes @@ -68,7 +72,8 @@ private ArrayList getTruncatedNodeList(Iterable sortedNodesItera if (ret.size() < maxNodes) { RasNode rasNode = nodes.getNodeById(node); Collection runningTopos = rasNode.getRunningTopologies(); - if (runningTopos.isEmpty() || runningTopos.size() == 1 && runningTopos.contains(topologyDetails.getId())) { + if (runningTopos.isEmpty() || runningTopos.size() == 1 && runningTopos + .contains(topologyDetails.getId())) { ret.add(node); } } @@ -80,18 +85,22 @@ private ArrayList getTruncatedNodeList(Iterable sortedNodesItera * For each component try to schedule executors in sequence on the nodes. * * @param orderedExecutors Executors sorted in the preferred order cannot be null - * @param sortedNodesIterable Node iterable which cannot be null, relies on behavior when {@link #sortNodesForEachExecutor} is false - * @return SchedulingResult with success attribute set to true or false indicting whether ALL executors were assigned. @{#} + * @param sortedNodesIterable Node iterable which cannot be null, relies on behavior when {@link + * #sortNodesForEachExecutor} is false + * @return SchedulingResult with success attribute set to true or false indicting whether ALL + * executors were assigned. @{#} */ @Override - protected SchedulingResult scheduleExecutorsOnNodes(List orderedExecutors, Iterable sortedNodesIterable) { + protected SchedulingResult scheduleExecutorsOnNodes(List orderedExecutors, + Iterable sortedNodesIterable) { long startTimeMilli = Time.currentTimeMillis(); int maxExecCnt = searcherState.getExecSize(); int nodeSortCnt = 1; Iterator sortedNodesIter = null; ArrayList sortedNodes = getTruncatedNodeList(sortedNodesIterable); - LOG.debug("scheduleExecutorsOnNodes: will assign {} executors for topo {}", maxExecCnt, topoName); + LOG.debug("scheduleExecutorsOnNodes: will assign {} executors for topo {}", maxExecCnt, + topoName); searcherState.setSortedExecs(orderedExecutors); @@ -100,7 +109,8 @@ protected SchedulingResult scheduleExecutorsOnNodes(List ordere LOG.debug("scheduleExecutorsOnNodes: loopCnt={}, execIndex={}, topo={}, nodeSortCnt={}", loopCnt, searcherState.getExecIndex(), topoName, nodeSortCnt); if (searcherState.areSearchLimitsExceeded()) { - LOG.warn("Limits exceeded, loopCnt={}, topo={}, nodeSortCnt={}", loopCnt, topoName, nodeSortCnt); + LOG.warn("Limits exceeded, loopCnt={}, topo={}, nodeSortCnt={}", loopCnt, topoName, + nodeSortCnt); return searcherState.createSchedulingResult(false, this.getClass().getSimpleName()); } @@ -116,12 +126,14 @@ protected SchedulingResult scheduleExecutorsOnNodes(List ordere // So we skip to the next. if (searcherState.getBoundAckers().contains(exec)) { if (searcherState.areAllExecsScheduled()) { - //Everything is scheduled correctly, so no need to search any more. - LOG.info("scheduleExecutorsOnNodes: Done at loopCnt={} in {}ms, state.elapsedtime={}, topo={}, nodeSortCnt={}", + // Everything is scheduled correctly, so no need to search any more. + LOG.info("scheduleExecutorsOnNodes: Done at loopCnt={} in {}ms, " + + "state.elapsedtime={}, topo={}, nodeSortCnt={}", loopCnt, Time.currentTimeMillis() - startTimeMilli, Time.currentTimeMillis() - searcherState.getStartTimeMillis(), topoName, nodeSortCnt); - return searcherState.createSchedulingResult(true, this.getClass().getSimpleName()); + return searcherState.createSchedulingResult(true, this.getClass() + .getSimpleName()); } searcherState = searcherState.nextExecutor(); continue OUTERMOST_LOOP; @@ -129,7 +141,8 @@ protected SchedulingResult scheduleExecutorsOnNodes(List ordere String comp = execToComp.get(exec); // start at the beginning of node list when component changes or when at end of nodes - if (sortedNodesIter == null || searcherState.isExecCompDifferentFromPrior() || !sortedNodesIter.hasNext()) { + if (sortedNodesIter == null || searcherState.isExecCompDifferentFromPrior() + || !sortedNodesIter.hasNext()) { sortedNodesIter = sortedNodes.iterator(); nodeSortCnt++; } @@ -143,44 +156,55 @@ protected SchedulingResult scheduleExecutorsOnNodes(List ordere for (WorkerSlot workerSlot : node.getSlotsAvailableToScheduleOn()) { if (!isExecAssignmentToWorkerValid(exec, workerSlot)) { // exec can't fit in this workerSlot, try next workerSlot - LOG.trace("Failed to assign exec={}, comp={}, topo={} to worker={} on node=({}, availCpu={}, availMem={}).", + LOG.trace("Failed to assign exec={}, comp={}, topo={} to worker={} on " + + "node=({}, availCpu={}, availMem={}).", exec, comp, topoName, workerSlot, - node.getId(), node.getAvailableCpuResources(), node.getAvailableMemoryResources()); + node.getId(), node.getAvailableCpuResources(), node + .getAvailableMemoryResources()); continue; } searcherState.incStatesSearched(); searcherState.assignCurrentExecutor(execToComp, node, workerSlot); - int numBoundAckerAssigned = assignBoundAckersForNewWorkerSlot(exec, node, workerSlot); + int numBoundAckerAssigned = assignBoundAckersForNewWorkerSlot(exec, node, + workerSlot); if (numBoundAckerAssigned > 0) { - // This exec with some of its bounded ackers have all been successfully assigned + // This exec with some of its bounded ackers have all been successfully + // assigned searcherState.getExecsWithBoundAckers().add(exec); } if (searcherState.areAllExecsScheduled()) { - //Everything is scheduled correctly, so no need to search any more. - LOG.info("scheduleExecutorsOnNodes: Done at loopCnt={} in {}ms, state.elapsedtime={}, topo={}, nodeSortCnt={}", + // Everything is scheduled correctly, so no need to search any more. + LOG.info("scheduleExecutorsOnNodes: Done at loopCnt={} in {}ms, " + + "state.elapsedtime={}, topo={}, nodeSortCnt={}", loopCnt, Time.currentTimeMillis() - startTimeMilli, Time.currentTimeMillis() - searcherState.getStartTimeMillis(), topoName, nodeSortCnt); - return searcherState.createSchedulingResult(true, this.getClass().getSimpleName()); + return searcherState.createSchedulingResult(true, this.getClass() + .getSimpleName()); } searcherState = searcherState.nextExecutor(); - LOG.debug("scheduleExecutorsOnNodes: Assigned execId={}, comp={} to node={}/cpu={}/mem={}, " + LOG.debug("scheduleExecutorsOnNodes: Assigned execId={}, comp={} to " + + "node={}/cpu={}/mem={}, " + "worker-port={} at loopCnt={}, topo={}, nodeSortCnt={}", - execIndex, comp, nodeId, node.getAvailableCpuResources(), node.getAvailableMemoryResources(), + execIndex, comp, nodeId, node.getAvailableCpuResources(), node + .getAvailableMemoryResources(), workerSlot.getPort(), loopCnt, topoName, nodeSortCnt); continue OUTERMOST_LOOP; } } // if here, then the executor was not assigned, scheduling failed - LOG.debug("scheduleExecutorsOnNodes: Failed to schedule execId={}, comp={} at loopCnt={}, topo={}, nodeSortCnt={}", + LOG.debug("scheduleExecutorsOnNodes: Failed to schedule execId={}, comp={} at " + + "loopCnt={}, topo={}, nodeSortCnt={}", execIndex, comp, loopCnt, topoName, nodeSortCnt); break; } boolean success = searcherState.areAllExecsScheduled(); - LOG.info("scheduleExecutorsOnNodes: Scheduled={} in {} milliseconds, state.elapsedtime={}, topo={}, nodeSortCnt={}", - success, Time.currentTimeMillis() - startTimeMilli, Time.currentTimeMillis() - searcherState.getStartTimeMillis(), + LOG.info("scheduleExecutorsOnNodes: Scheduled={} in {} milliseconds, " + + "state.elapsedtime={}, topo={}, nodeSortCnt={}", + success, Time.currentTimeMillis() - startTimeMilli, Time + .currentTimeMillis() - searcherState.getStartTimeMillis(), topoName, nodeSortCnt); return searcherState.createSchedulingResult(success, this.getClass().getSimpleName()); } diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/SchedulingSearcherState.java b/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/SchedulingSearcherState.java index 6d65a3e437a..0e28bffa2c7 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/SchedulingSearcherState.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/SchedulingSearcherState.java @@ -27,7 +27,6 @@ import java.util.Set; import java.util.TreeSet; import java.util.stream.Collectors; - import org.apache.storm.Config; import org.apache.storm.daemon.Acker; import org.apache.storm.scheduler.ExecutorDetails; @@ -49,15 +48,16 @@ public class SchedulingSearcherState { // A map of the worker to the components in the worker to be able to enforce constraints. private final Map> workerCompAssignmentCnts; private final boolean[] okToRemoveFromWorker; - // for the currently tested assignment a Map of the node to the components on it to be able to enforce constraints. + // for the currently tested assignment a Map of the node to the components on it to be able to + // enforce constraints. private final Map> nodeCompAssignmentCnts; private final boolean[] okToRemoveFromNode; // Static State // The list of all executors (preferably sorted to make assignments simpler). private List execs; - //The maximum number of state to search before stopping. + // The maximum number of state to search before stopping. private final int maxStatesSearched; - //The topology we are scheduling + // The topology we are scheduling private final TopologyDetails td; private final String topoName; // Metrics @@ -106,8 +106,10 @@ public SchedulingSearcherState(Map> workerCompA } this.execToComp = execToComp; - this.oneExecutorPerWorker = ObjectReader.getBoolean(td.getConf().get(Config.TOPOLOGY_RAS_ONE_EXECUTOR_PER_WORKER), false); - this.oneComponentPerWorker = ObjectReader.getBoolean(td.getConf().get(Config.TOPOLOGY_RAS_ONE_COMPONENT_PER_WORKER), false); + this.oneExecutorPerWorker = ObjectReader.getBoolean(td.getConf() + .get(Config.TOPOLOGY_RAS_ONE_EXECUTOR_PER_WORKER), false); + this.oneComponentPerWorker = ObjectReader.getBoolean(td.getConf() + .get(Config.TOPOLOGY_RAS_ONE_COMPONENT_PER_WORKER), false); this.unassignedAckers = unassignedAckers; this.boundAckers = new HashSet<>(); @@ -128,7 +130,9 @@ public void setSortedExecs(List sortedExecs) { if (execs == null || new HashSet<>(execs).equals(new HashSet<>(sortedExecs))) { this.execs = sortedExecs; } else { - String err = String.format("executors in sorted list (cnt=%d) are different from initial assignment (cnt=%d), topo=%s)", + String err = String + .format("executors in sorted list (cnt=%d) are different from initial " + + "assignment (cnt=%d), topo=%s)", sortedExecs.size(), execs.size(), topoName); throw new IllegalArgumentException(err); } @@ -185,7 +189,9 @@ public boolean areSearchLimitsExceeded() { public SchedulingSearcherState nextExecutor() { execIndex++; if (execIndex >= execs.size()) { - String err = String.format("Internal Error: topology %s: execIndex exceeded limit %d >= %d", topoName, execIndex, execs.size()); + String err = String + .format("Internal Error: topology %s: execIndex exceeded limit %d >= %d", + topoName, execIndex, execs.size()); throw new IllegalStateException(err); } return this; @@ -197,6 +203,7 @@ public boolean areAllExecsScheduled() { /** * Get the current unassigned executor. + * * @return the first unassigned executor in execs list. */ public ExecutorDetails currentExec() { @@ -204,31 +211,37 @@ public ExecutorDetails currentExec() { } /** - * Attempt to assign current executor (execIndex points to) to worker and node. - * Assignment validity check is done before calling this method. - * - * @param execToComp Mapping from executor to component name. - * @param node RasNode on which to schedule. - * @param workerSlot WorkerSlot on which to schedule. - */ - public void assignCurrentExecutor(Map execToComp, RasNode node, WorkerSlot workerSlot) { + * Attempt to assign current executor (execIndex points to) to worker and node. + * Assignment validity check is done before calling this method. + * + * @param execToComp Mapping from executor to component name. + * @param node RasNode on which to schedule. + * @param workerSlot WorkerSlot on which to schedule. + */ + public void assignCurrentExecutor(Map execToComp, RasNode node, + WorkerSlot workerSlot) { ExecutorDetails exec = currentExec(); String comp = execToComp.get(exec); LOG.trace("Topology {} Trying assignment of {} {} to {}", topoName, exec, comp, workerSlot); - // It is possible that this component is already scheduled on this node or worker. If so when we backtrack we cannot remove it - Map compToAssignmentCount = workerCompAssignmentCnts.computeIfAbsent(workerSlot, (k) -> new HashMap<>()); - compToAssignmentCount.put(comp, compToAssignmentCount.getOrDefault(comp, 0) + 1); // increment worker assignment count + // It is possible that this component is already scheduled on this node or worker. If so + // when we backtrack we cannot remove it + Map compToAssignmentCount = workerCompAssignmentCnts + .computeIfAbsent(workerSlot, (k) -> new HashMap<>()); + compToAssignmentCount.put(comp, compToAssignmentCount.getOrDefault(comp, 0) + + 1); // increment worker assignment count okToRemoveFromWorker[execIndex] = true; - Map nodeToAssignmentCount = nodeCompAssignmentCnts.computeIfAbsent(node, (k) -> new HashMap<>()); - nodeToAssignmentCount.put(comp, nodeToAssignmentCount.getOrDefault(comp, 0) + 1); // increment node assignment count + Map nodeToAssignmentCount = nodeCompAssignmentCnts.computeIfAbsent(node, + (k) -> new HashMap<>()); + nodeToAssignmentCount.put(comp, nodeToAssignmentCount.getOrDefault(comp, 0) + + 1); // increment node assignment count okToRemoveFromNode[execIndex] = true; node.assignSingleExecutor(workerSlot, exec, td); } /** - *

      - * Determine how many bound ackers to put in before assigning the executor to current workerSlot. + *

      Determine how many bound ackers to put in before assigning the executor to current + * workerSlot. * Note that the worker slot must be a new worker to build on scheduling. *

      * Return 0 if: @@ -237,11 +250,13 @@ public void assignCurrentExecutor(Map execToComp, RasNo * 2. The exec to assign is an acker. * 3. The workerSlot is not a new worker. * 4. No more unassigned ackers to use. - *

      - * A special scenario: - * If max heap limit is smaller than (this exec mem + {@link Config#TOPOLOGY_RAS_ACKER_EXECUTORS_PER_WORKER} ackers' mem), + * + *

      A special scenario: + * If max heap limit is smaller than (this exec mem + {@link + * Config#TOPOLOGY_RAS_ACKER_EXECUTORS_PER_WORKER} ackers' mem), * scheduler will bind fewer ackers based on max heap limit. *

      + * * @param exec the exec to assign into the workerSlot. * @param workerSlot the new worker slot to build. * @return the num of bound ackers to assign. @@ -265,7 +280,8 @@ public int getNumOfAckersToBind(ExecutorDetails exec, WorkerSlot workerSlot) { int maxBoundAckers = (int) Math.floor(workerHeapSpace / ackerOnHeapReq); if (maxBoundAckers < ackersPerWorker) { - LOG.debug("For exec {}, can only bind up to {} ackers due to {} limit. Acker Per worker setting: {}.", + LOG.debug("For exec {}, can only bind up to {} ackers due to {} limit. Acker Per " + + "worker setting: {}.", exec, maxBoundAckers, Config.TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB, ackersPerWorker); } int ret = Math.min(Math.min(maxBoundAckers, unassignedAckers.size()), ackersPerWorker); @@ -278,10 +294,13 @@ public int getNumOfAckersToBind(ExecutorDetails exec, WorkerSlot workerSlot) { * Backtrack to prior executor that was directly assigned. This excludes bound-ackers. * * @param execToComp map from executor to component. - * @param nodesForExec array of nodes for all execIndex - has null values for bound-acker indices. - * @param workerSlotForExec array of workerSlots for all execIndex - has null values for bound-acker indices. + * @param nodesForExec array of nodes for all execIndex - has null values for bound-acker + * indices. + * @param workerSlotForExec array of workerSlots for all execIndex - has null values for + * bound-acker indices. */ - public void backtrack(Map execToComp, RasNode[] nodesForExec, WorkerSlot[] workerSlotForExec) { + public void backtrack(Map execToComp, RasNode[] nodesForExec, + WorkerSlot[] workerSlotForExec) { execIndex--; /* After decrementing execIndex, it is expected to point to the target executor to backtrack to. @@ -306,7 +325,8 @@ public void backtrack(Map execToComp, RasNode[] nodesFo execIndex--; } if (execIndex < 0) { - throw new IllegalStateException("Internal Error: Topology " + topoName + " exec index became negative"); + throw new IllegalStateException("Internal Error: Topology " + topoName + + " exec index became negative"); } numBacktrack++; ExecutorDetails exec = currentExec(); @@ -316,7 +336,8 @@ public void backtrack(Map execToComp, RasNode[] nodesFo LOG.trace("Topology {} Backtracking {} {} from {}", topoName, exec, comp, workerSlot); if (okToRemoveFromWorker[execIndex]) { Map compToAssignmentCount = workerCompAssignmentCnts.get(workerSlot); - compToAssignmentCount.put(comp, compToAssignmentCount.getOrDefault(comp, 0) - 1); // decrement worker assignment count + compToAssignmentCount.put(comp, compToAssignmentCount.getOrDefault(comp, + 0) - 1); // decrement worker assignment count if (compToAssignmentCount.get(comp) == 0) { compToAssignmentCount.remove(comp); } @@ -324,7 +345,8 @@ public void backtrack(Map execToComp, RasNode[] nodesFo } if (okToRemoveFromNode[execIndex]) { Map nodeToAssignmentCount = nodeCompAssignmentCnts.get(node); - nodeToAssignmentCount.put(comp, nodeToAssignmentCount.getOrDefault(comp, 0) - 1); // decrement node assignment count + nodeToAssignmentCount.put(comp, nodeToAssignmentCount.getOrDefault(comp, + 0) - 1); // decrement node assignment count if (nodeToAssignmentCount.get(comp) == 0) { nodeToAssignmentCount.remove(comp); } @@ -342,8 +364,8 @@ public void backtrack(Map execToComp, RasNode[] nodesFo } /** - *

      - * Remove the head of unassigned ackers and attempt to assign it to a workerSlot as a bound acker. + *

      Remove the head of unassigned ackers and attempt to assign it to a workerSlot as a bound + * acker. *

      * * @param node RasNode on which to schedule. @@ -351,7 +373,9 @@ public void backtrack(Map execToComp, RasNode[] nodesFo */ public void assignSingleBoundAcker(RasNode node, WorkerSlot workerSlot) { if (unassignedAckers.isEmpty()) { - String msg = String.format("No more available ackers to assign for the new worker: %s of topology: %s", + String msg = String + .format("No more available ackers to assign for the new worker: %s of " + + "topology: %s", workerSlot, topoName); throw new IllegalStateException(msg); } @@ -364,10 +388,14 @@ public void assignSingleBoundAcker(RasNode node, WorkerSlot workerSlot) { boundAckers.add(acker); // bound ackers should not violate constraint solver String ackerCompId = Acker.ACKER_COMPONENT_ID; - Map compToAssignmentCount = workerCompAssignmentCnts.computeIfAbsent(workerSlot, (k) -> new HashMap<>()); - compToAssignmentCount.put(ackerCompId, compToAssignmentCount.getOrDefault(ackerCompId, 0) + 1); // increment worker assignment count - Map nodeToAssignmentCount = nodeCompAssignmentCnts.computeIfAbsent(node, (k) -> new HashMap<>()); - nodeToAssignmentCount.put(ackerCompId, nodeToAssignmentCount.getOrDefault(ackerCompId, 0) + 1); // increment node assignment count + Map compToAssignmentCount = workerCompAssignmentCnts + .computeIfAbsent(workerSlot, (k) -> new HashMap<>()); + compToAssignmentCount.put(ackerCompId, compToAssignmentCount.getOrDefault(ackerCompId, 0) + + 1); // increment worker assignment count + Map nodeToAssignmentCount = nodeCompAssignmentCnts.computeIfAbsent(node, + (k) -> new HashMap<>()); + nodeToAssignmentCount.put(ackerCompId, nodeToAssignmentCount.getOrDefault(ackerCompId, 0) + + 1); // increment node assignment count } /** @@ -385,16 +413,19 @@ public void freeWorkerSlotWithBoundAckers(RasNode node, WorkerSlot workerSlot) { ExecutorDetails acker = ackers.get(i); boundAckers.remove(acker); unassignedAckers.addFirst(acker); - Map compToAssignmentCount = workerCompAssignmentCnts.get(workerSlot); + Map compToAssignmentCount = workerCompAssignmentCnts + .get(workerSlot); compToAssignmentCount.put(ackerCompId, - compToAssignmentCount.getOrDefault(ackerCompId, 0) - 1); // decrement worker assignment count + compToAssignmentCount.getOrDefault(ackerCompId, + 0) - 1); // decrement worker assignment count if (compToAssignmentCount.get(ackerCompId) == 0) { compToAssignmentCount.remove(ackerCompId); } Map nodeToAssignmentCount = nodeCompAssignmentCnts.get(node); nodeToAssignmentCount.put(ackerCompId, - nodeToAssignmentCount.getOrDefault(ackerCompId, 0) - 1); // decrement node assignment count + nodeToAssignmentCount.getOrDefault(ackerCompId, + 0) - 1); // decrement node assignment count if (nodeToAssignmentCount.get(ackerCompId) == 0) { nodeToAssignmentCount.remove(ackerCompId); } @@ -427,11 +458,14 @@ public void logNodeCompAssignments() { String oneMapJoined = oneMap.entrySet() .stream().map(e -> String.format("%s: %s", e.getKey(), e.getValue())) .collect(Collectors.joining(",")); - sb.append(String.format("\n\t(%d) Node %s: %s", cntFilledNodes, node.getId(), oneMapJoined)); + sb.append(String.format("\n\t(%d) Node %s: %s", cntFilledNodes, node.getId(), + oneMapJoined)); } - LOG.info("Topology {} NodeCompAssignments available for {} of {} nodes {}", topoName, cntFilledNodes, cntAllNodes, sb); + LOG.info("Topology {} NodeCompAssignments available for {} of {} nodes {}", topoName, + cntFilledNodes, cntAllNodes, sb); LOG.info("Topology {} Executors assignments attempted (cnt={}) are: \n\t{}", - topoName, execs.size(), execs.stream().map(ExecutorDetails::toString).collect(Collectors.joining(",")) + topoName, execs.size(), execs.stream().map(ExecutorDetails::toString) + .collect(Collectors.joining(",")) ); } @@ -453,15 +487,20 @@ public int getComponentCntOnNode(RasNode rasNode, String comp) { return map.getOrDefault(comp, 0); } - public SchedulingResult createSchedulingResult(boolean success, String schedulerClassSimpleName) { + public SchedulingResult createSchedulingResult(boolean success, + String schedulerClassSimpleName) { String msg; if (success) { - msg = String.format("Fully Scheduled by %s (%d states traversed in %d ms, backtracked %d times)", + msg = String + .format("Fully Scheduled by %s (%d states traversed in %d ms, backtracked %d " + + "times)", schedulerClassSimpleName, this.getStatesSearched(), Time.currentTimeMillis() - this.getStartTimeMillis(), this.getNumBacktrack()); return SchedulingResult.success(msg); } else { - msg = String.format("Cannot schedule by %s (%d states traversed in %d ms, backtracked %d times, %d of %d executors scheduled)", + msg = String + .format("Cannot schedule by %s (%d states traversed in %d ms, backtracked %d " + + "times, %d of %d executors scheduled)", schedulerClassSimpleName, this.getStatesSearched(), Time.currentTimeMillis() - this.getStartTimeMillis(), this.getNumBacktrack(), this.getExecIndex(), this.getExecSize()); @@ -474,7 +513,8 @@ public SchedulingResult createSchedulingResult(boolean success, String scheduler * Check if the current executor has a different component from the previous one. * This flag can be used as a quick way to check if the nodes should be sorted. * - * @return true if first executor or if the component is same as previous executor. False other wise. + * @return true if first executor or if the component is same as previous executor. False other + * wise. */ public boolean isExecCompDifferentFromPrior() { if (execIndex == 0) { diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/sorter/ExecSorterByConnectionCount.java b/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/sorter/ExecSorterByConnectionCount.java index 699af94a574..31ae501a3b0 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/sorter/ExecSorterByConnectionCount.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/sorter/ExecSorterByConnectionCount.java @@ -26,7 +26,6 @@ import java.util.Queue; import java.util.Set; import java.util.TreeSet; - import org.apache.storm.scheduler.Component; import org.apache.storm.scheduler.ExecutorDetails; import org.apache.storm.scheduler.TopologyDetails; @@ -41,19 +40,26 @@ public ExecSorterByConnectionCount(TopologyDetails topologyDetails) { } /** - * Order executors based on how many in and out connections it will potentially need to make, in descending order. First order - * components by the number of in and out connections it will have. Then iterate through the sorted list of components. For each - * component sort the neighbors of that component by how many connections it will have to make with that component. - * Add an executor from this component and then from each neighboring component in sorted order. Do this until there is - * nothing left to schedule. Then add back executors not accounted for - which are system executors. + * Order executors based on how many in and out connections it will potentially need to make, in + * descending order. First order + * components by the number of in and out connections it will have. Then iterate through the + * sorted list of components. For each + * component sort the neighbors of that component by how many connections it will have to make + * with that component. + * Add an executor from this component and then from each neighboring component in sorted order. + * Do this until there is + * nothing left to schedule. Then add back executors not accounted for - which are system + * executors. * * @param unassignedExecutors an unmodifiable set of executors that need to be scheduled. * @return a list of executors in sorted order for scheduling. */ @Override public List sortExecutors(Set unassignedExecutors) { - Map componentMap = topologyDetails.getUserTopolgyComponents(); // excludes system components - LinkedHashSet orderedExecutorSet = new LinkedHashSet<>(); // in insert order + Map componentMap = topologyDetails + .getUserTopolgyComponents(); // excludes system components + LinkedHashSet orderedExecutorSet = + new LinkedHashSet<>(); // in insert order Map> compToExecsToSchedule = new HashMap<>(); for (Component component : componentMap.values()) { @@ -74,7 +80,8 @@ public List sortExecutors(Set unassignedExecut neighbors.put(compId, componentMap.get(compId)); } Set sortedNeighbors = sortNeighbors(currComp, neighbors); - Queue currCompExecsToSched = compToExecsToSchedule.get(currComp.getId()); + Queue currCompExecsToSched = compToExecsToSchedule.get(currComp + .getId()); boolean flag; do { @@ -85,7 +92,8 @@ public List sortExecutors(Set unassignedExecut } for (Component neighborComp : sortedNeighbors) { - Queue neighborCompExesToSched = compToExecsToSchedule.get(neighborComp.getId()); + Queue neighborCompExesToSched = compToExecsToSchedule + .get(neighborComp.getId()); if (!neighborCompExesToSched.isEmpty()) { orderedExecutorSet.add(neighborCompExesToSched.poll()); flag = true; @@ -100,7 +108,8 @@ public List sortExecutors(Set unassignedExecut } /** - * sort components by the number of in and out connections that need to be made, in descending order. + * Sort components by the number of in and out connections that need to be made, in descending + * order. * * @param componentMap The components that need to be sorted * @return a sorted set of components @@ -134,7 +143,8 @@ private Set sortComponents(final Map componentMap) } /** - * Sort a component's neighbors by the number of connections it needs to make with this component. + * Sort a component's neighbors by the number of connections it needs to make with this + * component. * * @param thisComp the component that we need to sort its neighbors * @param componentMap all the components to sort diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/sorter/ExecSorterByConstraintSeverity.java b/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/sorter/ExecSorterByConstraintSeverity.java index 918e865cd58..8099e27c257 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/sorter/ExecSorterByConstraintSeverity.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/sorter/ExecSorterByConstraintSeverity.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

      - * http://www.apache.org/licenses/LICENSE-2.0 - *

      - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

      http://www.apache.org/licenses/LICENSE-2.0 + * + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -22,7 +28,6 @@ import java.util.Set; import java.util.TreeMap; import java.util.stream.Collectors; - import org.apache.storm.scheduler.Cluster; import org.apache.storm.scheduler.ExecutorDetails; import org.apache.storm.scheduler.TopologyDetails; @@ -36,12 +41,13 @@ public ExecSorterByConstraintSeverity(Cluster cluster, TopologyDetails topologyD this.constraintSolverConfig = new ConstraintSolverConfig(topologyDetails); this.compToExecs = new HashMap<>(); topologyDetails.getExecutorToComponent() - .forEach((exec, comp) -> compToExecs.computeIfAbsent(comp, (k) -> new HashSet<>()).add(exec)); + .forEach((exec, comp) -> compToExecs.computeIfAbsent(comp, (k) -> new HashSet<>()) + .add(exec)); } @Override public List sortExecutors(Set unassignedExecutors) { - //get unassigned executors sorted based on number of constraints + // get unassigned executors sorted based on number of constraints List sortedExecs = getSortedExecs() .stream() .filter(unassignedExecutors::contains) @@ -50,7 +56,8 @@ public List sortExecutors(Set unassignedExecut } /** - * Sort executors such that components with more constraints are first. A component is more constrained if it + * Sort executors such that components with more constraints are first. A component is more + * constrained if it * has a higher number of incompatible components and/or it allows lesser instances on a node. * * @return a list of executors sorted constrained components first. @@ -58,21 +65,24 @@ public List sortExecutors(Set unassignedExecut private ArrayList getSortedExecs() { ArrayList retList = new ArrayList<>(); - //find number of constraints per component - //Key->Comp Value-># of constraints + // find number of constraints per component + // Key->Comp Value-># of constraints Map compConstraintCountMap = new HashMap<>(); - constraintSolverConfig.getIncompatibleComponentSets().forEach((comp, incompatibleComponents) -> { + constraintSolverConfig.getIncompatibleComponentSets().forEach((comp, + incompatibleComponents) -> { double constraintCnt = incompatibleComponents.size(); // check if component is declared for spreading if (constraintSolverConfig.getMaxNodeCoLocationCnts().containsKey(comp)) { // lower (1 and above only) value is most constrained should have higher count - constraintCnt += (compToExecs.size() / constraintSolverConfig.getMaxNodeCoLocationCnts().get(comp)); + constraintCnt += (compToExecs.size() / constraintSolverConfig + .getMaxNodeCoLocationCnts().get(comp)); } compConstraintCountMap.put(comp, constraintCnt); // higher count sorts to the front }); - //Sort comps by number of constraints - NavigableMap sortedCompConstraintCountMap = sortByValues(compConstraintCountMap); - //sort executors based on component constraints + // Sort comps by number of constraints + NavigableMap sortedCompConstraintCountMap = + sortByValues(compConstraintCountMap); + // sort executors based on component constraints for (String comp : sortedCompConstraintCountMap.keySet()) { retList.addAll(compToExecs.get(comp)); } diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/sorter/ExecSorterByProximity.java b/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/sorter/ExecSorterByProximity.java index 788eb1d997c..f0d45779a70 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/sorter/ExecSorterByProximity.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/sorter/ExecSorterByProximity.java @@ -28,7 +28,6 @@ import java.util.Set; import java.util.TreeSet; import java.util.stream.Collectors; - import org.apache.storm.generated.GlobalStreamId; import org.apache.storm.generated.Grouping; import org.apache.storm.scheduler.Component; @@ -56,8 +55,10 @@ public ExecSorterByProximity(TopologyDetails topologyDetails) { */ @Override public List sortExecutors(Set unassignedExecutors) { - Map componentMap = topologyDetails.getUserTopolgyComponents(); // excludes system components - LinkedHashSet orderedExecutorSet = new LinkedHashSet<>(); // in insert order + Map componentMap = topologyDetails + .getUserTopolgyComponents(); // excludes system components + LinkedHashSet orderedExecutorSet = + new LinkedHashSet<>(); // in insert order Map> compToExecsToSchedule = new HashMap<>(); for (Component component : componentMap.values()) { @@ -74,7 +75,8 @@ public List sortExecutors(Set unassignedExecut for (Component currComp : sortedComponents) { int numExecs = compToExecsToSchedule.get(currComp.getId()).size(); for (int i = 0; i < numExecs; i++) { - orderedExecutorSet.addAll(takeExecutors(currComp, componentMap, compToExecsToSchedule)); + orderedExecutorSet.addAll(takeExecutors(currComp, componentMap, + compToExecsToSchedule)); } } @@ -85,6 +87,7 @@ public List sortExecutors(Set unassignedExecut /** * Sort components topologically. + * * @param componentMap The map of component Id to Component Object. * @return The sorted components */ @@ -97,7 +100,7 @@ private List topologicalSortComponents(final Map c for (int i = 0; i < componentIds.size(); i++) { compIdToIndex.put(componentIds.get(i), i); } - //initialize the in-degree array + // initialize the in-degree array for (int i = 0; i < inDegree.length; i++) { String compId = componentIds.get(i); Component comp = componentMap.get(compId); @@ -105,7 +108,7 @@ private List topologicalSortComponents(final Map c inDegree[compIdToIndex.get(childId)] += 1; } } - //sorting components topologically + // sorting components topologically for (int t = 0; t < inDegree.length; t++) { for (int i = 0; i < inDegree.length; i++) { if (inDegree[i] == 0 && !visited[i]) { @@ -120,13 +123,15 @@ private List topologicalSortComponents(final Map c } } } - // add back components that could not be visited and issue warning about loop in component data flow + // add back components that could not be visited and issue warning about loop in component + // data flow if (sortedComponentsSet.size() != componentMap.size()) { String unvisitedComponentIds = componentMap.entrySet().stream() .filter(x -> !sortedComponentsSet.contains(x.getValue())) .map(x -> x.getKey()) .collect(Collectors.joining(",")); - LOG.warn("topologicalSortComponents for topology {} detected possible loop(s) involving components {}, " + LOG.warn("topologicalSortComponents for topology {} detected possible loop(s) " + + "involving components {}, " + "appending them to the end of the sorted component list", topologyDetails.getId(), unvisitedComponentIds); sortedComponentsSet.addAll(componentMap.values()); @@ -135,14 +140,17 @@ private List topologicalSortComponents(final Map c } /** - * Take unscheduled executors from current and all its downstream components in a particular order. + * Take unscheduled executors from current and all its downstream components in a particular + * order. * First, take one executor from the current component; * then for every child (direct downstream component) of this component, * if it's shuffle grouping from the current component to this child, * the number of executors to take from this child is the max of - * 1 and (the number of unscheduled executors this child has / the number of unscheduled executors the current component has); + * 1 and (the number of unscheduled executors this child has / the number of unscheduled + * executors the current component has); * otherwise, the number of executors to take is 1; * for every executor to take from this child, call takeExecutors(...). + * * @param currComp The current component. * @param componentMap The map from component Id to component object. * @param compToExecsToSchedule The map from component Id to unscheduled executors. @@ -154,7 +162,7 @@ private List takeExecutors(Component currComp, List execsScheduled = new ArrayList<>(); Queue currQueue = compToExecsToSchedule.get(currComp.getId()); int currUnscheduledNumExecs = currQueue.size(); - //Just for defensive programming as this won't actually happen. + // Just for defensive programming as this won't actually happen. if (currUnscheduledNumExecs == 0) { return execsScheduled; } @@ -173,13 +181,15 @@ private List takeExecutors(Component currComp, numExecsToTake = Math.max(1, childUnscheduledNumExecs / currUnscheduledNumExecs); } // otherwise, one-by-one for (int i = 0; i < numExecsToTake; i++) { - execsScheduled.addAll(takeExecutors(childComponent, componentMap, compToExecsToSchedule)); + execsScheduled.addAll(takeExecutors(childComponent, componentMap, + compToExecsToSchedule)); } } return execsScheduled; } - private Set getSortedChildren(Component component, final Map componentMap) { + private Set getSortedChildren(Component component, final Map componentMap) { Set children = component.getChildren(); Set sortedChildren = new TreeSet<>((o1, o2) -> { diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/sorter/IExecSorter.java b/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/sorter/IExecSorter.java index 5371928057b..9df6005e5dc 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/sorter/IExecSorter.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/sorter/IExecSorter.java @@ -20,10 +20,8 @@ import java.util.List; import java.util.Set; - import org.apache.storm.scheduler.ExecutorDetails; - public interface IExecSorter { /** * Sort the supplied unique collection of ExecutorDetails in the order diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/sorter/INodeSorter.java b/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/sorter/INodeSorter.java index f2a4e8da8ed..817bf81f3d3 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/sorter/INodeSorter.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/sorter/INodeSorter.java @@ -21,11 +21,11 @@ import org.apache.storm.scheduler.ExecutorDetails; import org.apache.storm.scheduler.resource.strategies.scheduling.ObjectResourcesItem; - public interface INodeSorter { /** - * Prepare for node sorting. This method must be called before {@link #getSortedRacks()} and {@link #sortAllNodes()}. + * Prepare for node sorting. This method must be called before {@link #getSortedRacks()} and + * {@link #sortAllNodes()}. * * @param exec optional, may be null. */ diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/sorter/NodeSorter.java b/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/sorter/NodeSorter.java index 0286a8a0f3d..b81f5128758 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/sorter/NodeSorter.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/sorter/NodeSorter.java @@ -32,7 +32,6 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import java.util.stream.Stream; - import org.apache.storm.Config; import org.apache.storm.networktopography.DNSToSwitchMapping; import org.apache.storm.scheduler.Cluster; @@ -78,9 +77,11 @@ public class NodeSorter implements INodeSorter { * * @param cluster for which nodes will be sorted. * @param topologyDetails the topology to sort for. - * @param nodeSortType type of sorting to be applied to object resource collection {@link BaseResourceAwareStrategy.NodeSortType}. + * @param nodeSortType type of sorting to be applied to object resource collection {@link + * BaseResourceAwareStrategy.NodeSortType}. */ - public NodeSorter(Cluster cluster, TopologyDetails topologyDetails, BaseResourceAwareStrategy.NodeSortType nodeSortType) { + public NodeSorter(Cluster cluster, TopologyDetails topologyDetails, + BaseResourceAwareStrategy.NodeSortType nodeSortType) { this.cluster = cluster; this.topologyDetails = topologyDetails; this.nodeSortType = nodeSortType; @@ -106,8 +107,10 @@ public NodeSorter(Cluster cluster, TopologyDetails topologyDetails, BaseResource Map topoConf = topologyDetails.getConf(); // From Cluster and TopologyDetails - and cleaned-up - favoredNodeIds = makeHostToNodeIds((List) topoConf.get(Config.TOPOLOGY_SCHEDULER_FAVORED_NODES)); - unFavoredNodeIds = makeHostToNodeIds((List) topoConf.get(Config.TOPOLOGY_SCHEDULER_UNFAVORED_NODES)); + favoredNodeIds = makeHostToNodeIds((List) topoConf + .get(Config.TOPOLOGY_SCHEDULER_FAVORED_NODES)); + unFavoredNodeIds = makeHostToNodeIds((List) topoConf + .get(Config.TOPOLOGY_SCHEDULER_UNFAVORED_NODES)); favoredNodeIds.removeAll(greyListedSupervisorIds); unFavoredNodeIds.removeAll(greyListedSupervisorIds); unFavoredNodeIds.removeAll(favoredNodeIds); @@ -122,9 +125,11 @@ public void prepare(ExecutorDetails exec) { * Scheduling uses {@link #sortAllNodes()} which eventually * calls this method whose behavior can be altered by setting {@link #nodeSortType}. * - * @param resourcesSummary contains all individual {@link ObjectResourcesItem} as well as cumulative stats + * @param resourcesSummary contains all individual {@link ObjectResourcesItem} as well as + * cumulative stats * @param exec executor for which the sorting is done - * @param existingScheduleFunc a function to get existing executors already scheduled on this object + * @param existingScheduleFunc a function to get existing executors already scheduled on this + * object * @return a sorted list of {@link ObjectResourcesItem} */ protected List sortObjectResources( @@ -145,28 +150,41 @@ protected List sortObjectResources( * Sort objects by the following three criteria. * *

    • - * The number executors of the topology that needs to be scheduled is already on the object (node or rack) - * in descending order. The reasoning to sort based on criterion 1 is so we schedule the rest of a topology on + * The number executors of the topology that needs to be scheduled is already on the object + * (node or rack) + * in descending order. The reasoning to sort based on criterion 1 is so we schedule the rest of + * a topology on * the same object (node or rack) as the existing executors of the topology. *
    • * *
    • - * The subordinate/subservient resource availability percentage of a rack in descending order We calculate the - * resource availability percentage by dividing the resource availability of the object (node or rack) by the - * resource availability of the entire rack or cluster depending on if object references a node or a rack. - * How this differs from the DefaultResourceAwareStrategy is that the percentage boosts the node or rack if it is + * The subordinate/subservient resource availability percentage of a rack in descending order We + * calculate the + * resource availability percentage by dividing the resource availability of the object (node or + * rack) by the + * resource availability of the entire rack or cluster depending on if object references a node + * or a rack. + * How this differs from the DefaultResourceAwareStrategy is that the percentage boosts the node + * or rack if it is * requested by the executor that the sorting is being done for and pulls it down if it is not. - * By doing this calculation, objects (node or rack) that have exhausted or little of one of the resources mentioned - * above will be ranked after racks that have more balanced resource availability and nodes or racks that have - * resources that are not requested will be ranked below . So we will be less likely to pick a rack that - * have a lot of one resource but a low amount of another and have a lot of resources that are not requested by the executor. + * By doing this calculation, objects (node or rack) that have exhausted or little of one of the + * resources mentioned + * above will be ranked after racks that have more balanced resource availability and nodes or + * racks that have + * resources that are not requested will be ranked below . So we will be less likely to pick a + * rack that + * have a lot of one resource but a low amount of another and have a lot of resources that are + * not requested by the executor. * This is similar to logic used in - * {@link NodeSorter#sortObjectResourcesGeneric(ObjectResourcesSummary, ExecutorDetails, ExistingScheduleFunc)}. + * {@link NodeSorter#sortObjectResourcesGeneric(ObjectResourcesSummary, ExecutorDetails, + * ExistingScheduleFunc)}. *
    • * *
    • - * The tie between two nodes with same resource availability is broken by using the node with lower minimum - * percentage used. This comparison was used in {@link #sortObjectResourcesDefault(ObjectResourcesSummary, ExistingScheduleFunc)} + * The tie between two nodes with same resource availability is broken by using the node with + * lower minimum + * percentage used. This comparison was used in {@link + * #sortObjectResourcesDefault(ObjectResourcesSummary, ExistingScheduleFunc)} * but here it is made subservient to modified resource availbility used in * {@code #sortObjectResourcesGeneric(ObjectResourcesSummary, ExecutorDetails, ExistingScheduleFunc)}. * @@ -174,7 +192,8 @@ protected List sortObjectResources( * * @param allResources contains all individual ObjectResources as well as cumulative stats * @param exec executor for which the sorting is done - * @param existingScheduleFunc a function to get existing executors already scheduled on this object + * @param existingScheduleFunc a function to get existing executors already scheduled on this + * object * @return a sorted list of ObjectResources */ private List sortObjectResourcesCommon( @@ -182,18 +201,23 @@ private List sortObjectResourcesCommon( final ExistingScheduleFunc existingScheduleFunc) { // Copy and modify allResources ObjectResourcesSummary affinityBasedAllResources = new ObjectResourcesSummary(allResources); - final NormalizedResourceOffer availableResourcesOverall = allResources.getAvailableResourcesOverall(); - final NormalizedResourceRequest requestedResources = (exec != null) ? topologyDetails.getTotalResources(exec) : null; + final NormalizedResourceOffer availableResourcesOverall = allResources + .getAvailableResourcesOverall(); + final NormalizedResourceRequest requestedResources = (exec != null) ? topologyDetails + .getTotalResources(exec) : null; affinityBasedAllResources.getObjectResources().forEach( x -> { - x.minResourcePercent = availableResourcesOverall.calculateMinPercentageUsedBy(x.availableResources); + x.minResourcePercent = availableResourcesOverall + .calculateMinPercentageUsedBy(x.availableResources); if (requestedResources != null) { // negate unrequested resources x.availableResources.updateForRareResourceAffinity(requestedResources); } - x.avgResourcePercent = availableResourcesOverall.calculateAveragePercentageUsedBy(x.availableResources); + x.avgResourcePercent = availableResourcesOverall + .calculateAveragePercentageUsedBy(x.availableResources); - LOG.trace("for {}: minResourcePercent={}, avgResourcePercent={}, numExistingSchedule={}", + LOG.trace("for {}: minResourcePercent={}, avgResourcePercent={}, " + + "numExistingSchedule={}", x.id, x.minResourcePercent, x.avgResourcePercent, existingScheduleFunc.getNumExistingSchedule(x.id)); } @@ -233,22 +257,33 @@ private List sortObjectResourcesCommon( * Sort objects by the following two criteria. * *
    • the number executors of the topology that needs to be scheduled is already on the - * object (node or rack) in descending order. The reasoning to sort based on criterion 1 is so we schedule the rest - * of a topology on the same object (node or rack) as the existing executors of the topology.
    • + * object (node or rack) in descending order. The reasoning to sort based on criterion 1 is so + * we schedule the rest + * of a topology on the same object (node or rack) as the existing executors of the + * topology. * - *
    • the subordinate/subservient resource availability percentage of a rack in descending order We calculate the - * resource availability percentage by dividing the resource availability of the object (node or rack) by the - * resource availability of the entire rack or cluster depending on if object references a node or a rack. - * How this differs from the DefaultResourceAwareStrategy is that the percentage boosts the node or rack if it is + *
    • the subordinate/subservient resource availability percentage of a rack in descending + * order We calculate the + * resource availability percentage by dividing the resource availability of the object (node or + * rack) by the + * resource availability of the entire rack or cluster depending on if object references a node + * or a rack. + * How this differs from the DefaultResourceAwareStrategy is that the percentage boosts the node + * or rack if it is * requested by the executor that the sorting is being done for and pulls it down if it is not. - * By doing this calculation, objects (node or rack) that have exhausted or little of one of the resources mentioned - * above will be ranked after racks that have more balanced resource availability and nodes or racks that have - * resources that are not requested will be ranked below . So we will be less likely to pick a rack that - * have a lot of one resource but a low amount of another and have a lot of resources that are not requested by the executor.
    • + * By doing this calculation, objects (node or rack) that have exhausted or little of one of the + * resources mentioned + * above will be ranked after racks that have more balanced resource availability and nodes or + * racks that have + * resources that are not requested will be ranked below . So we will be less likely to pick a + * rack that + * have a lot of one resource but a low amount of another and have a lot of resources that are + * not requested by the executor. * * @param allResources contains all individual ObjectResources as well as cumulative stats * @param exec executor for which the sorting is done - * @param existingScheduleFunc a function to get existing executors already scheduled on this object + * @param existingScheduleFunc a function to get existing executors already scheduled on this + * object * @return a sorted list of ObjectResources */ @Deprecated @@ -258,8 +293,10 @@ private List sortObjectResourcesGeneric( ObjectResourcesSummary affinityBasedAllResources = new ObjectResourcesSummary(allResources); NormalizedResourceRequest requestedResources = topologyDetails.getTotalResources(exec); affinityBasedAllResources.getObjectResources() - .forEach(x -> x.availableResources.updateForRareResourceAffinity(requestedResources)); - final NormalizedResourceOffer availableResourcesOverall = allResources.getAvailableResourcesOverall(); + .forEach(x -> x.availableResources + .updateForRareResourceAffinity(requestedResources)); + final NormalizedResourceOffer availableResourcesOverall = allResources + .getAvailableResourcesOverall(); List sortedObjectResources = new ArrayList<>(); Comparator comparator = (o1, o2) -> { @@ -270,8 +307,10 @@ private List sortObjectResourcesGeneric( } else if (execsScheduled1 < execsScheduled2) { return 1; } - double o1Avg = availableResourcesOverall.calculateAveragePercentageUsedBy(o1.availableResources); - double o2Avg = availableResourcesOverall.calculateAveragePercentageUsedBy(o2.availableResources); + double o1Avg = availableResourcesOverall + .calculateAveragePercentageUsedBy(o1.availableResources); + double o2Avg = availableResourcesOverall + .calculateAveragePercentageUsedBy(o2.availableResources); if (o1Avg > o2Avg) { return -1; } else if (o1Avg < o2Avg) { @@ -289,18 +328,26 @@ private List sortObjectResourcesGeneric( * Sort objects by the following two criteria. * *
    • the number executors of the topology that needs to be scheduled is already on the - * object (node or rack) in descending order. The reasoning to sort based on criterion 1 is so we schedule the rest - * of a topology on the same object (node or rack) as the existing executors of the topology.
    • + * object (node or rack) in descending order. The reasoning to sort based on criterion 1 is so + * we schedule the rest + * of a topology on the same object (node or rack) as the existing executors of the + * topology. * - *
    • the subordinate/subservient resource availability percentage of a rack in descending order We calculate the - * resource availability percentage by dividing the resource availability of the object (node or rack) by the - * resource availability of the entire rack or cluster depending on if object references a node or a rack. - * By doing this calculation, objects (node or rack) that have exhausted or little of one of the resources mentioned - * above will be ranked after racks that have more balanced resource availability. So we will be less likely to pick + *
    • the subordinate/subservient resource availability percentage of a rack in descending + * order We calculate the + * resource availability percentage by dividing the resource availability of the object (node or + * rack) by the + * resource availability of the entire rack or cluster depending on if object references a node + * or a rack. + * By doing this calculation, objects (node or rack) that have exhausted or little of one of the + * resources mentioned + * above will be ranked after racks that have more balanced resource availability. So we will be + * less likely to pick * a rack that have a lot of one resource but a low amount of another.
    • * * @param allResources contains all individual ObjectResources as well as cumulative stats - * @param existingScheduleFunc a function to get existing executors already scheduled on this object + * @param existingScheduleFunc a function to get existing executors already scheduled on this + * object * @return a sorted list of ObjectResources */ @Deprecated @@ -308,13 +355,17 @@ private List sortObjectResourcesDefault( final ObjectResourcesSummary allResources, final ExistingScheduleFunc existingScheduleFunc) { - final NormalizedResourceOffer availableResourcesOverall = allResources.getAvailableResourcesOverall(); + final NormalizedResourceOffer availableResourcesOverall = allResources + .getAvailableResourcesOverall(); for (ObjectResourcesItem objectResources : allResources.getObjectResources()) { objectResources.minResourcePercent = - availableResourcesOverall.calculateMinPercentageUsedBy(objectResources.availableResources); + availableResourcesOverall + .calculateMinPercentageUsedBy(objectResources.availableResources); objectResources.avgResourcePercent = - availableResourcesOverall.calculateAveragePercentageUsedBy(objectResources.availableResources); - LOG.trace("for {}: minResourcePercent={}, avgResourcePercent={}, numExistingSchedule={}", + availableResourcesOverall + .calculateAveragePercentageUsedBy(objectResources.availableResources); + LOG.trace("for {}: minResourcePercent={}, avgResourcePercent={}, " + + "numExistingSchedule={}", objectResources.id, objectResources.minResourcePercent, objectResources.avgResourcePercent, existingScheduleFunc.getNumExistingSchedule(objectResources.id)); } @@ -350,14 +401,19 @@ private List sortObjectResourcesDefault( /** * Nodes are sorted by two criteria. * - *

      1) the number executors of the topology that needs to be scheduled is already on the node in - * descending order. The reasoning to sort based on criterion 1 is so we schedule the rest of a topology on the same node as the + *

      1) the number executors of the topology that needs to be scheduled is already on the node + * in + * descending order. The reasoning to sort based on criterion 1 is so we schedule the rest of a + * topology on the same node as the * existing executors of the topology. * *

      2) the subordinate/subservient resource availability percentage of a node in descending - * order We calculate the resource availability percentage by dividing the resource availability that have exhausted or little of one of - * the resources mentioned above will be ranked after on the node by the resource availability of the entire rack By doing this - * calculation, nodes that have more balanced resource availability. So we will be less likely to pick a node that have a lot of + * order We calculate the resource availability percentage by dividing the resource availability + * that have exhausted or little of one of + * the resources mentioned above will be ranked after on the node by the resource availability + * of the entire rack By doing this + * calculation, nodes that have more balanced resource availability. So we will be less likely + * to pick a node that have a lot of * one resource but a low amount of another. * * @param availRasNodes a list of all the nodes we want to sort @@ -370,7 +426,8 @@ private List sortNodes( ObjectResourcesSummary rackResourcesSummary = new ObjectResourcesSummary("RACK"); availRasNodes.forEach(x -> rackResourcesSummary.addObjectResourcesItem( - new ObjectResourcesItem(x.getId(), x.getTotalAvailableResources(), x.getTotalResources(), 0, 0) + new ObjectResourcesItem(x.getId(), x.getTotalAvailableResources(), x + .getTotalResources(), 0, 0) ) ); @@ -431,7 +488,7 @@ private Iterator getNodeIterator() { if (nodeIterator != null && nodeIterator.hasNext()) { return nodeIterator; } - //need to get the next node iterator + // need to get the next node iterator if (rackIterator.hasNext()) { ObjectResourcesItem rack = rackIterator.next(); final String rackId = rack.id; @@ -451,7 +508,7 @@ public boolean hasNext() { return true; } while (true) { - //For the node we don't know if we have another one unless we look at the contents + // For the node we don't know if we have another one unless we look at the contents Iterator nodeIterator = getNodeIterator(); if (nodeIterator == null || !nodeIterator.hasNext()) { break; @@ -510,7 +567,8 @@ private class LazyNodeSorting implements Iterable { private List getSortedNodesFor(String rackId) { return cachedNodes.computeIfAbsent(rackId, - (rid) -> sortNodes(rackIdToNodes.getOrDefault(rid, Collections.emptyList()), exec, rid, perNodeScheduledCount)); + (rid) -> sortNodes(rackIdToNodes.getOrDefault(rid, Collections.emptyList()), exec, + rid, perNodeScheduledCount)); } @Override @@ -527,7 +585,7 @@ public Iterable sortAllNodes() { private ObjectResourcesSummary createClusterSummarizedResources() { ObjectResourcesSummary clusterResourcesSummary = new ObjectResourcesSummary("Cluster"); - //This is the first time so initialize the resources. + // This is the first time so initialize the resources. for (Map.Entry> entry : networkTopography.entrySet()) { String rackId = entry.getKey(); List nodeHosts = entry.getValue(); @@ -567,14 +625,20 @@ private Map getScheduledExecCntByRackId() { /** * Racks are sorted by two criteria. * - *

      1) the number executors of the topology that needs to be scheduled is already on the rack in descending order. - * The reasoning to sort based on criterion 1 is so we schedule the rest of a topology on the same rack as the existing executors of the + *

      1) the number executors of the topology that needs to be scheduled is already on the rack + * in descending order. + * The reasoning to sort based on criterion 1 is so we schedule the rest of a topology on the + * same rack as the existing executors of the * topology. * - *

      2) the subordinate/subservient resource availability percentage of a rack in descending order We calculate - * the resource availability percentage by dividing the resource availability on the rack by the resource availability of the entire - * cluster By doing this calculation, racks that have exhausted or little of one of the resources mentioned above will be ranked after - * racks that have more balanced resource availability. So we will be less likely to pick a rack that have a lot of one resource but a + *

      2) the subordinate/subservient resource availability percentage of a rack in descending + * order We calculate + * the resource availability percentage by dividing the resource availability on the rack by the + * resource availability of the entire + * cluster By doing this calculation, racks that have exhausted or little of one of the + * resources mentioned above will be ranked after + * racks that have more balanced resource availability. So we will be less likely to pick a rack + * that have a lot of one resource but a * low amount of another. * * @return a sorted list of racks @@ -598,7 +662,7 @@ public List getSortedRacks() { } /** - * hostname to Ids. + * Hostname to Ids. * * @param hostname the hostname. * @return the ids n that node. @@ -608,7 +672,8 @@ public List hostnameToNodes(String hostname) { } /** - * interface for calculating the number of existing executors scheduled on a object (rack or node). + * Interface for calculating the number of existing executors scheduled on a object (rack or + * node). */ public interface ExistingScheduleFunc { int getNumExistingSchedule(String objectId); diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/sorter/NodeSorterHostProximity.java b/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/sorter/NodeSorterHostProximity.java index b3552746908..fff87a75b61 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/sorter/NodeSorterHostProximity.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/sorter/NodeSorterHostProximity.java @@ -80,11 +80,14 @@ public NodeSorterHostProximity(Cluster cluster, TopologyDetails topologyDetails) /** * Initialize for the default implementation node sorting. + * * @param cluster for which nodes will be sorted. * @param topologyDetails the topology to sort for. - * @param nodeSortType type of sorting to be applied to object resource collection {@link BaseResourceAwareStrategy.NodeSortType}. + * @param nodeSortType type of sorting to be applied to object resource collection {@link + * BaseResourceAwareStrategy.NodeSortType}. */ - public NodeSorterHostProximity(Cluster cluster, TopologyDetails topologyDetails, BaseResourceAwareStrategy.NodeSortType nodeSortType) { + public NodeSorterHostProximity(Cluster cluster, TopologyDetails topologyDetails, + BaseResourceAwareStrategy.NodeSortType nodeSortType) { this.cluster = cluster; this.topologyDetails = topologyDetails; this.nodeSortType = nodeSortType; @@ -110,8 +113,10 @@ public NodeSorterHostProximity(Cluster cluster, TopologyDetails topologyDetails, Map topoConf = topologyDetails.getConf(); // From Cluster and TopologyDetails - and cleaned-up - favoredNodeIds = makeHostToNodeIds((List) topoConf.get(Config.TOPOLOGY_SCHEDULER_FAVORED_NODES)); - unFavoredNodeIds = makeHostToNodeIds((List) topoConf.get(Config.TOPOLOGY_SCHEDULER_UNFAVORED_NODES)); + favoredNodeIds = makeHostToNodeIds((List) topoConf + .get(Config.TOPOLOGY_SCHEDULER_FAVORED_NODES)); + unFavoredNodeIds = makeHostToNodeIds((List) topoConf + .get(Config.TOPOLOGY_SCHEDULER_UNFAVORED_NODES)); favoredNodeIds.removeAll(greyListedSupervisorIds); unFavoredNodeIds.removeAll(greyListedSupervisorIds); unFavoredNodeIds.removeAll(favoredNodeIds); @@ -131,9 +136,11 @@ public void prepare(ExecutorDetails exec) { * Scheduling uses {@link #sortAllNodes()} which eventually * calls this method whose behavior can be altered by setting {@link #nodeSortType}. * - * @param resourcesSummary contains all individual {@link ObjectResourcesItem} as well as cumulative stats + * @param resourcesSummary contains all individual {@link ObjectResourcesItem} as well as + * cumulative stats * @param exec executor for which the sorting is done - * @param existingScheduleFunc a function to get existing executors already scheduled on this object + * @param existingScheduleFunc a function to get existing executors already scheduled on this + * object * @return an {@link Iterable} of sorted {@link ObjectResourcesItem} */ protected Iterable sortObjectResources( @@ -154,28 +161,41 @@ protected Iterable sortObjectResources( * Sort objects by the following three criteria. * *

    • - * The number executors of the topology that needs to be scheduled is already on the object (node or rack) - * in descending order. The reasoning to sort based on criterion 1 is so we schedule the rest of a topology on + * The number executors of the topology that needs to be scheduled is already on the object + * (node or rack) + * in descending order. The reasoning to sort based on criterion 1 is so we schedule the rest of + * a topology on * the same object (node or rack) as the existing executors of the topology. *
    • * *
    • - * The subordinate/subservient resource availability percentage of a rack in descending order We calculate the - * resource availability percentage by dividing the resource availability of the object (node or rack) by the - * resource availability of the entire rack or cluster depending on if object references a node or a rack. - * How this differs from the DefaultResourceAwareStrategy is that the percentage boosts the node or rack if it is + * The subordinate/subservient resource availability percentage of a rack in descending order We + * calculate the + * resource availability percentage by dividing the resource availability of the object (node or + * rack) by the + * resource availability of the entire rack or cluster depending on if object references a node + * or a rack. + * How this differs from the DefaultResourceAwareStrategy is that the percentage boosts the node + * or rack if it is * requested by the executor that the sorting is being done for and pulls it down if it is not. - * By doing this calculation, objects (node or rack) that have exhausted or little of one of the resources mentioned - * above will be ranked after racks that have more balanced resource availability and nodes or racks that have - * resources that are not requested will be ranked below . So we will be less likely to pick a rack that - * have a lot of one resource but a low amount of another and have a lot of resources that are not requested by the executor. + * By doing this calculation, objects (node or rack) that have exhausted or little of one of the + * resources mentioned + * above will be ranked after racks that have more balanced resource availability and nodes or + * racks that have + * resources that are not requested will be ranked below . So we will be less likely to pick a + * rack that + * have a lot of one resource but a low amount of another and have a lot of resources that are + * not requested by the executor. * This is similar to logic used in - * {@link NodeSorterHostProximity#sortObjectResourcesGeneric(ObjectResourcesSummary, ExecutorDetails, ExistingScheduleFunc)}. + * {@link NodeSorterHostProximity#sortObjectResourcesGeneric(ObjectResourcesSummary, + * ExecutorDetails, ExistingScheduleFunc)}. *
    • * *
    • - * The tie between two nodes with same resource availability is broken by using the node with lower minimum - * percentage used. This comparison was used in {@link #sortObjectResourcesDefault(ObjectResourcesSummary, ExistingScheduleFunc)} + * The tie between two nodes with same resource availability is broken by using the node with + * lower minimum + * percentage used. This comparison was used in {@link + * #sortObjectResourcesDefault(ObjectResourcesSummary, ExistingScheduleFunc)} * but here it is made subservient to modified resource availability used in * {@code #sortObjectResourcesGeneric(ObjectResourcesSummary, ExecutorDetails, ExistingScheduleFunc)}. * @@ -183,7 +203,8 @@ protected Iterable sortObjectResources( * * @param allResources contains all individual ObjectResources as well as cumulative stats * @param exec executor for which the sorting is done - * @param existingScheduleFunc a function to get existing executors already scheduled on this object + * @param existingScheduleFunc a function to get existing executors already scheduled on this + * object * @return an {@link Iterable} of sorted {@link ObjectResourcesItem} */ private Iterable sortObjectResourcesCommon( @@ -191,18 +212,23 @@ private Iterable sortObjectResourcesCommon( final ExistingScheduleFunc existingScheduleFunc) { // Copy and modify allResources ObjectResourcesSummary affinityBasedAllResources = new ObjectResourcesSummary(allResources); - final NormalizedResourceOffer availableResourcesOverall = allResources.getAvailableResourcesOverall(); - final NormalizedResourceRequest requestedResources = (exec != null) ? topologyDetails.getTotalResources(exec) : null; + final NormalizedResourceOffer availableResourcesOverall = allResources + .getAvailableResourcesOverall(); + final NormalizedResourceRequest requestedResources = (exec != null) ? topologyDetails + .getTotalResources(exec) : null; affinityBasedAllResources.getObjectResources().forEach( x -> { if (requestedResources != null) { // negate unrequested resources x.availableResources.updateForRareResourceAffinity(requestedResources); } - x.minResourcePercent = availableResourcesOverall.calculateMinPercentageUsedBy(x.availableResources); - x.avgResourcePercent = availableResourcesOverall.calculateAveragePercentageUsedBy(x.availableResources); + x.minResourcePercent = availableResourcesOverall + .calculateMinPercentageUsedBy(x.availableResources); + x.avgResourcePercent = availableResourcesOverall + .calculateAveragePercentageUsedBy(x.availableResources); - LOG.trace("for {}: minResourcePercent={}, avgResourcePercent={}, numExistingSchedule={}", + LOG.trace("for {}: minResourcePercent={}, avgResourcePercent={}, " + + "numExistingSchedule={}", x.id, x.minResourcePercent, x.avgResourcePercent, existingScheduleFunc.getNumExistingSchedule(x.id)); } @@ -241,22 +267,33 @@ private Iterable sortObjectResourcesCommon( * Sort objects by the following two criteria. * *
    • the number executors of the topology that needs to be scheduled is already on the - * object (node or rack) in descending order. The reasoning to sort based on criterion 1 is so we schedule the rest - * of a topology on the same object (node or rack) as the existing executors of the topology.
    • + * object (node or rack) in descending order. The reasoning to sort based on criterion 1 is so + * we schedule the rest + * of a topology on the same object (node or rack) as the existing executors of the + * topology. * - *
    • the subordinate/subservient resource availability percentage of a rack in descending order We calculate the - * resource availability percentage by dividing the resource availability of the object (node or rack) by the - * resource availability of the entire rack or cluster depending on if object references a node or a rack. - * How this differs from the DefaultResourceAwareStrategy is that the percentage boosts the node or rack if it is + *
    • the subordinate/subservient resource availability percentage of a rack in descending + * order We calculate the + * resource availability percentage by dividing the resource availability of the object (node or + * rack) by the + * resource availability of the entire rack or cluster depending on if object references a node + * or a rack. + * How this differs from the DefaultResourceAwareStrategy is that the percentage boosts the node + * or rack if it is * requested by the executor that the sorting is being done for and pulls it down if it is not. - * By doing this calculation, objects (node or rack) that have exhausted or little of one of the resources mentioned - * above will be ranked after racks that have more balanced resource availability and nodes or racks that have - * resources that are not requested will be ranked below . So we will be less likely to pick a rack that - * have a lot of one resource but a low amount of another and have a lot of resources that are not requested by the executor.
    • + * By doing this calculation, objects (node or rack) that have exhausted or little of one of the + * resources mentioned + * above will be ranked after racks that have more balanced resource availability and nodes or + * racks that have + * resources that are not requested will be ranked below . So we will be less likely to pick a + * rack that + * have a lot of one resource but a low amount of another and have a lot of resources that are + * not requested by the executor. * * @param allResources contains all individual ObjectResources as well as cumulative stats * @param exec executor for which the sorting is done - * @param existingScheduleFunc a function to get existing executors already scheduled on this object + * @param existingScheduleFunc a function to get existing executors already scheduled on this + * object * @return an {@link Iterable} of sorted {@link ObjectResourcesItem} */ @Deprecated @@ -264,18 +301,23 @@ private Iterable sortObjectResourcesGeneric( final ObjectResourcesSummary allResources, ExecutorDetails exec, final ExistingScheduleFunc existingScheduleFunc) { ObjectResourcesSummary affinityBasedAllResources = new ObjectResourcesSummary(allResources); - final NormalizedResourceOffer availableResourcesOverall = allResources.getAvailableResourcesOverall(); - final NormalizedResourceRequest requestedResources = (exec != null) ? topologyDetails.getTotalResources(exec) : null; + final NormalizedResourceOffer availableResourcesOverall = allResources + .getAvailableResourcesOverall(); + final NormalizedResourceRequest requestedResources = (exec != null) ? topologyDetails + .getTotalResources(exec) : null; affinityBasedAllResources.getObjectResources().forEach( x -> { if (requestedResources != null) { // negate unrequested resources x.availableResources.updateForRareResourceAffinity(requestedResources); } - x.minResourcePercent = availableResourcesOverall.calculateMinPercentageUsedBy(x.availableResources); - x.avgResourcePercent = availableResourcesOverall.calculateAveragePercentageUsedBy(x.availableResources); + x.minResourcePercent = availableResourcesOverall + .calculateMinPercentageUsedBy(x.availableResources); + x.avgResourcePercent = availableResourcesOverall + .calculateAveragePercentageUsedBy(x.availableResources); - LOG.trace("for {}: minResourcePercent={}, avgResourcePercent={}, numExistingSchedule={}", + LOG.trace("for {}: minResourcePercent={}, avgResourcePercent={}, " + + "numExistingSchedule={}", x.id, x.minResourcePercent, x.avgResourcePercent, existingScheduleFunc.getNumExistingSchedule(x.id)); } @@ -308,18 +350,26 @@ private Iterable sortObjectResourcesGeneric( * Sort objects by the following two criteria. * *
    • the number executors of the topology that needs to be scheduled is already on the - * object (node or rack) in descending order. The reasoning to sort based on criterion 1 is so we schedule the rest - * of a topology on the same object (node or rack) as the existing executors of the topology.
    • + * object (node or rack) in descending order. The reasoning to sort based on criterion 1 is so + * we schedule the rest + * of a topology on the same object (node or rack) as the existing executors of the + * topology. * - *
    • the subordinate/subservient resource availability percentage of a rack in descending order We calculate the - * resource availability percentage by dividing the resource availability of the object (node or rack) by the - * resource availability of the entire rack or cluster depending on if object references a node or a rack. - * By doing this calculation, objects (node or rack) that have exhausted or little of one of the resources mentioned - * above will be ranked after racks that have more balanced resource availability. So we will be less likely to pick + *
    • the subordinate/subservient resource availability percentage of a rack in descending + * order We calculate the + * resource availability percentage by dividing the resource availability of the object (node or + * rack) by the + * resource availability of the entire rack or cluster depending on if object references a node + * or a rack. + * By doing this calculation, objects (node or rack) that have exhausted or little of one of the + * resources mentioned + * above will be ranked after racks that have more balanced resource availability. So we will be + * less likely to pick * a rack that have a lot of one resource but a low amount of another.
    • * * @param allResources contains all individual ObjectResources as well as cumulative stats - * @param existingScheduleFunc a function to get existing executors already scheduled on this object + * @param existingScheduleFunc a function to get existing executors already scheduled on this + * object * @return an {@link Iterable} of sorted {@link ObjectResourcesItem} */ @Deprecated @@ -327,13 +377,17 @@ private Iterable sortObjectResourcesDefault( final ObjectResourcesSummary allResources, final ExistingScheduleFunc existingScheduleFunc) { - final NormalizedResourceOffer availableResourcesOverall = allResources.getAvailableResourcesOverall(); + final NormalizedResourceOffer availableResourcesOverall = allResources + .getAvailableResourcesOverall(); for (ObjectResourcesItem objectResources : allResources.getObjectResources()) { objectResources.minResourcePercent = - availableResourcesOverall.calculateMinPercentageUsedBy(objectResources.availableResources); + availableResourcesOverall + .calculateMinPercentageUsedBy(objectResources.availableResources); objectResources.avgResourcePercent = - availableResourcesOverall.calculateAveragePercentageUsedBy(objectResources.availableResources); - LOG.trace("for {}: minResourcePercent={}, avgResourcePercent={}, numExistingSchedule={}", + availableResourcesOverall + .calculateAveragePercentageUsedBy(objectResources.availableResources); + LOG.trace("for {}: minResourcePercent={}, avgResourcePercent={}, " + + "numExistingSchedule={}", objectResources.id, objectResources.minResourcePercent, objectResources.avgResourcePercent, existingScheduleFunc.getNumExistingSchedule(objectResources.id)); } @@ -368,14 +422,19 @@ private Iterable sortObjectResourcesDefault( /** * Nodes are sorted by two criteria. * - *

      1) the number executors of the topology that needs to be scheduled is already on the node in - * descending order. The reasoning to sort based on criterion 1 is so we schedule the rest of a topology on the same node as the + *

      1) the number executors of the topology that needs to be scheduled is already on the node + * in + * descending order. The reasoning to sort based on criterion 1 is so we schedule the rest of a + * topology on the same node as the * existing executors of the topology. * *

      2) the subordinate/subservient resource availability percentage of a node in descending - * order We calculate the resource availability percentage by dividing the resource availability that have exhausted or little of one of - * the resources mentioned above will be ranked after on the node by the resource availability of the entire rack By doing this - * calculation, nodes that have more balanced resource availability. So we will be less likely to pick a node that have a lot of + * order We calculate the resource availability percentage by dividing the resource availability + * that have exhausted or little of one of + * the resources mentioned above will be ranked after on the node by the resource availability + * of the entire rack By doing this + * calculation, nodes that have more balanced resource availability. So we will be less likely + * to pick a node that have a lot of * one resource but a low amount of another. * * @param availHosts a collection of all the hosts we want to sort @@ -389,7 +448,8 @@ private Iterable sortHosts( availHosts.forEach(h -> { ObjectResourcesItem hostItem = new ObjectResourcesItem(h); for (RasNode x : hostnameToNodes.get(h)) { - hostItem.add(new ObjectResourcesItem(x.getId(), x.getTotalAvailableResources(), x.getTotalResources(), 0, 0)); + hostItem.add(new ObjectResourcesItem(x.getId(), x.getTotalAvailableResources(), x + .getTotalResources(), 0, 0)); } rackResourcesSummary.addObjectResourcesItem(hostItem); }); @@ -415,14 +475,19 @@ private Iterable sortHosts( /** * Nodes are sorted by two criteria. * - *

      1) the number executors of the topology that needs to be scheduled is already on the node in - * descending order. The reasoning to sort based on criterion 1 is so we schedule the rest of a topology on the same node as the + *

      1) the number executors of the topology that needs to be scheduled is already on the node + * in + * descending order. The reasoning to sort based on criterion 1 is so we schedule the rest of a + * topology on the same node as the * existing executors of the topology. * *

      2) the subordinate/subservient resource availability percentage of a node in descending - * order We calculate the resource availability percentage by dividing the resource availability that have exhausted or little of one of - * the resources mentioned above will be ranked after on the node by the resource availability of the entire rack By doing this - * calculation, nodes that have more balanced resource availability. So we will be less likely to pick a node that have a lot of + * order We calculate the resource availability percentage by dividing the resource availability + * that have exhausted or little of one of + * the resources mentioned above will be ranked after on the node by the resource availability + * of the entire rack By doing this + * calculation, nodes that have more balanced resource availability. So we will be less likely + * to pick a node that have a lot of * one resource but a low amount of another. * * @param availRasNodes a list of all the nodes we want to sort @@ -435,7 +500,8 @@ private Iterable sortNodes( ObjectResourcesSummary hostResourcesSummary = new ObjectResourcesSummary("HOST"); availRasNodes.forEach(x -> hostResourcesSummary.addObjectResourcesItem( - new ObjectResourcesItem(x.getId(), x.getTotalAvailableResources(), x.getTotalResources(), 0, 0) + new ObjectResourcesItem(x.getId(), x.getTotalAvailableResources(), x + .getTotalResources(), 0, 0) ) ); @@ -497,7 +563,7 @@ private Iterator getNodeIterator() { if (nodeIterator != null && nodeIterator.hasNext()) { return nodeIterator; } - //need to get the next host/node iterator + // need to get the next host/node iterator if (hostIterator != null && hostIterator.hasNext()) { ObjectResourcesItem host = hostIterator.next(); final String hostId = host.id; @@ -526,7 +592,7 @@ public boolean hasNext() { return true; } while (true) { - //For the node we don't know if we have another one unless we look at the contents + // For the node we don't know if we have another one unless we look at the contents Iterator nodeIterator = getNodeIterator(); if (nodeIterator == null || !nodeIterator.hasNext()) { break; @@ -562,7 +628,8 @@ private class LazyNodeSorting implements Iterable { private final Map perNodeScheduledCount = new HashMap<>(); private final Iterable sortedRacks; private final Map> cachedHosts = new HashMap<>(); - private final Map> cachedNodesByHost = new HashMap<>(); + private final Map> cachedNodesByHost = + new HashMap<>(); private final ExecutorDetails exec; private final Set skippedNodeIds = new HashSet<>(); @@ -590,12 +657,14 @@ private class LazyNodeSorting implements Iterable { private Iterable getSortedHostsForRack(String rackId) { return cachedHosts.computeIfAbsent(rackId, - id -> sortHosts(rackIdToHosts.getOrDefault(id, Collections.emptySet()), exec, id, perHostScheduledCount)); + id -> sortHosts(rackIdToHosts.getOrDefault(id, Collections.emptySet()), exec, id, + perHostScheduledCount)); } private Iterable getSortedNodesForHost(String hostId) { return cachedNodesByHost.computeIfAbsent(hostId, - id -> sortNodes(hostnameToNodes.getOrDefault(id, Collections.emptyList()), exec, id, perNodeScheduledCount)); + id -> sortNodes(hostnameToNodes.getOrDefault(id, Collections.emptyList()), exec, id, + perNodeScheduledCount)); } @Override @@ -653,14 +722,20 @@ public Map getScheduledExecCntByRackId() { /** * Racks are sorted by two criteria. * - *

      1) the number executors of the topology that needs to be scheduled is already on the rack in descending order. - * The reasoning to sort based on criterion 1 is so we schedule the rest of a topology on the same rack as the existing executors of the + *

      1) the number executors of the topology that needs to be scheduled is already on the rack + * in descending order. + * The reasoning to sort based on criterion 1 is so we schedule the rest of a topology on the + * same rack as the existing executors of the * topology. * - *

      2) the subordinate/subservient resource availability percentage of a rack in descending order We calculate - * the resource availability percentage by dividing the resource availability on the rack by the resource availability of the entire - * cluster By doing this calculation, racks that have exhausted or little of one of the resources mentioned above will be ranked after - * racks that have more balanced resource availability. So we will be less likely to pick a rack that have a lot of one resource but a + *

      2) the subordinate/subservient resource availability percentage of a rack in descending + * order We calculate + * the resource availability percentage by dividing the resource availability on the rack by the + * resource availability of the entire + * cluster By doing this calculation, racks that have exhausted or little of one of the + * resources mentioned above will be ranked after + * racks that have more balanced resource availability. So we will be less likely to pick a rack + * that have a lot of one resource but a * low amount of another. * * @return an iterable of sorted racks @@ -684,7 +759,7 @@ public Iterable getSortedRacks() { } /** - * hostname to Ids. + * Hostname to Ids. * * @param hostname the hostname. * @return the ids n that node. @@ -694,7 +769,8 @@ public List hostnameToNodes(String hostname) { } /** - * interface for calculating the number of existing executors scheduled on a object (rack or node). + * Interface for calculating the number of existing executors scheduled on a object (rack or + * node). */ public interface ExistingScheduleFunc { int getNumExistingSchedule(String objectId); diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/utils/ArtifactoryConfigLoader.java b/storm-server/src/main/java/org/apache/storm/scheduler/utils/ArtifactoryConfigLoader.java index 8bbad3f104a..20d07a27c2c 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/utils/ArtifactoryConfigLoader.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/utils/ArtifactoryConfigLoader.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -44,7 +50,8 @@ import org.yaml.snakeyaml.constructor.SafeConstructor; /** - * A dynamic loader that can load scheduler configurations for user resource guarantees from Artifactory (an artifact repository manager). + * A dynamic loader that can load scheduler configurations for user resource guarantees from + * Artifactory (an artifact repository manager). * This is not thread-safe. */ public class ArtifactoryConfigLoader implements IConfigLoader { @@ -76,17 +83,20 @@ public ArtifactoryConfigLoader(Map conf) { if (thisTimeout != null) { timeoutSeconds = thisTimeout; } - Integer thisPollTime = (Integer) conf.get(DaemonConfig.SCHEDULER_CONFIG_LOADER_POLLTIME_SECS); + Integer thisPollTime = (Integer) conf + .get(DaemonConfig.SCHEDULER_CONFIG_LOADER_POLLTIME_SECS); if (thisPollTime != null) { artifactoryPollTimeSecs = thisPollTime; } - String thisBase = (String) conf.get(DaemonConfig.SCHEDULER_CONFIG_LOADER_ARTIFACTORY_BASE_DIRECTORY); + String thisBase = (String) conf + .get(DaemonConfig.SCHEDULER_CONFIG_LOADER_ARTIFACTORY_BASE_DIRECTORY); if (thisBase != null) { baseDirectory = thisBase; } String uriString = (String) conf.get(DaemonConfig.SCHEDULER_CONFIG_LOADER_URI); if (uriString == null) { - LOG.error("No URI defined in {} configuration.", DaemonConfig.SCHEDULER_CONFIG_LOADER_URI); + LOG.error("No URI defined in {} configuration.", + DaemonConfig.SCHEDULER_CONFIG_LOADER_URI); } else { try { targetUri = new URI(uriString); @@ -100,6 +110,7 @@ public ArtifactoryConfigLoader(Map conf) { /** * Load the configs associated with the configKey from the targetURI. + * * @param configKey The key from which we want to get the scheduler config. * @return The scheduler configuration if exists; null otherwise. */ @@ -111,8 +122,10 @@ public Map load(String configKey) { // Check for new file every so often int currentTimeSecs = Time.currentTimeSecs(); - if (lastReturnedValue != null && ((currentTimeSecs - lastReturnedTime) < artifactoryPollTimeSecs)) { - LOG.debug("currentTimeSecs: {}; lastReturnedTime {}; artifactoryPollTimeSecs: {}. Returning our last map.", + if (lastReturnedValue != null + && ((currentTimeSecs - lastReturnedTime) < artifactoryPollTimeSecs)) { + LOG.debug("currentTimeSecs: {}; lastReturnedTime {}; artifactoryPollTimeSecs: {}. " + + "Returning our last map.", currentTimeSecs, lastReturnedTime, artifactoryPollTimeSecs); return (Map) lastReturnedValue.get(configKey); } @@ -152,8 +165,10 @@ protected String doGet(String api, String artifact, String host, Integer port) { path = path.replaceAll("/[/]+", "/"); builder.setPath(path); - RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(timeoutSeconds * 1000).build(); - HttpClient httpclient = HttpClientBuilder.create().setDefaultRequestConfig(requestConfig).build(); + RequestConfig requestConfig = RequestConfig.custom() + .setConnectTimeout(timeoutSeconds * 1000).build(); + HttpClient httpclient = HttpClientBuilder.create().setDefaultRequestConfig(requestConfig) + .build(); String returnValue; try { @@ -275,18 +290,21 @@ private void saveInArtifactoryCache(String yamlData) { fos.write(yamlData.getBytes()); fos.flush(); } catch (IOException e) { - LOG.error("Received exception when writing file {}. Attempting delete", localFileName, e); + LOG.error("Received exception when writing file {}. Attempting delete", localFileName, + e); try { cacheFile.delete(); } catch (Exception deleteException) { - LOG.error("Received exception when deleting file {}.", localFileName, deleteException); + LOG.error("Received exception when deleting file {}.", localFileName, + deleteException); } } } private void makeArtifactoryCache(String location) throws IOException { // First make the cache dir - String localDirName = ServerConfigUtils.masterLocalDir(conf) + File.separator + LOCAL_ARTIFACT_DIR; + String localDirName = ServerConfigUtils.masterLocalDir(conf) + File.separator + + LOCAL_ARTIFACT_DIR; File dir = new File(localDirName); if (!dir.exists()) { dir.mkdirs(); @@ -349,6 +367,7 @@ private static class GetStringResponseHandler implements ResponseHandler /** * Get instance. + * * @return a singleton httpclient GET response handler */ public static GetStringResponseHandler getInstance() { @@ -360,6 +379,7 @@ public static GetStringResponseHandler getInstance() { /** * Handle response. + * * @param response The http response to verify. * @return null on failure or the response string if return code is in 200 range */ diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/utils/ArtifactoryConfigLoaderFactory.java b/storm-server/src/main/java/org/apache/storm/scheduler/utils/ArtifactoryConfigLoaderFactory.java index 71b9eb5f157..70fc53f6474 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/utils/ArtifactoryConfigLoaderFactory.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/utils/ArtifactoryConfigLoaderFactory.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -26,15 +32,20 @@ public class ArtifactoryConfigLoaderFactory implements IConfigLoaderFactory { private static final Logger LOG = LoggerFactory.getLogger(ArtifactoryConfigLoaderFactory.class); /** - * Create a ArtifactoryConfigLoader if the scheme of the URI is "artifactory+http" or "artifactory+https"; otherwise return null. - * @param uri The URI which pointing to the config file/directory location on Artifactory server. + * Create a ArtifactoryConfigLoader if the scheme of the URI is "artifactory+http" or + * "artifactory+https"; otherwise return null. + * + * @param uri The URI which pointing to the config file/directory location on Artifactory + * server. * @param conf The storm configuration. - * @return A ArtifactoryConfigLoader if the scheme is "artifactory+http" or "artifactory+https"; otherwise, null. + * @return A ArtifactoryConfigLoader if the scheme is "artifactory+http" or "artifactory+https"; + * otherwise, null. */ @Override public IConfigLoader createIfSupported(URI uri, Map conf) { String scheme = uri.getScheme(); - if ("artifactory+http".equalsIgnoreCase(scheme) || "artifactory+https".equalsIgnoreCase(scheme)) { + if ("artifactory+http".equalsIgnoreCase(scheme) + || "artifactory+https".equalsIgnoreCase(scheme)) { return new ArtifactoryConfigLoader(conf); } else { LOG.debug("scheme {} not supported in this factory.", scheme); diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/utils/ConfigLoaderFactoryService.java b/storm-server/src/main/java/org/apache/storm/scheduler/utils/ConfigLoaderFactoryService.java index 4e1b3a5e6c6..1163392bf9a 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/utils/ConfigLoaderFactoryService.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/utils/ConfigLoaderFactoryService.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -30,12 +36,15 @@ public class ConfigLoaderFactoryService { /** * SerivceLoader loads all the implementations of IConfigLoaderFactory for use. */ - private static ServiceLoader serviceLoader = ServiceLoader.load(IConfigLoaderFactory.class); + private static ServiceLoader serviceLoader = ServiceLoader + .load(IConfigLoaderFactory.class); /** * The user interface to create an IConfigLoader instance. - * It iterates all the implementations of IConfigLoaderFactory and finds the one which supports the + * It iterates all the implementations of IConfigLoaderFactory and finds the one which supports + * the * specific scheme of the URI and then uses it to create an IConfigLoader instance. + * * @param conf The storm configuration. * @return A concrete IConfigLoader implementation which supports the scheme of the URI. * If multiple implementations are available, return the first one; otherwise, return null. diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/utils/FileConfigLoader.java b/storm-server/src/main/java/org/apache/storm/scheduler/utils/FileConfigLoader.java index 6a45882e5de..35be9819277 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/utils/FileConfigLoader.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/utils/FileConfigLoader.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -33,7 +39,8 @@ public FileConfigLoader(Map conf) { this.conf = conf; String uriString = (String) conf.get(DaemonConfig.SCHEDULER_CONFIG_LOADER_URI); if (uriString == null) { - LOG.error("No URI defined in {} configuration.", DaemonConfig.SCHEDULER_CONFIG_LOADER_URI); + LOG.error("No URI defined in {} configuration.", + DaemonConfig.SCHEDULER_CONFIG_LOADER_URI); } else { try { targetFilePath = new URI(uriString).getPath(); @@ -45,6 +52,7 @@ public FileConfigLoader(Map conf) { /** * Load the configs associated with the configKey from the targetFilePath. + * * @param configKey The key from which we want to get the scheduler config. * @return The scheduler configuration if exists; null otherwise. */ diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/utils/FileConfigLoaderFactory.java b/storm-server/src/main/java/org/apache/storm/scheduler/utils/FileConfigLoaderFactory.java index c9abd1b71a3..bf9e45cb7db 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/utils/FileConfigLoaderFactory.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/utils/FileConfigLoaderFactory.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -28,6 +34,7 @@ public class FileConfigLoaderFactory implements IConfigLoaderFactory { /** * Create a FileConfigLoader if the scheme of the URI is "file"; else return null. + * * @param uri The URI which pointing to the config file location. * @param conf The storm configuration. * @return A FileConfigLoader if the scheme is "file"; otherwise, null. diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/utils/IConfigLoader.java b/storm-server/src/main/java/org/apache/storm/scheduler/utils/IConfigLoader.java index 6bfd6bee076..056bb76695c 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/utils/IConfigLoader.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/utils/IConfigLoader.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -18,6 +24,7 @@ public interface IConfigLoader { /** * Load scheduler configs associated with the configKey. + * * @param configKey The key from which we want to get the scheduler config. * @return The scheduler configs */ diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/utils/IConfigLoaderFactory.java b/storm-server/src/main/java/org/apache/storm/scheduler/utils/IConfigLoaderFactory.java index bb73496ab53..30eea3be3b8 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/utils/IConfigLoaderFactory.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/utils/IConfigLoaderFactory.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -18,7 +24,8 @@ public interface IConfigLoaderFactory { /** - * Create an IConfigLoader implementation if the scheme of the URI is supported; otherwise returns null. + * Create an IConfigLoader implementation if the scheme of the URI is supported; otherwise + * returns null. * * @param uri The URI of the config location. * @param conf The storm configuration. diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/utils/SchedulerConfigCache.java b/storm-server/src/main/java/org/apache/storm/scheduler/utils/SchedulerConfigCache.java index e85bb9f7e99..2516badc2ce 100644 --- a/storm-server/src/main/java/org/apache/storm/scheduler/utils/SchedulerConfigCache.java +++ b/storm-server/src/main/java/org/apache/storm/scheduler/utils/SchedulerConfigCache.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -40,11 +46,12 @@ public SchedulerConfigCache(Map conf, Reloadable reloader) { schedulerConfigAtomicReference = new AtomicReference<>(); lastUpdateTimestamp = 0; this.reloader = reloader; - configCacheExpirationMs = ObjectReader.getInt(conf.get(DaemonConfig.SCHEDULER_CONFIG_CACHE_EXPIRATION_SECS), 60) * 1000L; + configCacheExpirationMs = ObjectReader.getInt(conf + .get(DaemonConfig.SCHEDULER_CONFIG_CACHE_EXPIRATION_SECS), 60) * 1000L; } public void prepare() { - //refresh the cache here to make sure cache is available at very beginning + // refresh the cache here to make sure cache is available at very beginning refresh(); } @@ -67,6 +74,7 @@ public void refresh() { /** * Get the scheduler config from cache. * This method is thead-safe and can be called in multiple threads. + * * @return the scheduler config */ public T get() { @@ -76,6 +84,7 @@ public T get() { public interface Reloadable { /** * Reload and return the configs. + * * @return reloaded configs. It can't be null. */ T reload(); diff --git a/storm-server/src/main/java/org/apache/storm/security/auth/DefaultHttpCredentialsPlugin.java b/storm-server/src/main/java/org/apache/storm/security/auth/DefaultHttpCredentialsPlugin.java index 41e2f0b980b..ebc6f42c232 100644 --- a/storm-server/src/main/java/org/apache/storm/security/auth/DefaultHttpCredentialsPlugin.java +++ b/storm-server/src/main/java/org/apache/storm/security/auth/DefaultHttpCredentialsPlugin.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -56,7 +62,8 @@ public String getUserName(jakarta.servlet.http.HttpServletRequest req) { } @Override - public ReqContext populateContext(ReqContext context, jakarta.servlet.http.HttpServletRequest req) { + public ReqContext populateContext(ReqContext context, + jakarta.servlet.http.HttpServletRequest req) { String userName = getUserName(req); String doAsUser = req.getHeader("doAsUser"); diff --git a/storm-server/src/main/java/org/apache/storm/security/auth/workertoken/WorkerTokenManager.java b/storm-server/src/main/java/org/apache/storm/security/auth/workertoken/WorkerTokenManager.java index 14903286606..baebc39afd2 100644 --- a/storm-server/src/main/java/org/apache/storm/security/auth/workertoken/WorkerTokenManager.java +++ b/storm-server/src/main/java/org/apache/storm/security/auth/workertoken/WorkerTokenManager.java @@ -45,7 +45,8 @@ public class WorkerTokenManager { private static final Logger LOG = LoggerFactory.getLogger(WorkerTokenManager.class); /** - * The length of the random keys to use in bits. This should be at least the length of WorkerTokenSigner.DEFAULT_HMAC_ALGORITHM. + * The length of the random keys to use in bits. This should be at least the length of + * WorkerTokenSigner.DEFAULT_HMAC_ALGORITHM. */ private static final int KEY_LENGTH = 256; /** @@ -56,7 +57,8 @@ public class WorkerTokenManager { private final long tokenLifetimeMillis; /** - * Constructor. This assumes that state can store the tokens securely, and that they should be enabled at all. Please use + * Constructor. This assumes that state can store the tokens securely, and that they should be + * enabled at all. Please use * ClientAuthUtils.areWorkerTokensEnabledServer to validate this first. * * @param daemonConf the config for nimbus. @@ -68,10 +70,12 @@ public WorkerTokenManager(Map daemonConf, IStormClusterState sta keyGen = KeyGenerator.getInstance(WorkerTokenSigner.DEFAULT_HMAC_ALGORITHM); keyGen.init(KEY_LENGTH); } catch (NoSuchAlgorithmException nsa) { - throw new IllegalArgumentException("Can't find " + WorkerTokenSigner.DEFAULT_HMAC_ALGORITHM + " algorithm."); + throw new IllegalArgumentException("Can't find " + + WorkerTokenSigner.DEFAULT_HMAC_ALGORITHM + " algorithm."); } this.tokenLifetimeMillis = TimeUnit.MILLISECONDS.convert( - ObjectReader.getLong(daemonConf.get(DaemonConfig.STORM_WORKER_TOKEN_LIFE_TIME_HOURS), 24L), + ObjectReader.getLong(daemonConf.get(DaemonConfig.STORM_WORKER_TOKEN_LIFE_TIME_HOURS), + 24L), TimeUnit.HOURS); } @@ -89,7 +93,8 @@ protected SecretKey generateSecret() { } /** - * Get the secret that should be used to sign a token. This may either reuse a secret or generate a new one so any user should call + * Get the secret that should be used to sign a token. This may either reuse a secret or + * generate a new one so any user should call * this once and save the result. * * @return the key to use. @@ -106,17 +111,22 @@ protected SecretKey getCurrentSecret() { * @param topologyId the topology the token is for * @return a newly generated token that should be good to start using form now until it expires. */ - public WorkerToken createOrUpdateTokenFor(WorkerTokenServiceType serviceType, String user, String topologyId) { + public WorkerToken createOrUpdateTokenFor(WorkerTokenServiceType serviceType, String user, + String topologyId) { long nextVersion = state.getNextPrivateWorkerKeyVersion(serviceType, topologyId); SecretKey topoSecret = getCurrentSecret(); long expirationTimeMillis = Time.currentTimeMillis() + tokenLifetimeMillis; - WorkerTokenInfo info = new WorkerTokenInfo(user, topologyId, nextVersion, expirationTimeMillis); + WorkerTokenInfo info = new WorkerTokenInfo(user, topologyId, nextVersion, + expirationTimeMillis); byte[] serializedInfo = ClientAuthUtils.serializeWorkerTokenInfo(info); byte[] signature = WorkerTokenSigner.createPassword(serializedInfo, topoSecret); - WorkerToken ret = new WorkerToken(serviceType, ByteBuffer.wrap(serializedInfo), ByteBuffer.wrap(signature)); - PrivateWorkerKey key = new PrivateWorkerKey(ByteBuffer.wrap(topoSecret.getEncoded()), user, expirationTimeMillis); + WorkerToken ret = new WorkerToken(serviceType, ByteBuffer.wrap(serializedInfo), ByteBuffer + .wrap(signature)); + PrivateWorkerKey key = new PrivateWorkerKey(ByteBuffer.wrap(topoSecret.getEncoded()), user, + expirationTimeMillis); state.addPrivateWorkerKey(serviceType, topologyId, nextVersion, key); - LOG.info("Created new WorkerToken for user {} topology {} on service {}", user, topologyId, serviceType); + LOG.info("Created new WorkerToken for user {} topology {} on service {}", user, topologyId, + serviceType); return ret; } @@ -127,10 +137,12 @@ public WorkerToken createOrUpdateTokenFor(WorkerTokenServiceType serviceType, St * @param user the user the credentials are for * @param topologyId the topology the credentials are for */ - public void upsertWorkerTokensInCredsForTopo(Map creds, String user, String topologyId) { + public void upsertWorkerTokensInCredsForTopo(Map creds, String user, + String topologyId) { Arrays.stream(WorkerTokenServiceType.values()) .filter(type -> shouldRenewWorkerToken(creds, type)) - .forEach(type -> ClientAuthUtils.setWorkerToken(creds, createOrUpdateTokenFor(type, user, topologyId))); + .forEach(type -> ClientAuthUtils.setWorkerToken(creds, createOrUpdateTokenFor(type, + user, topologyId))); } @VisibleForTesting @@ -140,14 +152,17 @@ public boolean shouldRenewWorkerToken(Map creds, WorkerTokenServ if (oldToken != null) { try { WorkerTokenInfo info = ClientAuthUtils.getWorkerTokenInfo(oldToken); - if (!info.is_set_expirationTimeMillis() || info.get_expirationTimeMillis() - Time.currentTimeMillis() > (tokenLifetimeMillis + if (!info.is_set_expirationTimeMillis() || info.get_expirationTimeMillis() - Time + .currentTimeMillis() > (tokenLifetimeMillis / 2)) { - //Found an existing token and it is not going to expire any time soon, so don't bother adding in a new + // Found an existing token and it is not going to expire any time soon, so don't + // bother adding in a new // token. shouldAdd = false; } } catch (Exception e) { - //The old token could not be deserialized. This is bad, but we are going to replace it anyways so just keep going. + // The old token could not be deserialized. This is bad, but we are going to replace + // it anyways so just keep going. LOG.error("Could not deserialize token info", e); } } diff --git a/storm-server/src/main/java/org/apache/storm/stats/StatsUtil.java b/storm-server/src/main/java/org/apache/storm/stats/StatsUtil.java index 095b6d2ac44..a6cb17679e2 100644 --- a/storm-server/src/main/java/org/apache/storm/stats/StatsUtil.java +++ b/storm-server/src/main/java/org/apache/storm/stats/StatsUtil.java @@ -24,8 +24,8 @@ import java.util.HashSet; import java.util.Iterator; import java.util.List; -import java.util.Map; import java.util.Map.Entry; +import java.util.Map; import java.util.Set; import org.apache.storm.cluster.ExecutorBeat; import org.apache.storm.cluster.IStormClusterState; @@ -106,8 +106,8 @@ public class StatsUtil { private static final String CID_SID_TO_IN_STATS = "cid+sid->input-stats"; private static final String WORKERS_SET = "workers-set"; private static final ToStringTransformer TO_STRING = new ToStringTransformer(); - private static final FromGlobalStreamIdTransformer FROM_GSID = new FromGlobalStreamIdTransformer(); - + private static final FromGlobalStreamIdTransformer FROM_GSID = + new FromGlobalStreamIdTransformer(); // aggregation stats methods @@ -130,7 +130,7 @@ public static Map aggBoltLatAndCount(Map, Double> i } /** - * aggregate number acked and complete latencies across all streams. + * Aggregate number acked and complete latencies across all streams. */ public static Map aggSpoutLatAndCount(Map id2compAvg, Map id2numAcked) { @@ -142,7 +142,7 @@ public static Map aggSpoutLatAndCount(Map id2com } /** - * aggregate number executed and process & execute latencies. + * Aggregate number executed and process & execute latencies. */ public static Map aggBoltStreamsLatAndCount(Map id2execAvg, Map id2procAvg, @@ -182,7 +182,8 @@ public static Map aggSpoutStreamsLatAndCount(Map id2compA } /** - * pre-merge component page bolt stats from an executor heartbeat 1. computes component capacity 2. converts map keys of stats 3. + * pre-merge component page bolt stats from an executor heartbeat 1. computes component capacity + * 2. converts map keys of stats 3. * filters streams if necessary * * @param beat executor heartbeat data @@ -190,7 +191,8 @@ public static Map aggSpoutStreamsLatAndCount(Map id2compA * @param includeSys whether to include system streams * @return per-merged stats */ - public static Map aggPreMergeCompPageBolt(Map beat, String window, boolean includeSys) { + public static Map aggPreMergeCompPageBolt(Map beat, + String window, boolean includeSys) { Map ret = new HashMap<>(); ret.put(EXECUTOR_ID, beat.get("exec-id")); @@ -201,12 +203,15 @@ public static Map aggPreMergeCompPageBolt(Map be ret.put(NUM_TASKS, beat.get(NUM_TASKS)); Map stat2win2sid2num = ClientStatsUtil.getMapByKey(beat, STATS); - ret.put(CAPACITY, computeAggCapacity(stat2win2sid2num, getByKeyOr0(beat, ClientStatsUtil.UPTIME).intValue())); + ret.put(CAPACITY, computeAggCapacity(stat2win2sid2num, getByKeyOr0(beat, + ClientStatsUtil.UPTIME).intValue())); // calc cid+sid->input_stats Map inputStats = new HashMap(); - Map sid2acked = (Map) windowSetConverter(ClientStatsUtil.getMapByKey(stat2win2sid2num, ACKED), TO_STRING).get(window); - Map sid2failed = (Map) windowSetConverter(ClientStatsUtil.getMapByKey(stat2win2sid2num, FAILED), TO_STRING).get(window); + Map sid2acked = (Map) windowSetConverter(ClientStatsUtil.getMapByKey(stat2win2sid2num, + ACKED), TO_STRING).get(window); + Map sid2failed = (Map) windowSetConverter(ClientStatsUtil.getMapByKey(stat2win2sid2num, + FAILED), TO_STRING).get(window); Object v1 = sid2acked != null ? sid2acked : new HashMap(); inputStats.put(ACKED, v1); Object v = sid2failed != null ? sid2failed : new HashMap(); @@ -214,16 +219,21 @@ public static Map aggPreMergeCompPageBolt(Map be inputStats = swapMapOrder(inputStats); - Map sid2execLat = (Map) windowSetConverter(ClientStatsUtil.getMapByKey(stat2win2sid2num, EXEC_LATENCIES), TO_STRING).get(window); - Map sid2procLat = (Map) windowSetConverter(ClientStatsUtil.getMapByKey(stat2win2sid2num, PROC_LATENCIES), TO_STRING).get(window); - Map sid2exec = (Map) windowSetConverter(ClientStatsUtil.getMapByKey(stat2win2sid2num, EXECUTED), TO_STRING).get(window); + Map sid2execLat = (Map) windowSetConverter(ClientStatsUtil.getMapByKey(stat2win2sid2num, + EXEC_LATENCIES), TO_STRING).get(window); + Map sid2procLat = (Map) windowSetConverter(ClientStatsUtil.getMapByKey(stat2win2sid2num, + PROC_LATENCIES), TO_STRING).get(window); + Map sid2exec = (Map) windowSetConverter(ClientStatsUtil.getMapByKey(stat2win2sid2num, + EXECUTED), TO_STRING).get(window); mergeMaps(inputStats, aggBoltStreamsLatAndCount(sid2execLat, sid2procLat, sid2exec)); ret.put(CID_SID_TO_IN_STATS, inputStats); // calc sid->output_stats Map outputStats = new HashMap(); - Map sid2emitted = (Map) windowSetConverter(ClientStatsUtil.getMapByKey(stat2win2sid2num, EMITTED), TO_STRING).get(window); - Map sid2transferred = (Map) windowSetConverter(ClientStatsUtil.getMapByKey(stat2win2sid2num, TRANSFERRED), TO_STRING).get(window); + Map sid2emitted = (Map) windowSetConverter(ClientStatsUtil.getMapByKey(stat2win2sid2num, + EMITTED), TO_STRING).get(window); + Map sid2transferred = (Map) windowSetConverter(ClientStatsUtil.getMapByKey(stat2win2sid2num, + TRANSFERRED), TO_STRING).get(window); if (sid2emitted != null) { outputStats.put(EMITTED, filterSysStreams2Stat(sid2emitted, includeSys)); } else { @@ -241,7 +251,8 @@ public static Map aggPreMergeCompPageBolt(Map be } /** - * pre-merge component page spout stats from an executor heartbeat 1. computes component capacity 2. converts map keys of stats 3. + * pre-merge component page spout stats from an executor heartbeat 1. computes component + * capacity 2. converts map keys of stats 3. * filters streams if necessary * * @param beat executor heartbeat data @@ -249,7 +260,8 @@ public static Map aggPreMergeCompPageBolt(Map be * @param includeSys whether to include system streams * @return per-merged stats */ - public static Map aggPreMergeCompPageSpout(Map beat, String window, boolean includeSys) { + public static Map aggPreMergeCompPageSpout(Map beat, + String window, boolean includeSys) { Map ret = new HashMap<>(); ret.put(EXECUTOR_ID, beat.get("exec-id")); ret.put(HOST, beat.get(HOST)); @@ -262,9 +274,12 @@ public static Map aggPreMergeCompPageSpout(Map b // calc sid->output-stats Map outputStats = new HashMap(); - Map win2sid2acked = windowSetConverter(ClientStatsUtil.getMapByKey(stat2win2sid2num, ACKED), TO_STRING); - Map win2sid2failed = windowSetConverter(ClientStatsUtil.getMapByKey(stat2win2sid2num, FAILED), TO_STRING); - Map win2sid2emitted = windowSetConverter(ClientStatsUtil.getMapByKey(stat2win2sid2num, EMITTED), TO_STRING); + Map win2sid2acked = windowSetConverter(ClientStatsUtil.getMapByKey(stat2win2sid2num, ACKED), + TO_STRING); + Map win2sid2failed = windowSetConverter(ClientStatsUtil.getMapByKey(stat2win2sid2num, + FAILED), TO_STRING); + Map win2sid2emitted = windowSetConverter(ClientStatsUtil.getMapByKey(stat2win2sid2num, + EMITTED), TO_STRING); outputStats.put(ACKED, win2sid2acked.get(window)); outputStats.put(FAILED, win2sid2failed.get(window)); @@ -274,7 +289,8 @@ public static Map aggPreMergeCompPageSpout(Map b } outputStats.put(EMITTED, filterSysStreams2Stat(sid2emitted, includeSys)); - Map win2sid2transferred = windowSetConverter(ClientStatsUtil.getMapByKey(stat2win2sid2num, TRANSFERRED), TO_STRING); + Map win2sid2transferred = windowSetConverter(ClientStatsUtil.getMapByKey(stat2win2sid2num, + TRANSFERRED), TO_STRING); Map sid2transferred = (Map) win2sid2transferred.get(window); if (sid2transferred == null) { sid2transferred = new HashMap<>(); @@ -282,7 +298,8 @@ public static Map aggPreMergeCompPageSpout(Map b outputStats.put(TRANSFERRED, filterSysStreams2Stat(sid2transferred, includeSys)); outputStats = swapMapOrder(outputStats); - Map win2sid2compLat = windowSetConverter(ClientStatsUtil.getMapByKey(stat2win2sid2num, COMP_LATENCIES), TO_STRING); + Map win2sid2compLat = windowSetConverter(ClientStatsUtil.getMapByKey(stat2win2sid2num, + COMP_LATENCIES), TO_STRING); Map sid2compLat = (Map) win2sid2compLat.get(window); Map sid2acked = (Map) win2sid2acked.get(window); mergeMaps(outputStats, aggSpoutStreamsLatAndCount(sid2compLat, sid2acked)); @@ -306,10 +323,12 @@ public static Map aggPreMergeTopoPageBolt( subRet.put(NUM_TASKS, beat.get(NUM_TASKS)); Map stat2win2sid2num = ClientStatsUtil.getMapByKey(beat, STATS); - subRet.put(CAPACITY, computeAggCapacity(stat2win2sid2num, getByKeyOr0(beat, ClientStatsUtil.UPTIME).intValue())); + subRet.put(CAPACITY, computeAggCapacity(stat2win2sid2num, getByKeyOr0(beat, + ClientStatsUtil.UPTIME).intValue())); for (String key : new String[]{ EMITTED, TRANSFERRED, ACKED, FAILED }) { - Map> stat = windowSetConverter(ClientStatsUtil.getMapByKey(stat2win2sid2num, key), TO_STRING); + Map> stat = windowSetConverter(ClientStatsUtil + .getMapByKey(stat2win2sid2num, key), TO_STRING); if (EMITTED.equals(key) || TRANSFERRED.equals(key)) { stat = filterSysStreams(stat, includeSys); } @@ -324,9 +343,11 @@ public static Map aggPreMergeTopoPageBolt( } Map, Double>> win2sid2execLat = - windowSetConverter(ClientStatsUtil.getMapByKey(stat2win2sid2num, EXEC_LATENCIES), TO_STRING); + windowSetConverter(ClientStatsUtil.getMapByKey(stat2win2sid2num, EXEC_LATENCIES), + TO_STRING); Map, Double>> win2sid2procLat = - windowSetConverter(ClientStatsUtil.getMapByKey(stat2win2sid2num, PROC_LATENCIES), TO_STRING); + windowSetConverter(ClientStatsUtil.getMapByKey(stat2win2sid2num, PROC_LATENCIES), + TO_STRING); Map, Long>> win2sid2exec = windowSetConverter(ClientStatsUtil.getMapByKey(stat2win2sid2num, EXECUTED), TO_STRING); subRet.putAll(aggBoltLatAndCount( @@ -347,7 +368,8 @@ public static Map aggPreMergeTopoPageSpout subRet.put(NUM_TASKS, m.get(NUM_TASKS)); // no capacity for spout - Map>> stat2win2sid2num = ClientStatsUtil.getMapByKey(m, STATS); + Map>> stat2win2sid2num = ClientStatsUtil.getMapByKey(m, + STATS); for (String key : new String[]{ EMITTED, TRANSFERRED, FAILED }) { Map> stat = windowSetConverter(stat2win2sid2num.get(key), TO_STRING); if (EMITTED.equals(key) || TRANSFERRED.equals(key)) { @@ -364,7 +386,8 @@ public static Map aggPreMergeTopoPageSpout } Map> win2sid2compLat = - windowSetConverter(ClientStatsUtil.getMapByKey(stat2win2sid2num, COMP_LATENCIES), TO_STRING); + windowSetConverter(ClientStatsUtil.getMapByKey(stat2win2sid2num, COMP_LATENCIES), + TO_STRING); Map> win2sid2acked = windowSetConverter(ClientStatsUtil.getMapByKey(stat2win2sid2num, ACKED), TO_STRING); subRet.putAll(aggSpoutLatAndCount(win2sid2compLat.get(window), win2sid2acked.get(window))); @@ -375,7 +398,7 @@ public static Map aggPreMergeTopoPageSpout } /** - * merge accumulated bolt stats with pre-merged component stats. + * Merge accumulated bolt stats with pre-merged component stats. * * @param accBoltStats accumulated bolt stats * @param boltStats pre-merged component stats @@ -385,10 +408,14 @@ public static Map mergeAggCompStatsCompPageBolt( Map accBoltStats, Map boltStats) { Map ret = new HashMap<>(); - Map, Map> accIn = ClientStatsUtil.getMapByKey(accBoltStats, CID_SID_TO_IN_STATS); - Map> accOut = ClientStatsUtil.getMapByKey(accBoltStats, SID_TO_OUT_STATS); - Map, Map> boltIn = ClientStatsUtil.getMapByKey(boltStats, CID_SID_TO_IN_STATS); - Map> boltOut = ClientStatsUtil.getMapByKey(boltStats, SID_TO_OUT_STATS); + Map, Map> accIn = ClientStatsUtil.getMapByKey(accBoltStats, + CID_SID_TO_IN_STATS); + Map> accOut = ClientStatsUtil.getMapByKey(accBoltStats, + SID_TO_OUT_STATS); + Map, Map> boltIn = ClientStatsUtil.getMapByKey(boltStats, + CID_SID_TO_IN_STATS); + Map> boltOut = ClientStatsUtil.getMapByKey(boltStats, + SID_TO_OUT_STATS); int numExecutors = getByKeyOr0(accBoltStats, NUM_EXECUTORS).intValue(); ret.put(NUM_EXECUTORS, numExecutors + 1); @@ -397,7 +424,8 @@ public static Map mergeAggCompStatsCompPageBolt( // (merge-with (partial merge-with sum-or-0) acc-out spout-out) ret.put(SID_TO_OUT_STATS, fullMergeWithSum(accOut, boltOut)); - // {component id -> metric -> value}, note that input may contain both long and double values + // {component id -> metric -> value}, note that input may contain both long and double + // values ret.put(CID_SID_TO_IN_STATS, fullMergeWithSum(accIn, boltIn)); long executed = sumStreamsLong(boltIn, EXECUTED); @@ -431,15 +459,18 @@ public static Map mergeAggCompStatsCompPageBolt( } /** - * merge accumulated bolt stats with pre-merged component stats. + * Merge accumulated bolt stats with pre-merged component stats. */ public static Map mergeAggCompStatsCompPageSpout( Map accSpoutStats, Map spoutStats) { Map ret = new HashMap<>(); - // {stream id -> metric -> value}, note that sid->out-stats may contain both long and double values - Map> accOut = ClientStatsUtil.getMapByKey(accSpoutStats, SID_TO_OUT_STATS); - Map> spoutOut = ClientStatsUtil.getMapByKey(spoutStats, SID_TO_OUT_STATS); + // {stream id -> metric -> value}, note that sid->out-stats may contain both long and double + // values + Map> accOut = ClientStatsUtil.getMapByKey(accSpoutStats, + SID_TO_OUT_STATS); + Map> spoutOut = ClientStatsUtil.getMapByKey(spoutStats, + SID_TO_OUT_STATS); int numExecutors = getByKeyOr0(accSpoutStats, NUM_EXECUTORS).intValue(); ret.put(NUM_EXECUTORS, numExecutors + 1); @@ -471,52 +502,67 @@ public static Map mergeAggCompStatsCompPageSpout( } /** - * merge accumulated bolt stats with new bolt stats. + * Merge accumulated bolt stats with new bolt stats. * * @param accBoltStats accumulated bolt stats * @param boltStats new input bolt stats * @return merged bolt stats */ - public static Map mergeAggCompStatsTopoPageBolt(Map accBoltStats, + public static Map mergeAggCompStatsTopoPageBolt(Map accBoltStats, Map boltStats) { Map ret = new HashMap<>(); Integer numExecutors = getByKeyOr0(accBoltStats, NUM_EXECUTORS).intValue(); ret.put(NUM_EXECUTORS, numExecutors + 1); - ret.put(NUM_TASKS, sumOr0(getByKeyOr0(accBoltStats, NUM_TASKS), getByKeyOr0(boltStats, NUM_TASKS))); - ret.put(EMITTED, sumOr0(getByKeyOr0(accBoltStats, EMITTED), getByKeyOr0(boltStats, EMITTED))); - ret.put(TRANSFERRED, sumOr0(getByKeyOr0(accBoltStats, TRANSFERRED), getByKeyOr0(boltStats, TRANSFERRED))); - ret.put(EXEC_LAT_TOTAL, sumOr0(getByKeyOr0(accBoltStats, EXEC_LAT_TOTAL), getByKeyOr0(boltStats, EXEC_LAT_TOTAL))); - ret.put(PROC_LAT_TOTAL, sumOr0(getByKeyOr0(accBoltStats, PROC_LAT_TOTAL), getByKeyOr0(boltStats, PROC_LAT_TOTAL))); - ret.put(EXECUTED, sumOr0(getByKeyOr0(accBoltStats, EXECUTED), getByKeyOr0(boltStats, EXECUTED))); + ret.put(NUM_TASKS, sumOr0(getByKeyOr0(accBoltStats, NUM_TASKS), getByKeyOr0(boltStats, + NUM_TASKS))); + ret.put(EMITTED, sumOr0(getByKeyOr0(accBoltStats, EMITTED), getByKeyOr0(boltStats, + EMITTED))); + ret.put(TRANSFERRED, sumOr0(getByKeyOr0(accBoltStats, TRANSFERRED), getByKeyOr0(boltStats, + TRANSFERRED))); + ret.put(EXEC_LAT_TOTAL, sumOr0(getByKeyOr0(accBoltStats, EXEC_LAT_TOTAL), + getByKeyOr0(boltStats, EXEC_LAT_TOTAL))); + ret.put(PROC_LAT_TOTAL, sumOr0(getByKeyOr0(accBoltStats, PROC_LAT_TOTAL), + getByKeyOr0(boltStats, PROC_LAT_TOTAL))); + ret.put(EXECUTED, sumOr0(getByKeyOr0(accBoltStats, EXECUTED), getByKeyOr0(boltStats, + EXECUTED))); ret.put(ACKED, sumOr0(getByKeyOr0(accBoltStats, ACKED), getByKeyOr0(boltStats, ACKED))); ret.put(FAILED, sumOr0(getByKeyOr0(accBoltStats, FAILED), getByKeyOr0(boltStats, FAILED))); - ret.put(CAPACITY, maxOr0(getByKeyOr0(accBoltStats, CAPACITY), getByKeyOr0(boltStats, CAPACITY))); + ret.put(CAPACITY, maxOr0(getByKeyOr0(accBoltStats, CAPACITY), getByKeyOr0(boltStats, + CAPACITY))); return ret; } /** - * merge accumulated bolt stats with new bolt stats. + * Merge accumulated bolt stats with new bolt stats. */ - public static Map mergeAggCompStatsTopoPageSpout(Map accSpoutStats, + public static Map mergeAggCompStatsTopoPageSpout(Map accSpoutStats, Map spoutStats) { Map ret = new HashMap<>(); Integer numExecutors = getByKeyOr0(accSpoutStats, NUM_EXECUTORS).intValue(); ret.put(NUM_EXECUTORS, numExecutors + 1); - ret.put(NUM_TASKS, sumOr0(getByKeyOr0(accSpoutStats, NUM_TASKS), getByKeyOr0(spoutStats, NUM_TASKS))); - ret.put(EMITTED, sumOr0(getByKeyOr0(accSpoutStats, EMITTED), getByKeyOr0(spoutStats, EMITTED))); - ret.put(TRANSFERRED, sumOr0(getByKeyOr0(accSpoutStats, TRANSFERRED), getByKeyOr0(spoutStats, TRANSFERRED))); - ret.put(COMP_LAT_TOTAL, sumOr0(getByKeyOr0(accSpoutStats, COMP_LAT_TOTAL), getByKeyOr0(spoutStats, COMP_LAT_TOTAL))); + ret.put(NUM_TASKS, sumOr0(getByKeyOr0(accSpoutStats, NUM_TASKS), getByKeyOr0(spoutStats, + NUM_TASKS))); + ret.put(EMITTED, sumOr0(getByKeyOr0(accSpoutStats, EMITTED), getByKeyOr0(spoutStats, + EMITTED))); + ret.put(TRANSFERRED, sumOr0(getByKeyOr0(accSpoutStats, TRANSFERRED), getByKeyOr0(spoutStats, + TRANSFERRED))); + ret.put(COMP_LAT_TOTAL, sumOr0(getByKeyOr0(accSpoutStats, COMP_LAT_TOTAL), + getByKeyOr0(spoutStats, COMP_LAT_TOTAL))); ret.put(ACKED, sumOr0(getByKeyOr0(accSpoutStats, ACKED), getByKeyOr0(spoutStats, ACKED))); - ret.put(FAILED, sumOr0(getByKeyOr0(accSpoutStats, FAILED), getByKeyOr0(spoutStats, FAILED))); + ret.put(FAILED, sumOr0(getByKeyOr0(accSpoutStats, FAILED), getByKeyOr0(spoutStats, + FAILED))); return ret; } /** - * A helper function that does the common work to aggregate stats of one executor with the given map for the topology page. + * A helper function that does the common work to aggregate stats of one executor with the given + * map for the topology page. */ public static Map aggTopoExecStats( String window, boolean includeSys, Map accStats, Map beat, String compType) { @@ -576,20 +622,24 @@ public static Map aggTopoExecStats( Map win2failed = ClientStatsUtil.getMapByKey(accStats, WIN_TO_FAILED); Object v = isSpout - ? mergeWithSumLong(aggregateCountStreams(ClientStatsUtil.getMapByKey(stats, FAILED)), win2failed) : win2failed; + ? mergeWithSumLong(aggregateCountStreams(ClientStatsUtil.getMapByKey(stats, FAILED)), + win2failed) : win2failed; ret.put(WIN_TO_FAILED, v); ret.put(TYPE, stats.get(TYPE)); - // (merge-with merge-agg-comp-stats-topo-page-bolt/spout (acc-stats comp-key) cid->statk->num) + // (merge-with merge-agg-comp-stats-topo-page-bolt/spout (acc-stats comp-key) + // cid->statk->num) // (acc-stats comp-key) ==> bolt2stats/spout2stats if (isSpout) { for (String spout : cid2stats.keySet()) { - spout2stats.put(spout, mergeAggCompStatsTopoPageSpout((Map) spout2stats.get(spout), (Map) cid2stats.get(spout))); + spout2stats.put(spout, mergeAggCompStatsTopoPageSpout((Map) spout2stats.get(spout), + (Map) cid2stats.get(spout))); } } else { for (String bolt : cid2stats.keySet()) { - bolt2stats.put(bolt, mergeAggCompStatsTopoPageBolt((Map) bolt2stats.get(bolt), (Map) cid2stats.get(bolt))); + bolt2stats.put(bolt, mergeAggCompStatsTopoPageBolt((Map) bolt2stats.get(bolt), + (Map) cid2stats.get(bolt))); } } @@ -597,7 +647,7 @@ public static Map aggTopoExecStats( } /** - * aggregate topo executors stats. + * Aggregate topo executors stats. * * @param topologyId topology id * @param exec2nodePort executor -> host+port @@ -612,12 +662,15 @@ public static Map aggTopoExecStats( public static TopologyPageInfo aggTopoExecsStats( String topologyId, Map exec2nodePort, Map task2component, Map, Map> beats, StormTopology topology, String window, boolean includeSys, IStormClusterState clusterState) { - List> beatList = extractDataFromHb(exec2nodePort, task2component, beats, includeSys, topology); + List> beatList = extractDataFromHb(exec2nodePort, task2component, beats, + includeSys, topology); Map topoStats = aggregateTopoStats(window, includeSys, beatList); - return postAggregateTopoStats(task2component, exec2nodePort, topoStats, topologyId, clusterState); + return postAggregateTopoStats(task2component, exec2nodePort, topoStats, topologyId, + clusterState); } - private static Map aggregateTopoStats(String win, boolean includeSys, List> heartbeats) { + private static Map aggregateTopoStats(String win, boolean includeSys, + List> heartbeats) { Map initVal = new HashMap<>(); initVal.put(WORKERS_SET, new HashSet()); initVal.put(BOLT_TO_STATS, new HashMap()); @@ -636,7 +689,8 @@ private static Map aggregateTopoStats(String win, boolean includ return initVal; } - private static TopologyPageInfo postAggregateTopoStats(Map task2comp, Map exec2nodePort, Map accData, + private static TopologyPageInfo postAggregateTopoStats(Map task2comp, Map exec2nodePort, + Map accData, String topologyId, IStormClusterState clusterState) { TopologyPageInfo ret = new TopologyPageInfo(topologyId); @@ -683,10 +737,14 @@ private static TopologyPageInfo postAggregateTopoStats(Map task2comp, Map exec2n } TopologyStats topologyStats = new TopologyStats(); - topologyStats.set_window_to_acked(mapKeyStr(ClientStatsUtil.getMapByKey(accData, WIN_TO_ACKED))); - topologyStats.set_window_to_emitted(mapKeyStr(ClientStatsUtil.getMapByKey(accData, WIN_TO_EMITTED))); - topologyStats.set_window_to_failed(mapKeyStr(ClientStatsUtil.getMapByKey(accData, WIN_TO_FAILED))); - topologyStats.set_window_to_transferred(mapKeyStr(ClientStatsUtil.getMapByKey(accData, WIN_TO_TRANSFERRED))); + topologyStats.set_window_to_acked(mapKeyStr(ClientStatsUtil.getMapByKey(accData, + WIN_TO_ACKED))); + topologyStats.set_window_to_emitted(mapKeyStr(ClientStatsUtil.getMapByKey(accData, + WIN_TO_EMITTED))); + topologyStats.set_window_to_failed(mapKeyStr(ClientStatsUtil.getMapByKey(accData, + WIN_TO_FAILED))); + topologyStats.set_window_to_transferred(mapKeyStr(ClientStatsUtil.getMapByKey(accData, + WIN_TO_TRANSFERRED))); topologyStats.set_window_to_complete_latencies_ms(computeWeightedAveragesPerWindow( accData, WIN_TO_COMP_LAT_WGT_AVG, WIN_TO_ACKED)); @@ -698,13 +756,14 @@ private static TopologyPageInfo postAggregateTopoStats(Map task2comp, Map exec2n } /** - * aggregate bolt stats. + * Aggregate bolt stats. * * @param statsSeq a seq of ExecutorStats * @param includeSys whether to include system streams * @return aggregated bolt stats: {metric -> win -> global stream id -> value} */ - public static Map aggregateBoltStats(List statsSeq, boolean includeSys) { + public static Map aggregateBoltStats(List statsSeq, + boolean includeSys) { Map ret = new HashMap<>(); Map>> commonStats = aggregateCommonStats(statsSeq); @@ -735,13 +794,14 @@ public static Map aggregateBoltStats(List stat } /** - * aggregate spout stats. + * Aggregate spout stats. * * @param statsSeq a seq of ExecutorStats * @param includeSys whether to include system streams * @return aggregated spout stats: {metric -> win -> global stream id -> value} */ - public static Map aggregateSpoutStats(List statsSeq, boolean includeSys) { + public static Map aggregateSpoutStats(List statsSeq, + boolean includeSys) { // actually Map>> Map ret = new HashMap<>(); @@ -767,7 +827,7 @@ public static Map aggregateSpoutStats(List statsSe } /** - * aggregate common stats from a spout/bolt, called in aggregateSpoutStats/aggregateBoltStats. + * Aggregate common stats from a spout/bolt, called in aggregateSpoutStats/aggregateBoltStats. */ public static Map>> aggregateCommonStats(List statsSeq) { Map>> ret = new HashMap<>(); @@ -785,12 +845,13 @@ public static Map>> aggregateCommonStats(Li } /** - * filter system streams of aggregated spout/bolt stats if necessary. + * Filter system streams of aggregated spout/bolt stats if necessary. */ public static Map>> preProcessStreamSummary( Map>> streamSummary, boolean includeSys) { Map> emitted = ClientStatsUtil.getMapByKey(streamSummary, EMITTED); - Map> transferred = ClientStatsUtil.getMapByKey(streamSummary, TRANSFERRED); + Map> transferred = ClientStatsUtil.getMapByKey(streamSummary, + TRANSFERRED); ((Map) streamSummary).put(EMITTED, filterSysStreams(emitted, includeSys)); ((Map) streamSummary).put(TRANSFERRED, filterSysStreams(transferred, includeSys)); @@ -799,7 +860,7 @@ public static Map>> preProcessStreamSummary } /** - * aggregate count streams by window. + * Aggregate count streams by window. * * @param stats a Map of value: {win -> stream -> value} * @return a Map of value: {win -> value} @@ -819,13 +880,15 @@ public static Map aggregateCountStreams( } /** - * compute an weighted average from a list of average maps and a corresponding count maps extracted from a list of ExecutorSummary. + * Compute an weighted average from a list of average maps and a corresponding count maps + * extracted from a list of ExecutorSummary. * * @param avgSeq a list of {win -> global stream id -> avg value} * @param countSeq a list of {win -> global stream id -> count value} * @return a Map of {win -> global stream id -> weighted avg value} */ - public static Map> aggregateAverages(List>> avgSeq, + public static Map> aggregateAverages(List>> avgSeq, List>> countSeq) { Map> ret = new HashMap<>(); @@ -840,7 +903,8 @@ public static Map> aggregateAverages(List inner = entry.getValue(); for (K kk : inner.keySet()) { List vv = inner.get(kk); - tmp.put(kk, valAvg(((Number) vv.get(0)).doubleValue(), ((Number) vv.get(1)).longValue())); + tmp.put(kk, valAvg(((Number) vv.get(0)).doubleValue(), ((Number) vv.get(1)) + .longValue())); } ret.put(k, tmp); } @@ -849,7 +913,7 @@ public static Map> aggregateAverages(List stream -> average value} * @param counts a Map of {win -> stream -> count value} @@ -878,9 +942,10 @@ public static Map aggregateAvgStreams(Map win -> aggregated value}. + * Aggregates spout stream stats, returns a Map of {metric -> win -> aggregated value}. */ - public static Map spoutStreamsStats(List summs, boolean includeSys) { + public static Map spoutStreamsStats(List summs, + boolean includeSys) { if (summs == null) { return new HashMap<>(); } @@ -890,9 +955,10 @@ public static Map spoutStreamsStats(List summs, bo } /** - * aggregates bolt stream stats, returns a Map of {metric -> win -> aggregated value}. + * Aggregates bolt stream stats, returns a Map of {metric -> win -> aggregated value}. */ - public static Map boltStreamsStats(List summs, boolean includeSys) { + public static Map boltStreamsStats(List summs, + boolean includeSys) { if (summs == null) { return new HashMap<>(); } @@ -901,7 +967,7 @@ public static Map boltStreamsStats(List summs, boo } /** - * aggregate all spout streams. + * Aggregate all spout streams. * * @param stats a Map of {metric -> win -> stream id -> value} * @return a Map of {metric -> win -> aggregated value} @@ -911,15 +977,18 @@ public static Map aggregateSpoutStreams(Map stats) { Map ret = new HashMap<>(); ((Map) ret).put(ACKED, aggregateCountStreams(ClientStatsUtil.getMapByKey(stats, ACKED))); ((Map) ret).put(FAILED, aggregateCountStreams(ClientStatsUtil.getMapByKey(stats, FAILED))); - ((Map) ret).put(EMITTED, aggregateCountStreams(ClientStatsUtil.getMapByKey(stats, EMITTED))); - ((Map) ret).put(TRANSFERRED, aggregateCountStreams(ClientStatsUtil.getMapByKey(stats, TRANSFERRED))); + ((Map) ret).put(EMITTED, aggregateCountStreams(ClientStatsUtil.getMapByKey(stats, + EMITTED))); + ((Map) ret).put(TRANSFERRED, aggregateCountStreams(ClientStatsUtil.getMapByKey(stats, + TRANSFERRED))); ((Map) ret).put(COMP_LATENCIES, aggregateAvgStreams( - ClientStatsUtil.getMapByKey(stats, COMP_LATENCIES), ClientStatsUtil.getMapByKey(stats, ACKED))); + ClientStatsUtil.getMapByKey(stats, COMP_LATENCIES), ClientStatsUtil.getMapByKey(stats, + ACKED))); return ret; } /** - * aggregate all bolt streams. + * Aggregate all bolt streams. * * @param stats a Map of {metric -> win -> stream id -> value} * @return a Map of {metric -> win -> aggregated value} @@ -928,18 +997,23 @@ public static Map aggregateBoltStreams(Map stats) { Map ret = new HashMap<>(); ((Map) ret).put(ACKED, aggregateCountStreams(ClientStatsUtil.getMapByKey(stats, ACKED))); ((Map) ret).put(FAILED, aggregateCountStreams(ClientStatsUtil.getMapByKey(stats, FAILED))); - ((Map) ret).put(EMITTED, aggregateCountStreams(ClientStatsUtil.getMapByKey(stats, EMITTED))); - ((Map) ret).put(TRANSFERRED, aggregateCountStreams(ClientStatsUtil.getMapByKey(stats, TRANSFERRED))); - ((Map) ret).put(EXECUTED, aggregateCountStreams(ClientStatsUtil.getMapByKey(stats, EXECUTED))); + ((Map) ret).put(EMITTED, aggregateCountStreams(ClientStatsUtil.getMapByKey(stats, + EMITTED))); + ((Map) ret).put(TRANSFERRED, aggregateCountStreams(ClientStatsUtil.getMapByKey(stats, + TRANSFERRED))); + ((Map) ret).put(EXECUTED, aggregateCountStreams(ClientStatsUtil.getMapByKey(stats, + EXECUTED))); ((Map) ret).put(PROC_LATENCIES, aggregateAvgStreams( - ClientStatsUtil.getMapByKey(stats, PROC_LATENCIES), ClientStatsUtil.getMapByKey(stats, ACKED))); + ClientStatsUtil.getMapByKey(stats, PROC_LATENCIES), ClientStatsUtil.getMapByKey(stats, + ACKED))); ((Map) ret).put(EXEC_LATENCIES, aggregateAvgStreams( - ClientStatsUtil.getMapByKey(stats, EXEC_LATENCIES), ClientStatsUtil.getMapByKey(stats, EXECUTED))); + ClientStatsUtil.getMapByKey(stats, EXEC_LATENCIES), ClientStatsUtil.getMapByKey(stats, + EXECUTED))); return ret; } /** - * aggregate windowed stats from a bolt executor stats with a Map of accumulated stats. + * Aggregate windowed stats from a bolt executor stats with a Map of accumulated stats. */ public static Map aggBoltExecWinStats( Map accStats, Map newStats, boolean includeSys) { @@ -959,13 +1033,18 @@ public static Map aggBoltExecWinStats( Map win2executed = ClientStatsUtil.getMapByKey(m, EXECUTED); Map> emitted = ClientStatsUtil.getMapByKey(newStats, EMITTED); - Map win2emitted = mergeWithSumLong(aggregateCountStreams(filterSysStreams(emitted, includeSys)), - ClientStatsUtil.getMapByKey(accStats, WIN_TO_EMITTED)); + Map win2emitted = + mergeWithSumLong(aggregateCountStreams(filterSysStreams(emitted, includeSys)), + ClientStatsUtil.getMapByKey(accStats, + WIN_TO_EMITTED)); ret.put(WIN_TO_EMITTED, win2emitted); - Map> transferred = ClientStatsUtil.getMapByKey(newStats, TRANSFERRED); - Map win2transferred = mergeWithSumLong(aggregateCountStreams(filterSysStreams(transferred, includeSys)), - ClientStatsUtil.getMapByKey(accStats, WIN_TO_TRANSFERRED)); + Map> transferred = ClientStatsUtil.getMapByKey(newStats, + TRANSFERRED); + Map win2transferred = + mergeWithSumLong(aggregateCountStreams(filterSysStreams(transferred, includeSys)), + ClientStatsUtil.getMapByKey(accStats, + WIN_TO_TRANSFERRED)); ret.put(WIN_TO_TRANSFERRED, win2transferred); ret.put(WIN_TO_EXEC_LAT_WGT_AVG, mergeWithSumDouble( @@ -975,15 +1054,17 @@ public static Map aggBoltExecWinStats( ret.put(WIN_TO_EXECUTED, mergeWithSumLong( ClientStatsUtil.getMapByKey(accStats, WIN_TO_EXECUTED), win2executed)); ret.put(WIN_TO_ACKED, mergeWithSumLong( - aggregateCountStreams(ClientStatsUtil.getMapByKey(newStats, ACKED)), ClientStatsUtil.getMapByKey(accStats, WIN_TO_ACKED))); + aggregateCountStreams(ClientStatsUtil.getMapByKey(newStats, ACKED)), ClientStatsUtil + .getMapByKey(accStats, WIN_TO_ACKED))); ret.put(WIN_TO_FAILED, mergeWithSumLong( - aggregateCountStreams(ClientStatsUtil.getMapByKey(newStats, FAILED)), ClientStatsUtil.getMapByKey(accStats, WIN_TO_FAILED))); + aggregateCountStreams(ClientStatsUtil.getMapByKey(newStats, FAILED)), ClientStatsUtil + .getMapByKey(accStats, WIN_TO_FAILED))); return ret; } /** - * aggregate windowed stats from a spout executor stats with a Map of accumulated stats. + * Aggregate windowed stats from a spout executor stats with a Map of accumulated stats. */ public static Map aggSpoutExecWinStats( Map accStats, Map beat, boolean includeSys) { @@ -1001,13 +1082,17 @@ public static Map aggSpoutExecWinStats( Map win2acked = ClientStatsUtil.getMapByKey(m, ACKED); Map> emitted = ClientStatsUtil.getMapByKey(beat, EMITTED); - Map win2emitted = mergeWithSumLong(aggregateCountStreams(filterSysStreams(emitted, includeSys)), - ClientStatsUtil.getMapByKey(accStats, WIN_TO_EMITTED)); + Map win2emitted = + mergeWithSumLong(aggregateCountStreams(filterSysStreams(emitted, includeSys)), + ClientStatsUtil.getMapByKey(accStats, + WIN_TO_EMITTED)); ret.put(WIN_TO_EMITTED, win2emitted); Map> transferred = ClientStatsUtil.getMapByKey(beat, TRANSFERRED); - Map win2transferred = mergeWithSumLong(aggregateCountStreams(filterSysStreams(transferred, includeSys)), - ClientStatsUtil.getMapByKey(accStats, WIN_TO_TRANSFERRED)); + Map win2transferred = + mergeWithSumLong(aggregateCountStreams(filterSysStreams(transferred, includeSys)), + ClientStatsUtil.getMapByKey(accStats, + WIN_TO_TRANSFERRED)); ret.put(WIN_TO_TRANSFERRED, win2transferred); ret.put(WIN_TO_COMP_LAT_WGT_AVG, mergeWithSumDouble( @@ -1015,18 +1100,19 @@ public static Map aggSpoutExecWinStats( ret.put(WIN_TO_ACKED, mergeWithSumLong( ClientStatsUtil.getMapByKey(accStats, WIN_TO_ACKED), win2acked)); ret.put(WIN_TO_FAILED, mergeWithSumLong( - aggregateCountStreams(ClientStatsUtil.getMapByKey(beat, FAILED)), ClientStatsUtil.getMapByKey(accStats, WIN_TO_FAILED))); + aggregateCountStreams(ClientStatsUtil.getMapByKey(beat, FAILED)), ClientStatsUtil + .getMapByKey(accStats, WIN_TO_FAILED))); return ret; } - /** - * aggregate a list of count maps into one map. + * Aggregate a list of count maps into one map. * * @param countsSeq a seq of {win -> GlobalStreamId -> value} */ - public static Map> aggregateCounts(List>> countsSeq) { + public static Map> aggregateCounts(List>> countsSeq) { Map> ret = new HashMap<>(); for (Map> counts : countsSeq) { for (Map.Entry> entry : counts.entrySet()) { @@ -1088,19 +1174,23 @@ public static Map aggregateCompStats( } /** - * Combines the aggregate stats of one executor with the given map, selecting the appropriate window and including system components as + * Combines the aggregate stats of one executor with the given map, selecting the appropriate + * window and including system components as * specified. */ - public static Map aggCompExecStats(String window, boolean includeSys, Map accStats, + public static Map aggCompExecStats(String window, boolean includeSys, + Map accStats, Map beat, String compType) { Map ret = new HashMap<>(); if (ClientStatsUtil.SPOUT.equals(compType)) { - ret.putAll(aggSpoutExecWinStats(accStats, ClientStatsUtil.getMapByKey(beat, STATS), includeSys)); + ret.putAll(aggSpoutExecWinStats(accStats, ClientStatsUtil.getMapByKey(beat, STATS), + includeSys)); ret.put(STATS, mergeAggCompStatsCompPageSpout( ClientStatsUtil.getMapByKey(accStats, STATS), aggPreMergeCompPageSpout(beat, window, includeSys))); } else { - ret.putAll(aggBoltExecWinStats(accStats, ClientStatsUtil.getMapByKey(beat, STATS), includeSys)); + ret.putAll(aggBoltExecWinStats(accStats, ClientStatsUtil.getMapByKey(beat, STATS), + includeSys)); ret.put(STATS, mergeAggCompStatsCompPageBolt( ClientStatsUtil.getMapByKey(accStats, STATS), aggPreMergeCompPageBolt(beat, window, includeSys))); @@ -1111,7 +1201,8 @@ public static Map aggCompExecStats(String window, boolean includ } /** - * post aggregate component stats: 1. computes execute-latency/process-latency from execute/process latency total 2. computes windowed + * Post aggregate component stats: 1. computes execute-latency/process-latency from + * execute/process latency total 2. computes windowed * weight avgs 3. transform Map keys * * @param compStats accumulated comp stats @@ -1130,7 +1221,8 @@ public static Map postAggregateCompStats(Map com ret.put(NUM_EXECUTORS, numExecutors); ret.put(ClientStatsUtil.EXECUTOR_STATS, stats.get(ClientStatsUtil.EXECUTOR_STATS)); ret.put(WIN_TO_EMITTED, mapKeyStr(ClientStatsUtil.getMapByKey(compStats, WIN_TO_EMITTED))); - ret.put(WIN_TO_TRANSFERRED, mapKeyStr(ClientStatsUtil.getMapByKey(compStats, WIN_TO_TRANSFERRED))); + ret.put(WIN_TO_TRANSFERRED, mapKeyStr(ClientStatsUtil.getMapByKey(compStats, + WIN_TO_TRANSFERRED))); ret.put(WIN_TO_ACKED, mapKeyStr(ClientStatsUtil.getMapByKey(compStats, WIN_TO_ACKED))); ret.put(WIN_TO_FAILED, mapKeyStr(ClientStatsUtil.getMapByKey(compStats, WIN_TO_FAILED))); @@ -1158,7 +1250,8 @@ public static Map postAggregateCompStats(Map com ret.put(CID_SID_TO_IN_STATS, inStats2); ret.put(SID_TO_OUT_STATS, outStats); - ret.put(WIN_TO_EXECUTED, mapKeyStr(ClientStatsUtil.getMapByKey(compStats, WIN_TO_EXECUTED))); + ret.put(WIN_TO_EXECUTED, mapKeyStr(ClientStatsUtil.getMapByKey(compStats, + WIN_TO_EXECUTED))); ret.put(WIN_TO_EXEC_LAT, computeWeightedAveragesPerWindow( compStats, WIN_TO_EXEC_LAT_WGT_AVG, WIN_TO_EXECUTED)); ret.put(WIN_TO_PROC_LAT, computeWeightedAveragesPerWindow( @@ -1188,7 +1281,7 @@ public static Map postAggregateCompStats(Map com } /** - * aggregate component executor stats. + * Aggregate component executor stats. * * @param exec2hostPort a Map of {executor -> host+port} * @param task2component a Map of {task id -> component} @@ -1205,14 +1298,16 @@ public static ComponentPageInfo aggCompExecsStats( String window, boolean includeSys, String topologyId, StormTopology topology, String componentId) { List> beatList = - extractDataFromHb(exec2hostPort, task2component, beats, includeSys, topology, componentId); - Map compStats = aggregateCompStats(window, includeSys, beatList, componentType(topology, componentId)); + extractDataFromHb(exec2hostPort, task2component, beats, includeSys, topology, + componentId); + Map compStats = aggregateCompStats(window, includeSys, beatList, + componentType(topology, componentId)); compStats = postAggregateCompStats(compStats); return thriftifyCompPageData(topologyId, topology, componentId, compStats); } /** - * aggregate statistics per worker for a topology. Optionally filtering on specific supervisors + * Aggregate statistics per worker for a topology. Optionally filtering on specific supervisors * * @param stormId topology id * @param stormName storm topology @@ -1316,9 +1411,10 @@ public static List aggWorkerStats(String stormId, String stormNam // convert thrift stats to java maps /** - * convert thrift executor heartbeats into a java HashMap. + * Convert thrift executor heartbeats into a java HashMap. */ - public static Map, Map> convertExecutorBeats(Map beats) { + public static Map, Map> convertExecutorBeats(Map beats) { Map, Map> ret = new HashMap<>(); for (Map.Entry beat : beats.entrySet()) { ExecutorInfo executorInfo = beat.getKey(); @@ -1331,7 +1427,7 @@ public static Map, Map> convertExecutorBeats(Map, Map> convertWorkerBeats(SupervisorWorkerHeartbeat workerHeartbeat) { Map, Map> ret = new HashMap<>(); @@ -1346,7 +1442,7 @@ public static Map, Map> convertWorkerBeats(Supervi } /** - * convert thrift ExecutorBeat into a java HashMap. + * Convert thrift ExecutorBeat into a java HashMap. */ public static Map convertZkExecutorHb(ExecutorBeat beat) { Map ret = new HashMap<>(); @@ -1360,13 +1456,14 @@ public static Map convertZkExecutorHb(ExecutorBeat beat) { } /** - * convert a thrift worker heartbeat into a java HashMap. + * Convert a thrift worker heartbeat into a java HashMap. */ public static Map convertZkWorkerHb(ClusterWorkerHeartbeat workerHb) { Map ret = new HashMap<>(); if (workerHb != null) { ret.put("storm-id", workerHb.get_storm_id()); - ret.put(ClientStatsUtil.EXECUTOR_STATS, convertExecutorsStats(workerHb.get_executor_stats())); + ret.put(ClientStatsUtil.EXECUTOR_STATS, convertExecutorsStats(workerHb + .get_executor_stats())); ret.put(ClientStatsUtil.UPTIME, workerHb.get_uptime_secs()); ret.put(ClientStatsUtil.TIME_SECS, workerHb.get_time_secs()); } @@ -1374,9 +1471,10 @@ public static Map convertZkWorkerHb(ClusterWorkerHeartbeat worke } /** - * convert executors stats into a HashMap, note that ExecutorStats are remained unchanged. + * Convert executors stats into a HashMap, note that ExecutorStats are remained unchanged. */ - public static Map, ExecutorStats> convertExecutorsStats(Map stats) { + public static Map, ExecutorStats> convertExecutorsStats(Map stats) { Map, ExecutorStats> ret = new HashMap<>(); for (Map.Entry entry : stats.entrySet()) { ExecutorInfo executorInfo = entry.getKey(); @@ -1389,7 +1487,7 @@ public static Map, ExecutorStats> convertExecutorsStats(Map convertExecutorStats(ExecutorStats stats) { Map ret = new HashMap<>(); @@ -1421,11 +1519,16 @@ private static Map convertSpecificStats(SpoutStats stats) { private static Map convertSpecificStats(BoltStats stats) { Map ret = new HashMap<>(); - Map acked = ClientStatsUtil.windowSetConverter(stats.get_acked(), FROM_GSID, ClientStatsUtil.IDENTITY); - Map failed = ClientStatsUtil.windowSetConverter(stats.get_failed(), FROM_GSID, ClientStatsUtil.IDENTITY); - Map processAvg = ClientStatsUtil.windowSetConverter(stats.get_process_ms_avg(), FROM_GSID, ClientStatsUtil.IDENTITY); - Map executed = ClientStatsUtil.windowSetConverter(stats.get_executed(), FROM_GSID, ClientStatsUtil.IDENTITY); - Map executeAvg = ClientStatsUtil.windowSetConverter(stats.get_execute_ms_avg(), FROM_GSID, ClientStatsUtil.IDENTITY); + Map acked = ClientStatsUtil.windowSetConverter(stats.get_acked(), FROM_GSID, + ClientStatsUtil.IDENTITY); + Map failed = ClientStatsUtil.windowSetConverter(stats.get_failed(), FROM_GSID, + ClientStatsUtil.IDENTITY); + Map processAvg = ClientStatsUtil.windowSetConverter(stats.get_process_ms_avg(), FROM_GSID, + ClientStatsUtil.IDENTITY); + Map executed = ClientStatsUtil.windowSetConverter(stats.get_executed(), FROM_GSID, + ClientStatsUtil.IDENTITY); + Map executeAvg = ClientStatsUtil.windowSetConverter(stats.get_execute_ms_avg(), FROM_GSID, + ClientStatsUtil.IDENTITY); ret.put(ACKED, acked); ret.put(FAILED, failed); @@ -1437,7 +1540,7 @@ private static Map convertSpecificStats(BoltStats stats) { } /** - * extract a list of host port info for specified component. + * Extract a list of host port info for specified component. * * @param exec2hostPort {executor -> host+port} * @param task2component {task id -> component} @@ -1458,7 +1561,8 @@ public static List> extractNodeInfosFromHbForComp( String host = (String) value.get(0); Integer port = (Integer) value.get(1); String comp = task2component.get(start); - if ((compId == null || compId.equals(comp)) && (includeSys || !Utils.isSystemId(comp))) { + if ((compId == null || compId.equals(comp)) && (includeSys || !Utils + .isSystemId(comp))) { hostPorts.add(Lists.newArrayList(host, port)); } } @@ -1473,22 +1577,24 @@ public static List> extractNodeInfosFromHbForComp( return ret; } - // heartbeats related /** - * extracts a list of executor data from heart beats. + * Extracts a list of executor data from heart beats. */ - public static List> extractDataFromHb(Map executor2hostPort, Map task2component, + public static List> extractDataFromHb(Map executor2hostPort, + Map task2component, Map, Map> beats, boolean includeSys, StormTopology topology) { - return extractDataFromHb(executor2hostPort, task2component, beats, includeSys, topology, null); + return extractDataFromHb(executor2hostPort, task2component, beats, includeSys, topology, + null); } /** - * extracts a list of executor data from heart beats. + * Extracts a list of executor data from heart beats. */ - public static List> extractDataFromHb(Map executor2hostPort, Map task2component, + public static List> extractDataFromHb(Map executor2hostPort, + Map task2component, Map, Map> beats, boolean includeSys, StormTopology topology, String compId) { List> ret = new ArrayList<>(); @@ -1537,7 +1643,7 @@ public static List> extractDataFromHb(Map executor2hostPort, } /** - * compute weighted avg from a Map of stats and given avg/count keys. + * Compute weighted avg from a Map of stats and given avg/count keys. * * @param accData a Map of {win -> key -> value} * @param wgtAvgKey weighted average key @@ -1551,7 +1657,8 @@ private static Map computeWeightedAveragesPerWindow(Map 0) { ret.put(window.toString(), wgtAvg / divisor); } @@ -1560,7 +1667,7 @@ private static Map computeWeightedAveragesPerWindow(Map executorSumms) { /** * Compute the capacity of a executor. approximation of the % of time spent doing real work. + * * @param summary the stats for the executor. * @return the capacity of the executor. */ @@ -1586,7 +1694,8 @@ public static double computeExecutorCapacity(ExecutorSummary summary) { if (stats == null) { return 0.0; } else { - // actual value of m is: Map> ({win -> stream -> value}) + // actual value of m is: Map> ({win -> + // stream -> value}) Map m = aggregateBoltStats(Lists.newArrayList(summary), true); // {metric -> win -> value} ==> {win -> metric -> value} m = swapMapOrder(aggregateBoltStreams(m)); @@ -1605,7 +1714,7 @@ public static double computeExecutorCapacity(ExecutorSummary summary) { } /** - * filter ExecutorSummary whose stats is null. + * Filter ExecutorSummary whose stats is null. * * @param summs a list of ExecutorSummary * @return filtered summs @@ -1659,7 +1768,7 @@ private static double sumStreamsDouble(Map> m, String ke } /** - * same as clojure's (merge-with merge m1 m2). + * Same as clojure's (merge-with merge m1 m2). */ private static Map mergeMaps(Map m1, Map m2) { if (m2 == null) { @@ -1679,15 +1788,15 @@ private static Map mergeMaps(Map m1, Map m2) { return m1; } - /** - * filter system streams from stats. + * Filter system streams from stats. * * @param stream2stat { stream id -> value } * @param includeSys whether to filter system streams * @return filtered stats */ - private static Map filterSysStreams2Stat(Map stream2stat, boolean includeSys) { + private static Map filterSysStreams2Stat(Map stream2stat, + boolean includeSys) { LOG.trace("Filter Sys Streams2Stat {}", stream2stat); if (!includeSys) { for (Iterator itr = stream2stat.keySet().iterator(); itr.hasNext(); ) { @@ -1701,13 +1810,14 @@ private static Map filterSysStreams2Stat(Map stream2stat, boo } /** - * filter system streams from stats. + * Filter system streams from stats. * * @param stats { win -> stream id -> value } * @param includeSys whether to filter system streams * @return filtered stats */ - private static Map> filterSysStreams(Map> stats, boolean includeSys) { + private static Map> filterSysStreams(Map> stats, + boolean includeSys) { LOG.trace("Filter Sys Streams {}", stats); if (!includeSys) { for (Iterator itr = stats.keySet().iterator(); itr.hasNext(); ) { @@ -1725,7 +1835,7 @@ private static Map> filterSysStreams(Map Map> fullMergeWithSum(Map> m1, Map> m2) { @@ -1815,7 +1925,7 @@ private static Map mergeWithSumDouble(Map m1, Map Map> mergeWithAddPair(Map> m1, Map> m2) { @@ -1853,9 +1963,11 @@ private static Map> mergeWithAddPair(Map Map> mergeWithAddPair(Map executorId) { + public static SupervisorWorkerHeartbeat thriftifyRpcWorkerHb(String stormId, + List executorId) { SupervisorWorkerHeartbeat supervisorWorkerHeartbeat = new SupervisorWorkerHeartbeat(); supervisorWorkerHeartbeat.set_storm_id(stormId); supervisorWorkerHeartbeat - .set_executors(Collections.singletonList(new ExecutorInfo(executorId.get(0).intValue(), executorId.get(1).intValue()))); + .set_executors(Collections.singletonList(new ExecutorInfo(executorId.get(0).intValue(), + executorId.get(1).intValue()))); supervisorWorkerHeartbeat.set_time_secs(Time.currentTimeSecsLong()); return supervisorWorkerHeartbeat; } @@ -1912,7 +2026,8 @@ private static ComponentAggregateStats thriftifyBoltAggStats(Map m) { return stats; } - private static ExecutorAggregateStats thriftifyExecAggStats(String compId, String compType, Map m) { + private static ExecutorAggregateStats thriftifyExecAggStats(String compId, String compType, + Map m) { ExecutorSummary executorSummary = new ExecutorSummary(); List executor = (List) m.get(EXECUTOR_ID); executorSummary.set_executor_info(new ExecutorInfo(((Number) executor.get(0)).intValue(), @@ -1960,7 +2075,8 @@ private static Map thriftifyBoltInputStats(Map cidSid2inputStats) { return ret; } - private static ComponentAggregateStats thriftifyCommonAggStats(ComponentAggregateStats stats, Map m) { + private static ComponentAggregateStats thriftifyCommonAggStats(ComponentAggregateStats stats, + Map m) { CommonAggregateStats commonStats = new CommonAggregateStats(); commonStats.set_num_tasks(getByKeyOr0(m, NUM_TASKS).intValue()); commonStats.set_num_executors(getByKeyOr0(m, NUM_EXECUTORS).intValue()); @@ -2013,15 +2129,18 @@ private static ComponentPageInfo thriftifyCompPageData( } win2stats = tmp; gsid2inputStats = null; - sid2outputStats = thriftifySpoutOutputStats(ClientStatsUtil.getMapByKey(data, SID_TO_OUT_STATS)); + sid2outputStats = thriftifySpoutOutputStats(ClientStatsUtil.getMapByKey(data, + SID_TO_OUT_STATS)); } else { Map tmp = new HashMap(); for (Object k : win2stats.keySet()) { tmp.put(k, thriftifyBoltAggStats((Map) win2stats.get(k))); } win2stats = tmp; - gsid2inputStats = thriftifyBoltInputStats(ClientStatsUtil.getMapByKey(data, CID_SID_TO_IN_STATS)); - sid2outputStats = thriftifyBoltOutputStats(ClientStatsUtil.getMapByKey(data, SID_TO_OUT_STATS)); + gsid2inputStats = thriftifyBoltInputStats(ClientStatsUtil.getMapByKey(data, + CID_SID_TO_IN_STATS)); + sid2outputStats = thriftifyBoltOutputStats(ClientStatsUtil.getMapByKey(data, + SID_TO_OUT_STATS)); } ret.set_num_executors(getByKeyOr0(data, NUM_EXECUTORS).intValue()); ret.set_num_tasks(getByKeyOr0(data, NUM_TASKS).intValue()); @@ -2037,6 +2156,7 @@ private static ComponentPageInfo thriftifyCompPageData( /** * Convert Executor stats to thrift data structure. + * * @param stats the stats in the form of a map. * @return teh thrift structure for the stats. */ @@ -2045,8 +2165,10 @@ public static ExecutorStats thriftifyExecutorStats(Map stats) { ExecutorSpecificStats specificStats = thriftifySpecificStats(stats); ret.set_specific(specificStats); - ret.set_emitted(ClientStatsUtil.windowSetConverter(ClientStatsUtil.getMapByKey(stats, EMITTED), TO_STRING, TO_STRING)); - ret.set_transferred(ClientStatsUtil.windowSetConverter(ClientStatsUtil.getMapByKey(stats, TRANSFERRED), TO_STRING, TO_STRING)); + ret.set_emitted(ClientStatsUtil.windowSetConverter(ClientStatsUtil.getMapByKey(stats, + EMITTED), TO_STRING, TO_STRING)); + ret.set_transferred(ClientStatsUtil.windowSetConverter(ClientStatsUtil.getMapByKey(stats, + TRANSFERRED), TO_STRING, TO_STRING)); ret.set_rate(((Number) stats.get(RATE)).doubleValue()); return ret; @@ -2059,28 +2181,35 @@ private static ExecutorSpecificStats thriftifySpecificStats(Map stats) { if (ClientStatsUtil.BOLT.equals(compType)) { BoltStats boltStats = new BoltStats(); boltStats.set_acked( - ClientStatsUtil.windowSetConverter(ClientStatsUtil.getMapByKey(stats, ACKED), ClientStatsUtil.TO_GSID, TO_STRING)); + ClientStatsUtil.windowSetConverter(ClientStatsUtil.getMapByKey(stats, ACKED), + ClientStatsUtil.TO_GSID, TO_STRING)); boltStats.set_executed( - ClientStatsUtil.windowSetConverter(ClientStatsUtil.getMapByKey(stats, EXECUTED), ClientStatsUtil.TO_GSID, TO_STRING)); + ClientStatsUtil.windowSetConverter(ClientStatsUtil.getMapByKey(stats, EXECUTED), + ClientStatsUtil.TO_GSID, TO_STRING)); boltStats.set_execute_ms_avg( - ClientStatsUtil.windowSetConverter(ClientStatsUtil.getMapByKey(stats, EXEC_LATENCIES), ClientStatsUtil.TO_GSID, TO_STRING)); + ClientStatsUtil.windowSetConverter(ClientStatsUtil.getMapByKey(stats, + EXEC_LATENCIES), ClientStatsUtil.TO_GSID, TO_STRING)); boltStats.set_failed( - ClientStatsUtil.windowSetConverter(ClientStatsUtil.getMapByKey(stats, FAILED), ClientStatsUtil.TO_GSID, TO_STRING)); + ClientStatsUtil.windowSetConverter(ClientStatsUtil.getMapByKey(stats, FAILED), + ClientStatsUtil.TO_GSID, TO_STRING)); boltStats.set_process_ms_avg( - ClientStatsUtil.windowSetConverter(ClientStatsUtil.getMapByKey(stats, PROC_LATENCIES), ClientStatsUtil.TO_GSID, TO_STRING)); + ClientStatsUtil.windowSetConverter(ClientStatsUtil.getMapByKey(stats, + PROC_LATENCIES), ClientStatsUtil.TO_GSID, TO_STRING)); specificStats.set_bolt(boltStats); } else { SpoutStats spoutStats = new SpoutStats(); - spoutStats.set_acked(ClientStatsUtil.windowSetConverter(ClientStatsUtil.getMapByKey(stats, ACKED), TO_STRING, TO_STRING)); - spoutStats.set_failed(ClientStatsUtil.windowSetConverter(ClientStatsUtil.getMapByKey(stats, FAILED), TO_STRING, TO_STRING)); + spoutStats.set_acked(ClientStatsUtil.windowSetConverter(ClientStatsUtil + .getMapByKey(stats, ACKED), TO_STRING, TO_STRING)); + spoutStats.set_failed(ClientStatsUtil.windowSetConverter(ClientStatsUtil + .getMapByKey(stats, FAILED), TO_STRING, TO_STRING)); spoutStats.set_complete_ms_avg( - ClientStatsUtil.windowSetConverter(ClientStatsUtil.getMapByKey(stats, COMP_LATENCIES), TO_STRING, TO_STRING)); + ClientStatsUtil.windowSetConverter(ClientStatsUtil.getMapByKey(stats, + COMP_LATENCIES), TO_STRING, TO_STRING)); specificStats.set_spout(spoutStats); } return specificStats; } - // helper methods private static GlobalStreamId toGlobalStreamId(List list) { @@ -2161,7 +2290,8 @@ private static Number getByKeyOr0(Map m, String k) { return n; } - private static Double weightAvgAndSum(Map id2Avg, Map id2num) { + private static Double weightAvgAndSum(Map id2Avg, Map id2num) { double ret = 0; if (id2Avg == null || id2num == null) { return ret; @@ -2174,7 +2304,8 @@ private static Double weightAvgAndSum( return ret; } - private static double weightAvg(Map id2Avg, Map id2num, K key) { + private static double weightAvg(Map id2Avg, + Map id2num, K key) { if (id2Avg == null || id2num == null) { return 0.0; } @@ -2183,6 +2314,7 @@ private static double weightAvg(Map {:Y {:a :pear, :b :orange}, :X {:a :banana, :b :apple}}" + * For a nested map, rearrange data such that the top-level keys become the nested map's keys + * and vice versa. Example: {:a {:X :banana, + * :Y :pear}, :b {:X :apple, :Y :orange}} -> {:Y {:a :pear, :b :orange}, :X {:a :banana, :b + * :apple}}" */ private static Map swapMapOrder(Map m) { if (m.size() == 0) { @@ -2264,6 +2398,7 @@ private static Map swapMapOrder(Map m) { /** * Expand the count/average out into total, count. + * * @param avgs a HashMap of values: { win -> GlobalStreamId -> value } * @param counts a HashMap of values: { win -> GlobalStreamId -> value } * @return a HashMap of values: {win -> GlobalStreamId -> [cnt*avg, cnt]} @@ -2291,7 +2426,7 @@ private static Map> expandAverages(Map GlobalStreamId -> value}, ...] * @param countSeq list of counts like [{win -> GlobalStreamId -> value}, ...] @@ -2323,6 +2458,7 @@ private static double valAvg(double t, long c) { /** * Convert a float to a string for display. + * * @param n the value to format. * @return the string ready for display. */ @@ -2337,14 +2473,15 @@ public static String errorSubset(String errorStr) { return errorStr.substring(0, 200); } - private static ErrorInfo getLastError(IStormClusterState stormClusterState, String stormId, String compId) { + private static ErrorInfo getLastError(IStormClusterState stormClusterState, String stormId, + String compId) { return stormClusterState.lastError(stormId, compId); } - // key transformers - public static Map windowSetConverter(Map stats, ClientStatsUtil.KeyTransformer firstKeyFunc) { + public static Map windowSetConverter(Map stats, + ClientStatsUtil.KeyTransformer firstKeyFunc) { return ClientStatsUtil.windowSetConverter(stats, ClientStatsUtil.IDENTITY, firstKeyFunc); } diff --git a/storm-server/src/main/java/org/apache/storm/testing/CompleteTopologyParam.java b/storm-server/src/main/java/org/apache/storm/testing/CompleteTopologyParam.java index 5ad60fbd10d..3c9f66c0796 100644 --- a/storm-server/src/main/java/org/apache/storm/testing/CompleteTopologyParam.java +++ b/storm-server/src/main/java/org/apache/storm/testing/CompleteTopologyParam.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -25,7 +31,7 @@ public class CompleteTopologyParam { */ private MockedSources mockedSources = new MockedSources(); /** - * the config for the topology when it was submitted to the cluster. + * The config for the topology when it was submitted to the cluster. */ private Map topoConf = new Config(); /** @@ -33,12 +39,12 @@ public class CompleteTopologyParam { */ private boolean cleanupState = true; /** - * the topology name you want to submit to the cluster. + * The topology name you want to submit to the cluster. */ private String topologyName; /** - * the timeout of topology you want to submit to the cluster. + * The timeout of topology you want to submit to the cluster. */ private int timeoutMs = Testing.TEST_TIMEOUT_MS; diff --git a/storm-server/src/main/java/org/apache/storm/testing/InProcessZookeeper.java b/storm-server/src/main/java/org/apache/storm/testing/InProcessZookeeper.java index 65395177f23..37ce0fa08c2 100644 --- a/storm-server/src/main/java/org/apache/storm/testing/InProcessZookeeper.java +++ b/storm-server/src/main/java/org/apache/storm/testing/InProcessZookeeper.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -35,6 +41,7 @@ public InProcessZookeeper() throws Exception { /** * Get port. + * * @return the port ZK is listening on (localhost) */ public long getPort() { diff --git a/storm-server/src/main/java/org/apache/storm/testing/TestJob.java b/storm-server/src/main/java/org/apache/storm/testing/TestJob.java index fc636b582ec..50e405d8b12 100644 --- a/storm-server/src/main/java/org/apache/storm/testing/TestJob.java +++ b/storm-server/src/main/java/org/apache/storm/testing/TestJob.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -29,9 +35,10 @@ */ public interface TestJob { /** - * run the testing logic with the cluster. + * Run the testing logic with the cluster. * - * @param cluster the cluster which created by Testing.withSimulatedTimeLocalCluster + * @param cluster the cluster which created by + * Testing.withSimulatedTimeLocalCluster * and Testing.withTrackedCluster. */ void run(ILocalCluster cluster) throws Exception; diff --git a/storm-server/src/main/java/org/apache/storm/testing/TrackedTopology.java b/storm-server/src/main/java/org/apache/storm/testing/TrackedTopology.java index 4719ce618bc..aebce3a98ac 100644 --- a/storm-server/src/main/java/org/apache/storm/testing/TrackedTopology.java +++ b/storm-server/src/main/java/org/apache/storm/testing/TrackedTopology.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -45,6 +51,7 @@ public class TrackedTopology { /** * Create a new topology to be tracked. + * * @param origTopo the original topology. * @param cluster a cluster that should have been launched with tracking enabled. */ @@ -59,7 +66,8 @@ public TrackedTopology(StormTopology origTopo, ILocalCluster cluster) { bolt.set_bolt_object(Thrift.serializeComponentObject(new BoltTracker(obj, id))); } for (SpoutSpec spout : topology.get_spouts().values()) { - IRichSpout obj = (IRichSpout) Thrift.deserializeComponentObject(spout.get_spout_object()); + IRichSpout obj = (IRichSpout) Thrift.deserializeComponentObject(spout + .get_spout_object()); spout.set_spout_object(Thrift.serializeComponentObject(new SpoutTracker(obj, id))); } } @@ -67,7 +75,8 @@ public TrackedTopology(StormTopology origTopo, ILocalCluster cluster) { @SuppressWarnings("unchecked") private static int globalAmt(String id, String key) { LOG.warn("Reading tracked metrics for ID {}", id); - return ((ConcurrentHashMap) RegisteredGlobalState.getState(id)).get(key).get(); + return ((ConcurrentHashMap) RegisteredGlobalState.getState(id)) + .get(key).get(); } public StormTopology getTopology() { @@ -104,7 +113,8 @@ public void trackedWait(int amt, int timeoutMs) { int se = globalAmt(id, "spout-emitted"); int transferred = globalAmt(id, "transferred"); int processed = globalAmt(id, "processed"); - LOG.info("emitted {} target {} transferred {} processed {}", se, target, transferred, processed); + LOG.info("emitted {} target {} transferred {} processed {}", se, target, + transferred, processed); return (target != se) || (transferred != processed); }, () -> { @@ -120,6 +130,7 @@ public void trackedWait(int amt, int timeoutMs) { /** * Read a metric from the tracked cluster (NOT JUST THIS TOPOLOGY). + * * @param key one of "spout-emitted", "processed", or "transferred" * @return the amount of that metric */ diff --git a/storm-server/src/main/java/org/apache/storm/utils/BufferInputStream.java b/storm-server/src/main/java/org/apache/storm/utils/BufferInputStream.java index 5bf91ca5eb0..a430cd3e8cb 100644 --- a/storm-server/src/main/java/org/apache/storm/utils/BufferInputStream.java +++ b/storm-server/src/main/java/org/apache/storm/utils/BufferInputStream.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -16,7 +22,6 @@ import java.io.InputStream; import java.util.Arrays; - public class BufferInputStream implements AutoCloseable { byte[] buffer; InputStream stream; diff --git a/storm-server/src/main/java/org/apache/storm/utils/DaemonConfigValidation.java b/storm-server/src/main/java/org/apache/storm/utils/DaemonConfigValidation.java index a0312589493..5e299bcd66a 100644 --- a/storm-server/src/main/java/org/apache/storm/utils/DaemonConfigValidation.java +++ b/storm-server/src/main/java/org/apache/storm/utils/DaemonConfigValidation.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -51,7 +57,8 @@ public void validateField(String name, Object o) { } try { Map numaGenericResources = - (Map) numa.getOrDefault(NUMA_GENERIC_RESOURCES_MAP, Collections.EMPTY_MAP); + (Map) numa.getOrDefault(NUMA_GENERIC_RESOURCES_MAP, + Collections.EMPTY_MAP); } catch (Exception e) { throw new IllegalArgumentException( "Invalid generic resources in NUMA config" diff --git a/storm-server/src/main/java/org/apache/storm/utils/EnumUtil.java b/storm-server/src/main/java/org/apache/storm/utils/EnumUtil.java index 33ff149830e..573e837904f 100644 --- a/storm-server/src/main/java/org/apache/storm/utils/EnumUtil.java +++ b/storm-server/src/main/java/org/apache/storm/utils/EnumUtil.java @@ -1,15 +1,19 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. - * See the NOTICE file distributed with this work for additional information regarding copyright ownership. + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. + * See the NOTICE file distributed with this work for additional information regarding copyright + * ownership. * The ASF licenses this file to you under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. You may obtain a copy of the License at + * you may not use this file except in compliance with the License. You may obtain a copy of the + * License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, + *

      Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and limitations under the License. + * See the License for the specific language governing permissions and limitations under the + * License. */ package org.apache.storm.utils; @@ -24,11 +28,13 @@ public static String toMetricName(Enum type) { /** * Create an Enum map with given lambda mapper. + * * @param klass the Enum class * @param mapper The mapper producing value with key (enum constant) * @return An Enum map */ - public static , U> EnumMap toEnumMap(Class klass, Function mapper) { + public static , U> EnumMap toEnumMap(Class klass, Function mapper) { EnumMap map = new EnumMap<>(klass); for (T elem : klass.getEnumConstants()) { map.put(elem, mapper.apply(elem)); diff --git a/storm-server/src/main/java/org/apache/storm/utils/EquivalenceUtils.java b/storm-server/src/main/java/org/apache/storm/utils/EquivalenceUtils.java index 66be019e55f..ceca2485848 100644 --- a/storm-server/src/main/java/org/apache/storm/utils/EquivalenceUtils.java +++ b/storm-server/src/main/java/org/apache/storm/utils/EquivalenceUtils.java @@ -30,13 +30,15 @@ public class EquivalenceUtils { /** - * Decide the equivalence of two local assignments, ignoring the order of executors This is different from #equal method. + * Decide the equivalence of two local assignments, ignoring the order of executors This is + * different from #equal method. * * @param first Local assignment A * @param second Local assignment B * @return True if A and B are equivalent, ignoring the order of the executors */ - public static boolean areLocalAssignmentsEquivalent(LocalAssignment first, LocalAssignment second) { + public static boolean areLocalAssignmentsEquivalent(LocalAssignment first, + LocalAssignment second) { if (first == null && second == null) { return true; } @@ -62,7 +64,7 @@ public static boolean areLocalAssignmentsEquivalent(LocalAssignment first, Local } /** - * This method compares WorkerResources while considering any resources are NULL to be 0.0 + * This method compares WorkerResources while considering any resources are NULL to be 0.0. * * @param first WorkerResources A * @param second WorkerResources B @@ -101,20 +103,22 @@ static boolean customWorkerResourcesEquality(WorkerResources first, WorkerResour if (!customResourceMapEquality(first.get_resources(), second.get_resources())) { return false; } - if (!customResourceMapEquality(first.get_shared_resources(), second.get_shared_resources())) { + if (!customResourceMapEquality(first.get_shared_resources(), second + .get_shared_resources())) { return false; } return true; } /** - * This method compares Resource Maps while considering any resources are NULL to be 0.0 + * This method compares Resource Maps while considering any resources are NULL to be 0.0. * * @param firstMap Resource Map A * @param secondMap Resource Map B * @return True if A and B are equivalent, treating the absent resources as 0.0 */ - private static boolean customResourceMapEquality(Map firstMap, Map secondMap) { + private static boolean customResourceMapEquality(Map firstMap, Map secondMap) { if (firstMap == null && secondMap == null) { return true; } @@ -128,7 +132,8 @@ private static boolean customResourceMapEquality(Map firstMap, M Set keys = new HashSet<>(firstMap.keySet()); keys.addAll(secondMap.keySet()); for (String key : keys) { - if (firstMap.getOrDefault(key, 0.0).doubleValue() != secondMap.getOrDefault(key, 0.0).doubleValue()) { + if (firstMap.getOrDefault(key, 0.0).doubleValue() != secondMap.getOrDefault(key, 0.0) + .doubleValue()) { return false; } } diff --git a/storm-server/src/main/java/org/apache/storm/utils/LruMap.java b/storm-server/src/main/java/org/apache/storm/utils/LruMap.java index ea0ea544ca3..9a5fe5ca0bd 100644 --- a/storm-server/src/main/java/org/apache/storm/utils/LruMap.java +++ b/storm-server/src/main/java/org/apache/storm/utils/LruMap.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-server/src/main/java/org/apache/storm/utils/ServerConfigUtils.java b/storm-server/src/main/java/org/apache/storm/utils/ServerConfigUtils.java index a6b0ba6feb1..ee0f198c7e5 100644 --- a/storm-server/src/main/java/org/apache/storm/utils/ServerConfigUtils.java +++ b/storm-server/src/main/java/org/apache/storm/utils/ServerConfigUtils.java @@ -31,7 +31,6 @@ import org.apache.storm.Config; import org.apache.storm.DaemonConfig; - public class ServerConfigUtils { public static final String FILE_SEPARATOR = File.separator; public static final String NIMBUS_DO_NOT_REASSIGN = "NIMBUS-DO-NOT-REASSIGN"; @@ -42,7 +41,8 @@ public class ServerConfigUtils { private static ServerConfigUtils _instance = new ServerConfigUtils(); /** - * Provide an instance of this class for delegates to use. To mock out delegated methods, provide an instance of a subclass that + * Provide an instance of this class for delegates to use. To mock out delegated methods, + * provide an instance of a subclass that * overrides the implementation of the delegated method. * * @param u a ServerConfigUtils instance @@ -81,14 +81,16 @@ public static String masterStormDistRoot(Map conf) throws IOExce return ret; } - public static String masterStormDistRoot(Map conf, String stormId) throws IOException { + public static String masterStormDistRoot(Map conf, + String stormId) throws IOException { return (masterStormDistRoot(conf) + FILE_SEPARATOR + stormId); } /* TODO: make sure test these two functions in manual tests */ public static List getTopoLogsUsers(Map topologyConf) { List logsUsers = ObjectReader.getStrings(topologyConf.get(DaemonConfig.LOGS_USERS)); - List topologyUsers = ObjectReader.getStrings(topologyConf.get(Config.TOPOLOGY_USERS)); + List topologyUsers = ObjectReader.getStrings(topologyConf + .get(Config.TOPOLOGY_USERS)); Set mergedUsers = new HashSet<>(logsUsers); mergedUsers.addAll(topologyUsers); List ret = new ArrayList<>(mergedUsers); @@ -97,8 +99,10 @@ public static List getTopoLogsUsers(Map topologyConf) { } public static List getTopoLogsGroups(Map topologyConf) { - List logsGroups = ObjectReader.getStrings(topologyConf.get(DaemonConfig.LOGS_GROUPS)); - List topologyGroups = ObjectReader.getStrings(topologyConf.get(Config.TOPOLOGY_GROUPS)); + List logsGroups = ObjectReader.getStrings(topologyConf + .get(DaemonConfig.LOGS_GROUPS)); + List topologyGroups = ObjectReader.getStrings(topologyConf + .get(Config.TOPOLOGY_GROUPS)); Set mergedGroups = new HashSet<>(logsGroups); mergedGroups.addAll(topologyGroups); List ret = new ArrayList<>(mergedGroups); @@ -136,14 +140,16 @@ public static String absoluteHealthCheckDir(Map conf) { } public static File getLogMetaDataFile(String fname) { - String[] subStrings = fname.split(Pattern.quote(FILE_SEPARATOR)); // TODO: does this work well on windows? + String[] subStrings = fname.split(Pattern + .quote(FILE_SEPARATOR)); // TODO: does this work well on windows? String id = subStrings[0]; Integer port = Integer.parseInt(subStrings[1]); return getLogMetaDataFile(Utils.readStormConfig(), id, port); } public static File getLogMetaDataFile(Map conf, String id, Integer port) { - String fname = ConfigUtils.workerArtifactsRoot(conf, id, port) + FILE_SEPARATOR + "worker.yaml"; + String fname = ConfigUtils.workerArtifactsRoot(conf, id, port) + FILE_SEPARATOR + + "worker.yaml"; return new File(fname); } @@ -152,7 +158,8 @@ public static String masterStormJarPath(String stormRoot) { } public LocalState supervisorStateImpl(Map conf) throws IOException { - return new LocalState((ConfigUtils.supervisorLocalDir(conf) + FILE_SEPARATOR + "localstate"), true); + return new LocalState((ConfigUtils.supervisorLocalDir(conf) + FILE_SEPARATOR + + "localstate"), true); } public LocalState nimbusTopoHistoryStateImpl(Map conf) throws IOException { diff --git a/storm-server/src/main/java/org/apache/storm/utils/ServerUtils.java b/storm-server/src/main/java/org/apache/storm/utils/ServerUtils.java index 066334c83ea..430995aaa18 100644 --- a/storm-server/src/main/java/org/apache/storm/utils/ServerUtils.java +++ b/storm-server/src/main/java/org/apache/storm/utils/ServerUtils.java @@ -186,7 +186,8 @@ public static String shellCmd(List command) { } /** - * Takes an input dir or file and returns the disk usage on that local directory. Very basic implementation. + * Takes an input dir or file and returns the disk usage on that local directory. Very basic + * implementation. * * @param dir The input dir to get the disk space of this local dir * @return The total disk space of the input local directory @@ -233,7 +234,6 @@ public static String currentClasspath() { return _instance.currentClasspathImpl(); } - /** * Returns the current thread classloader. */ @@ -279,7 +279,8 @@ public static String scriptFilePath(String dir) { * * @param dir the directory under which the script is to be written * @param command the command the script is to execute - * @param environment optional environment variables to set before running the script's command. May be null. + * @param environment optional environment variables to set before running the script's command. + * May be null. * @return the path to the script that has been written */ public static String writeScript(String dir, List command, @@ -292,7 +293,8 @@ public static String writeScript(String dir, List command, * * @param dir the directory under which the script is to be written * @param command the command the script is to execute - * @param environment optional environment variables to set before running the script's command. May be null. + * @param environment optional environment variables to set before running the script's command. + * May be null. * @param umask umask to be set. It can be null. * @return the path to the script that has been written */ @@ -363,22 +365,26 @@ public static void forceKillProcess(String pid) throws IOException { sendSignalToProcess(Long.parseLong(pid), SIGKILL); } - public static long nimbusVersionOfBlob(String key, ClientBlobStore cb) throws AuthorizationException, KeyNotFoundException { + public static long nimbusVersionOfBlob(String key, + ClientBlobStore cb) throws AuthorizationException, KeyNotFoundException { long nimbusBlobVersion = 0; ReadableBlobMeta metadata = cb.getBlobMeta(key); nimbusBlobVersion = metadata.get_version(); return nimbusBlobVersion; } - public static boolean canUserReadBlob(ReadableBlobMeta meta, String user, Map conf) { + public static boolean canUserReadBlob(ReadableBlobMeta meta, String user, Map conf) { - if (!ObjectReader.getBoolean(conf.get(Config.STORM_BLOBSTORE_ACL_VALIDATION_ENABLED), false)) { + if (!ObjectReader.getBoolean(conf.get(Config.STORM_BLOBSTORE_ACL_VALIDATION_ENABLED), + false)) { return true; } SettableBlobMeta settable = meta.get_settable(); for (AccessControl acl : settable.get_acl()) { - if (acl.get_type().equals(AccessControlType.OTHER) && (acl.get_access() & BlobStoreAclHandler.READ) > 0) { + if (acl.get_type().equals(AccessControlType.OTHER) && (acl + .get_access() & BlobStoreAclHandler.READ) > 0) { return true; } if (acl.get_name().equals(user) && (acl.get_access() & BlobStoreAclHandler.READ) > 0) { @@ -389,7 +395,8 @@ public static boolean canUserReadBlob(ReadableBlobMeta meta, String user, Map * This utility will untar ".tar" files and ".tar.gz","tgz" files. * @@ -420,7 +428,8 @@ private static void ensureDirectory(File dir) throws IOException { * @param untarDir The untar directory where to untar the tar file * @param symlinksDisabled true if symlinks should be disabled, else false */ - public static void unTar(File inFile, File untarDir, boolean symlinksDisabled) throws IOException { + public static void unTar(File inFile, File untarDir, + boolean symlinksDisabled) throws IOException { ensureDirectory(untarDir); boolean gzipped = inFile.toString().endsWith("gz"); @@ -518,7 +527,8 @@ private static void unpackEntries(TarArchiveInputStream tis, } else if (entry.isFile()) { LOG.trace("Extracting file {}", target); ensureDirectory(target.getParentFile()); - try (BufferedOutputStream outputStream = new BufferedOutputStream(new FileOutputStream(target))) { + try (BufferedOutputStream outputStream = + new BufferedOutputStream(new FileOutputStream(target))) { IOUtils.copy(tis, outputStream); } } else { @@ -528,11 +538,11 @@ private static void unpackEntries(TarArchiveInputStream tis, Path p = target.toPath(); if (Files.exists(p)) { try { - //We created it so lets chmod it properly + // We created it so lets chmod it properly int mode = entry.getMode(); Files.setPosixFilePermissions(p, parsePerms(mode)); } catch (UnsupportedOperationException e) { - //Ignored the file system we are on does not support this, so don't do it. + // Ignored the file system we are on does not support this, so don't do it. } } } @@ -569,7 +579,8 @@ private static Set parsePerms(int mode) { return ret; } - public static void unpack(File localrsrc, File dst, boolean symLinksDisabled) throws IOException { + public static void unpack(File localrsrc, File dst, + boolean symLinksDisabled) throws IOException { String lowerDst = localrsrc.getName().toLowerCase(); if (lowerDst.endsWith(".jar") || lowerDst.endsWith("_jar")) { @@ -597,15 +608,18 @@ public static void unpack(File localrsrc, File dst, boolean symLinksDisabled) th } /** - * Extracts the given file to the given directory. Only zip entries starting with the given prefix are extracted. + * Extracts the given file to the given directory. Only zip entries starting with the given + * prefix are extracted. * The prefix is stripped off entry names before extraction. * * @param zipFile The zip file to extract * @param toDir The directory to extract to - * @param prefix The prefix to look for in the zip file. If not null only paths starting with the prefix will be + * @param prefix The prefix to look for in the zip file. If not null only paths starting with + * the prefix will be * extracted */ - public static void extractZipFile(ZipFile zipFile, File toDir, String prefix) throws IOException { + public static void extractZipFile(ZipFile zipFile, File toDir, + String prefix) throws IOException { ensureDirectory(toDir); final String base = toDir.getCanonicalPath(); @@ -614,13 +628,14 @@ public static void extractZipFile(ZipFile zipFile, File toDir, String prefix) th ZipEntry entry = entries.nextElement(); if (!entry.isDirectory()) { if (prefix != null && !entry.getName().startsWith(prefix)) { - //No need to extract it, it is not what we are looking for. + // No need to extract it, it is not what we are looking for. continue; } String entryName; if (prefix != null) { entryName = entry.getName().substring(prefix.length()); - LOG.debug("Extracting {} shortened to {} into {}", entry.getName(), entryName, toDir); + LOG.debug("Extracting {} shortened to {} into {}", entry.getName(), entryName, + toDir); } else { entryName = entry.getName(); } @@ -642,7 +657,8 @@ public static void extractZipFile(ZipFile zipFile, File toDir, String prefix) th } /** - * Given a File input it will unzip the file in a the unzip directory passed as the second parameter. + * Given a File input it will unzip the file in a the unzip directory passed as the second + * parameter. * * @param inFile The zip file as input * @param toDir The unzip directory where to unzip the zip file @@ -654,7 +670,8 @@ public static void unZip(File inFile, File toDir) throws IOException { } /** - * Given a zip File input it will return its size Only works for zip files whose uncompressed size is less than 4 GB, otherwise returns + * Given a zip File input it will return its size Only works for zip files whose uncompressed + * size is less than 4 GB, otherwise returns * the size module 2^32, per gzip specifications. * * @param myFile The zip file as input @@ -679,32 +696,40 @@ public static long zipFileSize(File myFile) throws IOException { */ public static boolean isRas(Map conf) { if (conf.containsKey(DaemonConfig.STORM_SCHEDULER)) { - if (conf.get(DaemonConfig.STORM_SCHEDULER).equals("org.apache.storm.scheduler.resource.ResourceAwareScheduler")) { + if (conf.get(DaemonConfig.STORM_SCHEDULER) + .equals("org.apache.storm.scheduler.resource.ResourceAwareScheduler")) { return true; } } return false; } - public static int getEstimatedWorkerCountForRasTopo(Map topoConf, StormTopology topology) + public static int getEstimatedWorkerCountForRasTopo(Map topoConf, + StormTopology topology) throws InvalidTopologyException { - Double defaultWorkerMaxHeap = ObjectReader.getDouble(topoConf.get(Config.WORKER_HEAP_MEMORY_MB), 768d); - Double topologyWorkerMaxHeap = ObjectReader.getDouble(topoConf.get(Config.TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB), defaultWorkerMaxHeap); - return (int) Math.ceil(getEstimatedTotalHeapMemoryRequiredByTopo(topoConf, topology) / topologyWorkerMaxHeap); + Double defaultWorkerMaxHeap = ObjectReader.getDouble(topoConf + .get(Config.WORKER_HEAP_MEMORY_MB), 768d); + Double topologyWorkerMaxHeap = ObjectReader.getDouble(topoConf + .get(Config.TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB), defaultWorkerMaxHeap); + return (int) Math.ceil(getEstimatedTotalHeapMemoryRequiredByTopo(topoConf, + topology) / topologyWorkerMaxHeap); } - public static double getEstimatedTotalHeapMemoryRequiredByTopo(Map topoConf, StormTopology topology) + public static double getEstimatedTotalHeapMemoryRequiredByTopo(Map topoConf, + StormTopology topology) throws InvalidTopologyException { Map componentParallelism = getComponentParallelism(topoConf, topology); double totalMemoryRequired = 0.0; - for (Map.Entry entry : ResourceUtils.getBoltsResources(topology, topoConf).entrySet()) { + for (Map.Entry entry : ResourceUtils + .getBoltsResources(topology, topoConf).entrySet()) { int parallelism = componentParallelism.getOrDefault(entry.getKey(), 1); double memoryRequirement = entry.getValue().getOnHeapMemoryMb(); totalMemoryRequired += memoryRequirement * parallelism; } - for (Map.Entry entry : ResourceUtils.getSpoutsResources(topology, topoConf).entrySet()) { + for (Map.Entry entry : ResourceUtils + .getSpoutsResources(topology, topoConf).entrySet()) { int parallelism = componentParallelism.getOrDefault(entry.getKey(), 1); double memoryRequirement = entry.getValue().getOnHeapMemoryMb(); totalMemoryRequired += memoryRequirement * parallelism; @@ -720,7 +745,8 @@ private static double getTotalAckerExecutorMemoryUsageForTopo( StormTopology topology, Map topologyConf) throws InvalidTopologyException { topology = StormCommon.systemTopology(topologyConf, topology); - Map boltResources = ResourceUtils.getBoltsResources(topology, topologyConf); + Map boltResources = ResourceUtils + .getBoltsResources(topology, topologyConf); NormalizedResourceRequest entry = boltResources.get(Acker.ACKER_COMPONENT_ID); if (entry == null) { return 0.0d; @@ -730,7 +756,8 @@ private static double getTotalAckerExecutorMemoryUsageForTopo( return entry.getTotalMemoryMb() * parallelism; } - public static Map getComponentParallelism(Map topoConf, StormTopology topology) + public static Map getComponentParallelism(Map topoConf, + StormTopology topology) throws InvalidTopologyException { Map ret = new HashMap<>(); Map components = StormCommon.allComponents(topology); @@ -740,10 +767,14 @@ public static Map getComponentParallelism(Map t return ret; } - public static int getComponentParallelism(Map topoConf, Object component) throws InvalidTopologyException { - Map combinedConf = Utils.merge(topoConf, StormCommon.componentConf(component)); - int numTasks = ObjectReader.getInt(combinedConf.get(Config.TOPOLOGY_TASKS), StormCommon.numStartExecutors(component)); - Integer maxParallel = ObjectReader.getInt(combinedConf.get(Config.TOPOLOGY_MAX_TASK_PARALLELISM), null); + public static int getComponentParallelism(Map topoConf, + Object component) throws InvalidTopologyException { + Map combinedConf = Utils.merge(topoConf, StormCommon + .componentConf(component)); + int numTasks = ObjectReader.getInt(combinedConf.get(Config.TOPOLOGY_TASKS), StormCommon + .numStartExecutors(component)); + Integer maxParallel = ObjectReader.getInt(combinedConf + .get(Config.TOPOLOGY_MAX_TASK_PARALLELISM), null); int ret = numTasks; if (maxParallel != null) { ret = Math.min(maxParallel, numTasks); @@ -759,17 +790,22 @@ public static Subject principalNameToSubject(String name) { } /** - * Resolve a name that came from the topology conf (for example the "localname" of a topology.blobstore.map entry) - * against a base directory. The name is only allowed to point at something strictly inside the base directory, so - * that a topology cannot make the supervisor create a symlink, and force delete whatever was there before, outside + * Resolve a name that came from the topology conf (for example the "localname" of a + * topology.blobstore.map entry) + * against a base directory. The name is only allowed to point at something strictly inside the + * base directory, so + * that a topology cannot make the supervisor create a symlink, and force delete whatever was + * there before, outside * of the directories the supervisor manages for that topology. * * @param baseDir the directory the name has to resolve inside of * @param name the name from the topology conf * @return the resolved file - * @throws IOException if the name is empty, is absolute, contains a ".." component, or resolves outside of baseDir + * @throws IOException if the name is empty, is absolute, contains a ".." component, or resolves + * outside of baseDir */ - public static File resolveTopologyConfSuppliedName(File baseDir, String name) throws IOException { + public static File resolveTopologyConfSuppliedName(File baseDir, + String name) throws IOException { if (StringUtils.isEmpty(name)) { throw new IOException("Invalid local name, it can't be null or empty string"); } @@ -791,7 +827,8 @@ public static File resolveTopologyConfSuppliedName(File baseDir, String name) th Path base = baseDir.toPath().toAbsolutePath().normalize(); Path resolved = ret.toPath().toAbsolutePath().normalize(); if (resolved.equals(base) || !resolved.startsWith(base)) { - throw new IOException("Invalid local name '" + name + "', it does not resolve inside of " + baseDir); + throw new IOException("Invalid local name '" + name + + "', it does not resolve inside of " + baseDir); } return ret; } @@ -801,22 +838,23 @@ public String currentClasspathImpl() { return System.getProperty("java.class.path"); } - public URL getResourceFromClassloaderImpl(String name) { return Thread.currentThread().getContextClassLoader().getResource(name); } - private static final Pattern MEMINFO_PATTERN = Pattern.compile("^([^:\\s]+):\\s*([0-9]+)\\s*kB$"); + private static final Pattern MEMINFO_PATTERN = Pattern + .compile("^([^:\\s]+):\\s*([0-9]+)\\s*kB$"); /** * Get system free memory in megabytes. + * * @return system free memory in megabytes * @throws IOException on I/O exception */ public static long getMemInfoFreeMb() throws IOException { - //MemFree: 14367072 kB - //Buffers: 536512 kB - //Cached: 1192096 kB + // MemFree: 14367072 kB + // Buffers: 536512 kB + // Cached: 1192096 kB // MemFree + Buffers + Cached long memFree = 0; long buffers = 0; @@ -859,29 +897,35 @@ public static boolean isProcessAlive(long pid, String user) throws IOException { private static boolean isWindowsProcessAlive(long pid, String user) throws IOException { boolean ret = false; LOG.debug("CMD: tasklist /fo list /fi \"pid eq {}\" /v", pid); - ProcessBuilder pb = new ProcessBuilder("tasklist", "/fo", "list", "/fi", "pid eq " + pid, "/v"); + ProcessBuilder pb = new ProcessBuilder("tasklist", "/fo", "list", "/fi", "pid eq " + pid, + "/v"); pb.redirectError(ProcessBuilder.Redirect.INHERIT); - try (BufferedReader in = new BufferedReader(new InputStreamReader(pb.start().getInputStream(), StandardCharsets.UTF_8))) { + try (BufferedReader in = new BufferedReader(new InputStreamReader(pb.start() + .getInputStream(), StandardCharsets.UTF_8))) { int lineNo = 0; String line; while ((line = in.readLine()) != null) { lineNo++; LOG.debug("CMD=LINE#{}: {}", lineNo, line); - if (line.contains("User Name:")) { //Check for : in case someone called their user "User Name" - //This line contains the user name for the pid we're looking up - //Example line: "User Name: exampleDomain\exampleUser" + if (line.contains("User Name:")) { // Check for : in case someone called their user "User Name" + // This line contains the user name for the pid we're looking up + // Example line: "User Name: exampleDomain\exampleUser" List userNameLineSplitOnWhitespace = Arrays.asList(line.split(":")); if (userNameLineSplitOnWhitespace.size() == 2) { - List userAndMaybeDomain = Arrays.asList(userNameLineSplitOnWhitespace.get(1).trim().split("\\\\")); - String processUser = userAndMaybeDomain.size() == 2 ? userAndMaybeDomain.get(1) : userAndMaybeDomain.get(0); + List userAndMaybeDomain = Arrays + .asList(userNameLineSplitOnWhitespace.get(1).trim().split("\\\\")); + String processUser = userAndMaybeDomain.size() == 2 ? userAndMaybeDomain + .get(1) : userAndMaybeDomain.get(0); processUser = processUser.trim(); if (user.equals(processUser)) { ret = true; } else { - LOG.info("Found {} running as {}, but expected it to be {}", pid, processUser, user); + LOG.info("Found {} running as {}, but expected it to be {}", pid, + processUser, user); } } else { - LOG.error("Received unexpected output from tasklist command. Expected one colon in user name line. Line was {}", + LOG.error("Received unexpected output from tasklist command. Expected one " + + "colon in user name line. Line was {}", line); } break; @@ -895,7 +939,8 @@ private static boolean isPosixProcessAlive(long pid, String user) throws IOExcep LOG.debug("CMD: ps -o user -p {}", pid); ProcessBuilder pb = new ProcessBuilder("ps", "-o", "user", "-p", String.valueOf(pid)); pb.redirectError(ProcessBuilder.Redirect.INHERIT); - try (BufferedReader in = new BufferedReader(new InputStreamReader(pb.start().getInputStream(), StandardCharsets.UTF_8))) { + try (BufferedReader in = new BufferedReader(new InputStreamReader(pb.start() + .getInputStream(), StandardCharsets.UTF_8))) { int lineNo = 1; String line = in.readLine(); LOG.debug("CMD-LINE#{}: {}", lineNo, line); @@ -920,7 +965,8 @@ private static boolean isPosixProcessAlive(long pid, String user) throws IOExcep } /** - * Are any of the processes alive and running for the specified user. If collection is empty or null + * Are any of the processes alive and running for the specified user. If collection is empty or + * null * then the return value is trivially false. * * @param pids the PIDs of the running processes @@ -940,7 +986,8 @@ public static boolean isAnyProcessAlive(Collection pids, String user) thro } /** - * Are any of the processes alive and running for the specified userId. If collection is empty or null + * Are any of the processes alive and running for the specified userId. If collection is empty + * or null * then the return value is trivially false. * * @param pids the PIDs of the running processes @@ -967,7 +1014,8 @@ public static boolean isAnyProcessAlive(Collection pids, int uid) throws I * @return true if any one of the processes is owned by user and alive, else false * @throws IOException on I/O exception */ - private static boolean isAnyWindowsProcessAlive(Collection pids, String user) throws IOException { + private static boolean isAnyWindowsProcessAlive(Collection pids, + String user) throws IOException { List unexpectedUsers = new ArrayList<>(); for (Long pid : pids) { List cmdArgs = new ArrayList<>(); @@ -980,33 +1028,39 @@ private static boolean isAnyWindowsProcessAlive(Collection pids, String us LOG.debug("CMD: {}", String.join(" ", cmdArgs)); ProcessBuilder pb = new ProcessBuilder(cmdArgs); pb.redirectError(ProcessBuilder.Redirect.INHERIT); - try (BufferedReader in = new BufferedReader(new InputStreamReader(pb.start().getInputStream(), StandardCharsets.UTF_8))) { + try (BufferedReader in = new BufferedReader(new InputStreamReader(pb.start() + .getInputStream(), StandardCharsets.UTF_8))) { int lineNo = 0; String line; while ((line = in.readLine()) != null) { lineNo++; LOG.debug("CMD-LINE#{}: {}", lineNo, line); - if (line.contains("User Name:")) { //Check for : in case someone called their user "User Name" - //This line contains the user name for the pid we're looking up - //Example line: "User Name: exampleDomain\exampleUser" + if (line.contains("User Name:")) { // Check for : in case someone called their user "User Name" + // This line contains the user name for the pid we're looking up + // Example line: "User Name: exampleDomain\exampleUser" List userNameLineSplitOnWhitespace = Arrays.asList(line.split(":")); if (userNameLineSplitOnWhitespace.size() == 2) { - List userAndMaybeDomain = Arrays.asList(userNameLineSplitOnWhitespace.get(1).trim().split("\\\\")); - String processUser = userAndMaybeDomain.size() == 2 ? userAndMaybeDomain.get(1) : userAndMaybeDomain.get(0); + List userAndMaybeDomain = Arrays + .asList(userNameLineSplitOnWhitespace.get(1).trim() + .split("\\\\")); + String processUser = userAndMaybeDomain.size() == 2 ? userAndMaybeDomain + .get(1) : userAndMaybeDomain.get(0); processUser = processUser.trim(); if (user.equals(processUser)) { return true; } unexpectedUsers.add(processUser); } else { - LOG.error("Received unexpected output from tasklist command. Expected one colon in user name line. Line was {}", + LOG.error("Received unexpected output from tasklist command. Expected " + + "one colon in user name line. Line was {}", line); } break; } } } catch (IOException ex) { - String err = String.format("Cannot read output of command \"%s\"", String.join(" ", cmdArgs)); + String err = String.format("Cannot read output of command \"%s\"", String.join(" ", + cmdArgs)); throw new IOException(err, ex); } } @@ -1015,7 +1069,8 @@ private static boolean isAnyWindowsProcessAlive(Collection pids, String us LOG.info("None of the processes {} are alive", pidsAsStr); } else { LOG.info("{} of the Processes {} are running as user(s) {}: but expected user is {}", - unexpectedUsers.size(), pidsAsStr, String.join(",", new TreeSet<>(unexpectedUsers)), user); + unexpectedUsers.size(), pidsAsStr, String.join(",", + new TreeSet<>(unexpectedUsers)), user); } return false; } @@ -1029,7 +1084,8 @@ private static boolean isAnyWindowsProcessAlive(Collection pids, String us * @return true if any one of the processes is owned by user and alive, else false * @throws IOException on I/O exception */ - private static boolean isAnyWindowsProcessAlive(Collection pids, int uid) throws IOException { + private static boolean isAnyWindowsProcessAlive(Collection pids, + int uid) throws IOException { throw new IllegalArgumentException("UID is not supported on Windows"); } @@ -1041,13 +1097,15 @@ private static boolean isAnyWindowsProcessAlive(Collection pids, int uid) * @return true if any one of the processes is owned by user and alive, else false * @throws IOException on I/O exception */ - private static boolean isAnyPosixProcessAlive(Collection pids, String user) throws IOException { + private static boolean isAnyPosixProcessAlive(Collection pids, + String user) throws IOException { String pidParams = StringUtils.join(pids, ","); LOG.debug("CMD: ps -o user -p {}", pidParams); ProcessBuilder pb = new ProcessBuilder("ps", "-o", "user", "-p", pidParams); pb.redirectError(ProcessBuilder.Redirect.INHERIT); List unexpectedUsers = new ArrayList<>(); - try (BufferedReader in = new BufferedReader(new InputStreamReader(pb.start().getInputStream(), StandardCharsets.UTF_8))) { + try (BufferedReader in = new BufferedReader(new InputStreamReader(pb.start() + .getInputStream(), StandardCharsets.UTF_8))) { int lineNo = 1; String line = in.readLine(); LOG.debug("CMD-LINE#{}: {}", lineNo, line); @@ -1065,14 +1123,16 @@ private static boolean isAnyPosixProcessAlive(Collection pids, String user unexpectedUsers.add(line); } } catch (IOException ex) { - String err = String.format("Cannot read output of command \"ps -o user -p %s\"", pidParams); + String err = String.format("Cannot read output of command \"ps -o user -p %s\"", + pidParams); throw new IOException(err, ex); } if (unexpectedUsers.isEmpty()) { LOG.info("None of the processes {} are alive", pidParams); } else { LOG.info("{} of {} Processes {} are running as user(s) {}: but expected user is {}", - unexpectedUsers.size(), pids.size(), pidParams, String.join(",", new TreeSet<>(unexpectedUsers)), user); + unexpectedUsers.size(), pids.size(), pidParams, String.join(",", + new TreeSet<>(unexpectedUsers)), user); } return false; } @@ -1085,13 +1145,15 @@ private static boolean isAnyPosixProcessAlive(Collection pids, String user * @return true if any one of the processes is owned by user and alive, else false * @throws IOException on I/O exception */ - private static boolean isAnyPosixProcessAlive(Collection pids, int uid) throws IOException { + private static boolean isAnyPosixProcessAlive(Collection pids, + int uid) throws IOException { String pidParams = StringUtils.join(pids, ","); LOG.debug("CMD: ps -o uid -p {}", pidParams); ProcessBuilder pb = new ProcessBuilder("ps", "-o", "uid", "-p", pidParams); pb.redirectError(ProcessBuilder.Redirect.INHERIT); List unexpectedUsers = new ArrayList<>(); - try (BufferedReader in = new BufferedReader(new InputStreamReader(pb.start().getInputStream(), StandardCharsets.UTF_8))) { + try (BufferedReader in = new BufferedReader(new InputStreamReader(pb.start() + .getInputStream(), StandardCharsets.UTF_8))) { int lineNo = 1; String line = in.readLine(); LOG.debug("CMD-LINE#{}: {}", lineNo, line); @@ -1113,14 +1175,16 @@ private static boolean isAnyPosixProcessAlive(Collection pids, int uid) th unexpectedUsers.add(line); } } catch (IOException ex) { - String err = String.format("Cannot read output of command \"ps -o uid -p %s\"", pidParams); + String err = String.format("Cannot read output of command \"ps -o uid -p %s\"", + pidParams); throw new IOException(err, ex); } if (unexpectedUsers.isEmpty()) { LOG.info("None of the processes {} are alive", pidParams); } else { LOG.info("{} of {} Processes {} are running as UIDs {}: but expected userId is {}", - unexpectedUsers.size(), pids.size(), pidParams, String.join(",", new TreeSet<>(unexpectedUsers)), uid); + unexpectedUsers.size(), pids.size(), pidParams, String.join(",", + new TreeSet<>(unexpectedUsers)), uid); } return false; } @@ -1129,8 +1193,10 @@ private static boolean isAnyPosixProcessAlive(Collection pids, int uid) th * Get the userId for a user name. This works on Posix systems by using "id -u" command. * Throw IllegalArgumentException on Windows. * - * @param user username to be converted to UID. This is optional, in which case current user is returned. - * @return UID for the specified user (if supplied), else UID of current user, -1 upon Exception. + * @param user username to be converted to UID. This is optional, in which case current user is + * returned. + * @return UID for the specified user (if supplied), else UID of current user, -1 upon + * Exception. */ public static int getUserId(String user) { if (ServerUtils.IS_ON_WINDOWS) { @@ -1148,7 +1214,8 @@ public static int getUserId(String user) { // Ignore } finally { if (exitCode != 0) { - LOG.debug("CMD: '{}' returned exit code of {}", String.join(" ", cmdArgs), exitCode); + LOG.debug("CMD: '{}' returned exit code of {}", String.join(" ", cmdArgs), + exitCode); cmdArgs.remove(user); } } @@ -1156,17 +1223,20 @@ public static int getUserId(String user) { LOG.debug("CMD: {}", String.join(" ", cmdArgs)); ProcessBuilder pb = new ProcessBuilder(cmdArgs); pb.redirectError(ProcessBuilder.Redirect.INHERIT); - try (BufferedReader in = new BufferedReader(new InputStreamReader(pb.start().getInputStream(), StandardCharsets.UTF_8))) { + try (BufferedReader in = new BufferedReader(new InputStreamReader(pb.start() + .getInputStream(), StandardCharsets.UTF_8))) { String line = in.readLine(); LOG.debug("CMD-LINE#1: {}", line); try { return Integer.parseInt(line.trim()); } catch (NumberFormatException ex) { - LOG.error("Expecting UID integer but got {} in output of \"id -u {}\" command", line, user); + LOG.error("Expecting UID integer but got {} in output of \"id -u {}\" command", + line, user); return -1; } } catch (IOException ex) { - LOG.error(String.format("Cannot read output of command \"%s\"", String.join(" ", cmdArgs)), ex); + LOG.error(String.format("Cannot read output of command \"%s\"", String.join(" ", + cmdArgs)), ex); return -1; } } @@ -1190,19 +1260,22 @@ public static int getPathOwnerUid(String fpath) { LOG.debug("CMD: ls -dn {}", fpath); ProcessBuilder pb = new ProcessBuilder("ls", "-dn", fpath); pb.redirectError(ProcessBuilder.Redirect.INHERIT); - try (BufferedReader in = new BufferedReader(new InputStreamReader(pb.start().getInputStream(), StandardCharsets.UTF_8))) { + try (BufferedReader in = new BufferedReader(new InputStreamReader(pb.start() + .getInputStream(), StandardCharsets.UTF_8))) { String line = in.readLine(); LOG.debug("CMD-OUTLINE: {}", line); line = line.trim(); String[] parts = line.split("\\s+"); if (parts.length < 3) { - LOG.error("Expecting at least 3 space separated fields in \"ls -dn {}\" output, got {}", fpath, line); + LOG.error("Expecting at least 3 space separated fields in \"ls -dn {}\" output, " + + "got {}", fpath, line); return -1; } try { return Integer.parseInt(parts[2]); } catch (NumberFormatException ex) { - LOG.error("Expecting at third field {} to be numeric UID \"ls -dn {}\" output, got {}", parts[2], fpath, line); + LOG.error("Expecting at third field {} to be numeric UID \"ls -dn {}\" output, " + + "got {}", parts[2], fpath, line); return -1; } } catch (IOException ex) { @@ -1225,15 +1298,16 @@ private static int getWorkerPathOwnerUid(Map conf, String worker /** * Find if all processes for the user on workId are dead. * This method attempts to optimize the calls by: - *

      - *

    • checking a collection of ProcessIds at once
    • + * + *

    • checking a collection of ProcessIds at once
    • *
    • using userId one Posix systems instead of user
    • *

      * * @return true if all processes for the user are dead on the worker * @throws IOException if external commands have exception. */ - public static boolean areAllProcessesDead(Map conf, String user, String workerId, Set pids) throws IOException { + public static boolean areAllProcessesDead(Map conf, String user, + String workerId, Set pids) throws IOException { if (pids == null || pids.isEmpty()) { return true; } @@ -1245,7 +1319,8 @@ public static boolean areAllProcessesDead(Map conf, String user, try { return !isAnyPosixProcessPidDirAlive(pids, user); } catch (IOException ex) { - LOG.warn("Failed to determine if processes {} for user {} are dead using filesystem, will try \"ps\" command: {}", + LOG.warn("Failed to determine if processes {} for user {} are dead using filesystem, " + + "will try \"ps\" command: {}", pids, user, ex); } if (!cachedUserToUidMap.containsKey(user)) { @@ -1269,8 +1344,7 @@ public static boolean areAllProcessesDead(Map conf, String user, * owned by the supplied user. This is an alternative to "ps -p pid -u uid" command * used in {@link #isAnyPosixProcessAlive(Collection, int)} * - *

      - * Processes are tracked using the existence of the directory "/proc/<pid> + *

      Processes are tracked using the existence of the directory "/proc/<pid> * For each of the supplied PIDs, their PID directory is checked for existence and ownership * by the specified uid. *

      @@ -1280,7 +1354,8 @@ public static boolean areAllProcessesDead(Map conf, String user, * @return true if any one of the processes is owned by user and alive, else false * @throws IOException on I/O exception */ - public static boolean isAnyPosixProcessPidDirAlive(Collection pids, String user) throws IOException { + public static boolean isAnyPosixProcessPidDirAlive(Collection pids, + String user) throws IOException { return isAnyPosixProcessPidDirAlive(pids, user, false); } @@ -1289,8 +1364,7 @@ public static boolean isAnyPosixProcessPidDirAlive(Collection pids, String * owned by the supplied expectedUser. This is an alternative to "ps -p pid -u uid" command * used in {@link #isAnyPosixProcessAlive(Collection, int)} * - *

      - * Processes are tracked using the existence of the directory "/proc/<pid> + *

      Processes are tracked using the existence of the directory "/proc/<pid> * For each of the supplied PIDs, their PID directory is checked for existence and ownership * by the specified uid. *

      @@ -1302,11 +1376,13 @@ public static boolean isAnyPosixProcessPidDirAlive(Collection pids, String * @throws IOException on I/O exception */ @VisibleForTesting - public static boolean isAnyPosixProcessPidDirAlive(Collection pids, String expectedUser, boolean mockFileOwnerToUid) + public static boolean isAnyPosixProcessPidDirAlive(Collection pids, String expectedUser, + boolean mockFileOwnerToUid) throws IOException { File procDir = new File("/proc"); if (!procDir.exists()) { - throw new IOException("Missing process directory " + procDir.getAbsolutePath() + ": method not supported on " + throw new IOException("Missing process directory " + procDir.getAbsolutePath() + + ": method not supported on " + "os.name=" + System.getProperty("os.name")); } for (long pid : pids) { @@ -1314,7 +1390,8 @@ public static boolean isAnyPosixProcessPidDirAlive(Collection pids, String if (!pidDir.exists()) { continue; } - // check if existing process is owned by the specified expectedUser, if not, the process is dead + // check if existing process is owned by the specified expectedUser, if not, the process + // is dead String actualUser; try { actualUser = Files.getOwner(pidDir.toPath()).getName(); @@ -1322,27 +1399,34 @@ public static boolean isAnyPosixProcessPidDirAlive(Collection pids, String continue; // process died before the expectedUser can be checked } if (mockFileOwnerToUid) { - // code activated in testing to simulate Files.getOwner returning UID (which sometimes happens in runtime) + // code activated in testing to simulate Files.getOwner returning UID (which + // sometimes happens in runtime) if (StringUtils.isNumeric(actualUser)) { - LOG.info("Skip mocking, since owner {} of pidDir {} is already numeric", actualUser, pidDir); + LOG.info("Skip mocking, since owner {} of pidDir {} is already numeric", + actualUser, pidDir); } else { Integer actualUid = cachedUserToUidMap.get(actualUser); if (actualUid == null) { actualUid = ServerUtils.getUserId(actualUser); if (actualUid < 0) { - String err = String.format("Cannot get UID for %s, while mocking the owner of pidDir %s", + String err = String + .format("Cannot get UID for %s, while mocking the owner of " + + "pidDir %s", actualUser, pidDir.getAbsolutePath()); throw new IOException(err); } cachedUserToUidMap.put(actualUser, actualUid); - LOG.info("Found UID {} for {}, while mocking the owner of pidDir {}", actualUid, actualUser, pidDir); + LOG.info("Found UID {} for {}, while mocking the owner of pidDir {}", + actualUid, actualUser, pidDir); } else { - LOG.info("Found cached UID {} for {}, while mocking the owner of pidDir {}", actualUid, actualUser, pidDir); + LOG.info("Found cached UID {} for {}, while mocking the owner of pidDir {}", + actualUid, actualUser, pidDir); } actualUser = String.valueOf(actualUid); } } - //sometimes uid is returned instead of username - if so, try to convert and compare with uid + // sometimes uid is returned instead of username - if so, try to convert and compare + // with uid if (StringUtils.isNumeric(actualUser)) { // numeric actualUser - this is UID not user LOG.debug("Process directory {} owner is uid={}", pidDir, actualUser); @@ -1351,32 +1435,39 @@ public static boolean isAnyPosixProcessPidDirAlive(Collection pids, String if (expectedUid == null) { expectedUid = ServerUtils.getUserId(expectedUser); if (expectedUid < 0) { - String err = String.format("Cannot get uid for %s to compare with owner id=%d of process directory %s", + String err = String + .format("Cannot get uid for %s to compare with owner id=%d of " + + "process directory %s", expectedUser, actualUid, pidDir.getAbsolutePath()); throw new IOException(err); } cachedUserToUidMap.put(expectedUser, expectedUid); } if (expectedUid == actualUid) { - LOG.debug("Process {} is alive and owned by expectedUser {}/{}", pid, expectedUser, expectedUid); + LOG.debug("Process {} is alive and owned by expectedUser {}/{}", pid, + expectedUser, expectedUid); return true; } - LOG.info("Prior process is dead, since directory {} owner {} is not same as expectedUser {}/{}, " + LOG.info("Prior process is dead, since directory {} owner {} is not same as " + + "expectedUser {}/{}, " + "likely pid {} was reused for a new process for uid {}, {}", pidDir, actualUser, expectedUser, expectedUid, pid, actualUid, getProcessDesc(pidDir)); } else { // actualUser is a string LOG.debug("Process directory {} owner is {}", pidDir, actualUser); if (expectedUser.equals(actualUser)) { - LOG.debug("Process {} is alive and owned by expectedUser {}", pid, expectedUser); + LOG.debug("Process {} is alive and owned by expectedUser {}", pid, + expectedUser); return true; } - LOG.info("Prior process is dead, since directory {} owner {} is not same as expectedUser {}, " + LOG.info("Prior process is dead, since directory {} owner {} is not same as " + + "expectedUser {}, " + "likely pid {} was reused for a new process for actualUser {}, {}}", pidDir, actualUser, expectedUser, pid, actualUser, getProcessDesc(pidDir)); } } - LOG.info("None of the processes {} are alive AND owned by expectedUser {}", pids, expectedUser); + LOG.info("None of the processes {} are alive AND owned by expectedUser {}", pids, + expectedUser); return false; } @@ -1387,25 +1478,30 @@ public static void validateTopologyWorkerMaxHeapSizeConfigs( double largestMemReq = getMaxExecutorMemoryUsageForTopo(topology, stormConf); double topologyWorkerMaxHeapSize = - ObjectReader.getDouble(stormConf.get(Config.TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB), defaultWorkerMaxHeapSizeMb); + ObjectReader.getDouble(stormConf.get(Config.TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB), + defaultWorkerMaxHeapSizeMb); if (topologyWorkerMaxHeapSize < largestMemReq) { throw new InvalidTopologyException( "Topology will not be able to be successfully scheduled: Config " + "TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB=" + topologyWorkerMaxHeapSize - + " < " + largestMemReq + " (Largest memory requirement of a component in the topology)." + + " < " + largestMemReq + + " (Largest memory requirement of a component in the topology)." + " Perhaps set TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB to a larger amount"); } } /** - * RAS scheduler will try to distribute ackers evenly over workers by adding some ackers to each newly launched worker. + * RAS scheduler will try to distribute ackers evenly over workers by adding some ackers to each + * newly launched worker. * Validations are performed here: * ({@link Config#TOPOLOGY_RAS_ACKER_EXECUTORS_PER_WORKER} * memory for an acker * + memory for the biggest topo executor) < max worker heap memory. * When RAS tries to schedule an executor to a new worker, - * it will put {@link Config#TOPOLOGY_RAS_ACKER_EXECUTORS_PER_WORKER} ackers into the worker first. + * it will put {@link Config#TOPOLOGY_RAS_ACKER_EXECUTORS_PER_WORKER} ackers into the worker + * first. * So {@link Config#TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB} need to be able to accommodate this. + * * @param topoConf Topology conf * @param topology Topology (not system topology) * @param topoName The name of the topology @@ -1414,18 +1510,22 @@ public static void validateTopologyAckerBundleResource(Map topoC StormTopology topology, String topoName) throws InvalidTopologyException { - boolean oneExecutorPerWorker = (Boolean) topoConf.getOrDefault(Config.TOPOLOGY_RAS_ONE_EXECUTOR_PER_WORKER, false); - boolean oneComponentPerWorker = (Boolean) topoConf.getOrDefault(Config.TOPOLOGY_RAS_ONE_COMPONENT_PER_WORKER, false); + boolean oneExecutorPerWorker = (Boolean) topoConf + .getOrDefault(Config.TOPOLOGY_RAS_ONE_EXECUTOR_PER_WORKER, false); + boolean oneComponentPerWorker = (Boolean) topoConf + .getOrDefault(Config.TOPOLOGY_RAS_ONE_COMPONENT_PER_WORKER, false); double topologyWorkerMaxHeapSize = ObjectReader.getDouble(topoConf.get(Config.TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB)); - int numOfAckerExecutorsPerWorker = ObjectReader.getInt(topoConf.get(Config.TOPOLOGY_RAS_ACKER_EXECUTORS_PER_WORKER)); + int numOfAckerExecutorsPerWorker = ObjectReader.getInt(topoConf + .get(Config.TOPOLOGY_RAS_ACKER_EXECUTORS_PER_WORKER)); double maxTopoExecMem = getMaxExecutorMemoryUsageForTopo(topology, topoConf); double ackerExecMem = getAckerExecutorMemoryUsageForTopo(topology, topoConf); double minMemReqForWorker = maxTopoExecMem + ackerExecMem * numOfAckerExecutorsPerWorker; - // A worker need to have enough resources for a bigest topo executor + topology.acker.executors.per.worker ackers + // A worker need to have enough resources for a bigest topo executor + + // topology.acker.executors.per.worker ackers if (!oneExecutorPerWorker && !oneComponentPerWorker && topologyWorkerMaxHeapSize < minMemReqForWorker) { @@ -1464,7 +1564,8 @@ private static double getAckerExecutorMemoryUsageForTopo( StormTopology topology, Map topologyConf) throws InvalidTopologyException { topology = StormCommon.systemTopology(topologyConf, topology); - Map boltResources = ResourceUtils.getBoltsResources(topology, topologyConf); + Map boltResources = ResourceUtils + .getBoltsResources(topology, topologyConf); NormalizedResourceRequest entry = boltResources.get(Acker.ACKER_COMPONENT_ID); if (entry == null) { return 0.0d; @@ -1473,8 +1574,10 @@ private static double getAckerExecutorMemoryUsageForTopo( } /** - * Support method to obtain additional log info for the process. Use the contents of comm and cmdline - * in the process directory. Note that this method works properly only on posix systems with /proc directory. + * Support method to obtain additional log info for the process. Use the contents of comm and + * cmdline + * in the process directory. Note that this method works properly only on posix systems with + * /proc directory. * * @param pidDir PID directory (/proc/<pid>) * @return process description string diff --git a/storm-server/src/main/java/org/apache/storm/utils/StormCommonInstaller.java b/storm-server/src/main/java/org/apache/storm/utils/StormCommonInstaller.java index e47ff102a2a..c210100dc9a 100644 --- a/storm-server/src/main/java/org/apache/storm/utils/StormCommonInstaller.java +++ b/storm-server/src/main/java/org/apache/storm/utils/StormCommonInstaller.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-server/src/main/java/org/apache/storm/utils/ZookeeperServerCnxnFactory.java b/storm-server/src/main/java/org/apache/storm/utils/ZookeeperServerCnxnFactory.java index 07a6eb3312d..96010e9e85d 100644 --- a/storm-server/src/main/java/org/apache/storm/utils/ZookeeperServerCnxnFactory.java +++ b/storm-server/src/main/java/org/apache/storm/utils/ZookeeperServerCnxnFactory.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -25,7 +31,7 @@ public class ZookeeperServerCnxnFactory { NIOServerCnxnFactory factory; public ZookeeperServerCnxnFactory(int port, int maxClientCnxns) { - //port range + // port range int max; if (port <= 0) { this.port = 2000; @@ -37,14 +43,14 @@ public ZookeeperServerCnxnFactory(int port, int maxClientCnxns) { factory = new NIOServerCnxnFactory(); - //look for available port + // look for available port for (; this.port <= max; this.port++) { try { factory.configure(new InetSocketAddress(this.port), maxClientCnxns); LOG.debug("Zookeeper server successfully binded at port " + this.port); break; } catch (BindException e1) { - //ignore + // ignore } catch (IOException e2) { this.port = 0; factory = null; diff --git a/storm-server/src/main/java/org/apache/storm/zookeeper/AclEnforcement.java b/storm-server/src/main/java/org/apache/storm/zookeeper/AclEnforcement.java index 1ecdfe36e5f..e87856a7807 100644 --- a/storm-server/src/main/java/org/apache/storm/zookeeper/AclEnforcement.java +++ b/storm-server/src/main/java/org/apache/storm/zookeeper/AclEnforcement.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -51,6 +57,7 @@ public class AclEnforcement { /** * Verify the ZK ACLs are correct and optionally fix them if needed. + * * @param conf the cluster config. * @param fixUp true if we want to fix the ACLs else false. * @throws Exception on any error. @@ -58,7 +65,7 @@ public class AclEnforcement { public static void verifyAcls(Map conf, final boolean fixUp) throws Exception { if (!Utils.isZkAuthenticationConfiguredStormServer(conf)) { LOG.info("SECURITY IS DISABLED NO FURTHER CHECKS..."); - //There is no security so we are done. + // There is no security so we are done. return; } ACL superUserAcl = Utils.getSuperUserAcl(conf); @@ -82,7 +89,7 @@ public static void verifyAcls(Map conf, final boolean fixUp) thr try (CuratorFramework zk = ClientZookeeper.mkClient(conf, zkServers, port, "", new DefaultWatcherCallBack(), conf, DaemonType.NIMBUS)) { if (zk.checkExists().forPath(stormRoot) != null) { - //First off we want to verify that ROOT is good + // First off we want to verify that ROOT is good verifyAclStrict(zk, superAcl, stormRoot, fixUp); } else { LOG.warn("{} does not exist no need to check any more...", stormRoot); @@ -93,23 +100,25 @@ public static void verifyAcls(Map conf, final boolean fixUp) thr // Now that the root is fine we can start to look at the other paths under it. try (CuratorFramework zk = ClientZookeeper.mkClient(conf, zkServers, port, stormRoot, new DefaultWatcherCallBack(), conf, DaemonType.NIMBUS)) { - //Next verify that the blob store is correct before we start it up. + // Next verify that the blob store is correct before we start it up. if (zk.checkExists().forPath(ClusterUtils.BLOBSTORE_SUBTREE) != null) { verifyAclStrictRecursive(zk, superAcl, ClusterUtils.BLOBSTORE_SUBTREE, fixUp); } - if (zk.checkExists().forPath(ClusterUtils.BLOBSTORE_MAX_KEY_SEQUENCE_NUMBER_SUBTREE) != null) { - verifyAclStrict(zk, superAcl, ClusterUtils.BLOBSTORE_MAX_KEY_SEQUENCE_NUMBER_SUBTREE, fixUp); + if (zk.checkExists() + .forPath(ClusterUtils.BLOBSTORE_MAX_KEY_SEQUENCE_NUMBER_SUBTREE) != null) { + verifyAclStrict(zk, superAcl, + ClusterUtils.BLOBSTORE_MAX_KEY_SEQUENCE_NUMBER_SUBTREE, fixUp); } - //The blobstore is good, now lets get the list of all topo Ids + // The blobstore is good, now lets get the list of all topo Ids Set topoIds = new HashSet<>(); if (zk.checkExists().forPath(ClusterUtils.STORMS_SUBTREE) != null) { topoIds.addAll(zk.getChildren().forPath(ClusterUtils.STORMS_SUBTREE)); } Map topoToZkCreds = new HashMap<>(); - //Now lets get the creds for the topos so we can verify those as well. + // Now lets get the creds for the topos so we can verify those as well. BlobStore bs = ServerUtils.getNimbusBlobStore(conf, NimbusInfo.fromConf(conf), null); try { Subject nimbusSubject = new Subject(); @@ -117,10 +126,13 @@ public static void verifyAcls(Map conf, final boolean fixUp) thr for (String topoId : topoIds) { try { String blobKey = topoId + "-stormconf.ser"; - Map topoConf = Utils.fromCompressedJsonConf(bs.readBlob(blobKey, nimbusSubject)); - String payload = (String) topoConf.get(Config.STORM_ZOOKEEPER_TOPOLOGY_AUTH_PAYLOAD); + Map topoConf = Utils.fromCompressedJsonConf(bs + .readBlob(blobKey, nimbusSubject)); + String payload = (String) topoConf + .get(Config.STORM_ZOOKEEPER_TOPOLOGY_AUTH_PAYLOAD); try { - topoToZkCreds.put(topoId, new Id("digest", DigestAuthenticationProvider.generateDigest(payload))); + topoToZkCreds.put(topoId, new Id("digest", DigestAuthenticationProvider + .generateDigest(payload))); } catch (NoSuchAlgorithmException e) { throw new RuntimeException(e); } @@ -134,34 +146,46 @@ public static void verifyAcls(Map conf, final boolean fixUp) thr } } - verifyParentWithReadOnlyTopoChildren(zk, superUserAcl, ClusterUtils.STORMS_SUBTREE, topoToZkCreds, fixUp); - verifyParentWithReadOnlyTopoChildren(zk, superUserAcl, ClusterUtils.ASSIGNMENTS_SUBTREE, topoToZkCreds, fixUp); - //There is a race on credentials where they can be leaked in some versions of storm. - verifyParentWithReadOnlyTopoChildrenDeleteDead(zk, superUserAcl, ClusterUtils.CREDENTIALS_SUBTREE, topoToZkCreds, fixUp); - //There is a race on logconfig where they can be leaked in some versions of storm. - verifyParentWithReadOnlyTopoChildrenDeleteDead(zk, superUserAcl, ClusterUtils.LOGCONFIG_SUBTREE, topoToZkCreds, fixUp); - //There is a race on backpressure too... - verifyParentWithReadWriteTopoChildrenDeleteDead(zk, superUserAcl, ClusterUtils.BACKPRESSURE_SUBTREE, topoToZkCreds, fixUp); + verifyParentWithReadOnlyTopoChildren(zk, superUserAcl, ClusterUtils.STORMS_SUBTREE, + topoToZkCreds, fixUp); + verifyParentWithReadOnlyTopoChildren(zk, superUserAcl, ClusterUtils.ASSIGNMENTS_SUBTREE, + topoToZkCreds, fixUp); + // There is a race on credentials where they can be leaked in some versions of storm. + verifyParentWithReadOnlyTopoChildrenDeleteDead(zk, superUserAcl, + ClusterUtils.CREDENTIALS_SUBTREE, topoToZkCreds, fixUp); + // There is a race on logconfig where they can be leaked in some versions of storm. + verifyParentWithReadOnlyTopoChildrenDeleteDead(zk, superUserAcl, + ClusterUtils.LOGCONFIG_SUBTREE, topoToZkCreds, fixUp); + // There is a race on backpressure too... + verifyParentWithReadWriteTopoChildrenDeleteDead(zk, superUserAcl, + ClusterUtils.BACKPRESSURE_SUBTREE, topoToZkCreds, fixUp); if (zk.checkExists().forPath(ClusterUtils.ERRORS_SUBTREE) != null) { - //errors is a bit special because in older versions of storm the worker created the parent directories lazily - // because of this it means we need to auto create at least the topo-id directory for all running topos. + // errors is a bit special because in older versions of storm the worker created the + // parent directories lazily + // because of this it means we need to auto create at least the topo-id directory + // for all running topos. for (String topoId : topoToZkCreds.keySet()) { String path = ClusterUtils.errorStormRoot(topoId); if (zk.checkExists().forPath(path) == null) { LOG.warn("Creating missing errors location {}", path); - zk.create().withACL(getTopoReadWrite(path, topoId, topoToZkCreds, superUserAcl, fixUp)).forPath(path); + zk.create().withACL(getTopoReadWrite(path, topoId, topoToZkCreds, + superUserAcl, fixUp)).forPath(path); } } } - //Error should not be leaked according to the code, but they are not important enough to fail the build if + // Error should not be leaked according to the code, but they are not important enough + // to fail the build if // for some odd reason they are leaked. - verifyParentWithReadWriteTopoChildrenDeleteDead(zk, superUserAcl, ClusterUtils.ERRORS_SUBTREE, topoToZkCreds, fixUp); + verifyParentWithReadWriteTopoChildrenDeleteDead(zk, superUserAcl, + ClusterUtils.ERRORS_SUBTREE, topoToZkCreds, fixUp); if (zk.checkExists().forPath(ClusterUtils.SECRET_KEYS_SUBTREE) != null) { verifyAclStrict(zk, superAcl, ClusterUtils.SECRET_KEYS_SUBTREE, fixUp); - verifyAclStrictRecursive(zk, superAcl, ClusterUtils.secretKeysPath(WorkerTokenServiceType.NIMBUS), fixUp); - verifyAclStrictRecursive(zk, drpcFullAcl, ClusterUtils.secretKeysPath(WorkerTokenServiceType.DRPC), fixUp); + verifyAclStrictRecursive(zk, superAcl, ClusterUtils + .secretKeysPath(WorkerTokenServiceType.NIMBUS), fixUp); + verifyAclStrictRecursive(zk, drpcFullAcl, ClusterUtils + .secretKeysPath(WorkerTokenServiceType.DRPC), fixUp); } if (zk.checkExists().forPath(ClusterUtils.NIMBUSES_SUBTREE) != null) { @@ -181,16 +205,20 @@ public static void verifyAcls(Map conf, final boolean fixUp) thr } // When moving to pacemaker workerbeats can be leaked too... - verifyParentWithReadWriteTopoChildrenDeleteDead(zk, superUserAcl, ClusterUtils.WORKERBEATS_SUBTREE, topoToZkCreds, fixUp); + verifyParentWithReadWriteTopoChildrenDeleteDead(zk, superUserAcl, + ClusterUtils.WORKERBEATS_SUBTREE, topoToZkCreds, fixUp); } } - private static List getTopoAcl(String path, String topoId, Map topoToZkCreds, ACL superAcl, boolean fixUp, int perms) { + private static List getTopoAcl(String path, String topoId, Map topoToZkCreds, + ACL superAcl, boolean fixUp, int perms) { Id id = topoToZkCreds.get(topoId); if (id == null) { - String error = "Could not find credentials for topology " + topoId + " at path " + path + "."; + String error = "Could not find credentials for topology " + topoId + " at path " + path + + "."; if (fixUp) { - error += " Don't know how to fix this automatically. Please add needed ACLs, or delete the path."; + error += " Don't know how to fix this automatically. Please add needed ACLs, or " + + "delete the path."; } throw new IllegalStateException(error); } @@ -200,11 +228,13 @@ private static List getTopoAcl(String path, String topoId, Map return ret; } - private static List getTopoReadWrite(String path, String topoId, Map topoToZkCreds, ACL superAcl, boolean fixUp) { + private static List getTopoReadWrite(String path, String topoId, Map topoToZkCreds, ACL superAcl, boolean fixUp) { return getTopoAcl(path, topoId, topoToZkCreds, superAcl, fixUp, ZooDefs.Perms.ALL); } - private static void verifyParentWithTopoChildrenDeleteDead(CuratorFramework zk, ACL superUserAcl, String path, + private static void verifyParentWithTopoChildrenDeleteDead(CuratorFramework zk, + ACL superUserAcl, String path, Map topoToZkCreds, boolean fixUp, int perms) throws Exception { if (zk.checkExists().forPath(path) != null) { verifyAclStrict(zk, Arrays.asList(superUserAcl), path, fixUp); @@ -212,19 +242,21 @@ private static void verifyParentWithTopoChildrenDeleteDead(CuratorFramework zk, for (String topoId : zk.getChildren().forPath(path)) { String childPath = path + ClusterUtils.ZK_SEPERATOR + topoId; if (!topoToZkCreds.containsKey(topoId)) { - //Save it to try again later... + // Save it to try again later... possiblyBadIds.add(topoId); } else { - List rwAcl = getTopoAcl(path, topoId, topoToZkCreds, superUserAcl, fixUp, perms); + List rwAcl = getTopoAcl(path, topoId, topoToZkCreds, superUserAcl, fixUp, + perms); verifyAclStrictRecursive(zk, rwAcl, childPath, fixUp); } } if (!possiblyBadIds.isEmpty()) { - //Lets reread the children in STORMS as the source of truth and see if a new one was created in the background + // Lets reread the children in STORMS as the source of truth and see if a new one + // was created in the background possiblyBadIds.removeAll(zk.getChildren().forPath(ClusterUtils.STORMS_SUBTREE)); for (String topoId : possiblyBadIds) { - //Now we know for sure that this is a bad id + // Now we know for sure that this is a bad id String childPath = path + ClusterUtils.ZK_SEPERATOR + topoId; zk.delete().deletingChildrenIfNeeded().forPath(childPath); } @@ -232,39 +264,50 @@ private static void verifyParentWithTopoChildrenDeleteDead(CuratorFramework zk, } } - private static void verifyParentWithReadOnlyTopoChildrenDeleteDead(CuratorFramework zk, ACL superUserAcl, String path, + private static void verifyParentWithReadOnlyTopoChildrenDeleteDead(CuratorFramework zk, + ACL superUserAcl, String path, Map topoToZkCreds, boolean fixUp) throws Exception { - verifyParentWithTopoChildrenDeleteDead(zk, superUserAcl, path, topoToZkCreds, fixUp, ZooDefs.Perms.READ); + verifyParentWithTopoChildrenDeleteDead(zk, superUserAcl, path, topoToZkCreds, fixUp, + ZooDefs.Perms.READ); } - private static void verifyParentWithReadWriteTopoChildrenDeleteDead(CuratorFramework zk, ACL superUserAcl, String path, + private static void verifyParentWithReadWriteTopoChildrenDeleteDead(CuratorFramework zk, + ACL superUserAcl, String path, Map topoToZkCreds, boolean fixUp) throws Exception { - verifyParentWithTopoChildrenDeleteDead(zk, superUserAcl, path, topoToZkCreds, fixUp, ZooDefs.Perms.ALL); + verifyParentWithTopoChildrenDeleteDead(zk, superUserAcl, path, topoToZkCreds, fixUp, + ZooDefs.Perms.ALL); } - private static void verifyParentWithTopoChildren(CuratorFramework zk, ACL superUserAcl, String path, + private static void verifyParentWithTopoChildren(CuratorFramework zk, ACL superUserAcl, + String path, Map topoToZkCreds, boolean fixUp, int perms) throws Exception { if (zk.checkExists().forPath(path) != null) { verifyAclStrict(zk, Arrays.asList(superUserAcl), path, fixUp); for (String topoId : zk.getChildren().forPath(path)) { String childPath = path + ClusterUtils.ZK_SEPERATOR + topoId; - List rwAcl = getTopoAcl(path, topoId, topoToZkCreds, superUserAcl, fixUp, perms); + List rwAcl = getTopoAcl(path, topoId, topoToZkCreds, superUserAcl, fixUp, + perms); verifyAclStrictRecursive(zk, rwAcl, childPath, fixUp); } } } - private static void verifyParentWithReadOnlyTopoChildren(CuratorFramework zk, ACL superUserAcl, String path, + private static void verifyParentWithReadOnlyTopoChildren(CuratorFramework zk, ACL superUserAcl, + String path, Map topoToZkCreds, boolean fixUp) throws Exception { - verifyParentWithTopoChildren(zk, superUserAcl, path, topoToZkCreds, fixUp, ZooDefs.Perms.READ); + verifyParentWithTopoChildren(zk, superUserAcl, path, topoToZkCreds, fixUp, + ZooDefs.Perms.READ); } - private static void verifyParentWithReadWriteTopoChildren(CuratorFramework zk, ACL superUserAcl, String path, + private static void verifyParentWithReadWriteTopoChildren(CuratorFramework zk, ACL superUserAcl, + String path, Map topoToZkCreds, boolean fixUp) throws Exception { - verifyParentWithTopoChildren(zk, superUserAcl, path, topoToZkCreds, fixUp, ZooDefs.Perms.ALL); + verifyParentWithTopoChildren(zk, superUserAcl, path, topoToZkCreds, fixUp, + ZooDefs.Perms.ALL); } - private static void verifyAclStrictRecursive(CuratorFramework zk, List strictAcl, String path, boolean fixUp) throws Exception { + private static void verifyAclStrictRecursive(CuratorFramework zk, List strictAcl, + String path, boolean fixUp) throws Exception { verifyAclStrict(zk, strictAcl, path, fixUp); for (String child : zk.getChildren().forPath(path)) { String newPath = path + ClusterUtils.ZK_SEPERATOR + child; @@ -272,15 +315,18 @@ private static void verifyAclStrictRecursive(CuratorFramework zk, List stri } } - private static void verifyAclStrict(CuratorFramework zk, List strictAcl, String path, boolean fixUp) throws Exception { + private static void verifyAclStrict(CuratorFramework zk, List strictAcl, String path, + boolean fixUp) throws Exception { try { List foundAcl = zk.getACL().forPath(path); if (!equivalent(foundAcl, strictAcl)) { if (fixUp) { - LOG.warn("{} expected to have ACL {}, but has {}. Fixing...", path, strictAcl, foundAcl); + LOG.warn("{} expected to have ACL {}, but has {}. Fixing...", path, strictAcl, + foundAcl); zk.setACL().withACL(strictAcl).forPath(path); } else { - throw new IllegalStateException(path + " did not have the correct ACL found " + foundAcl + " expected " + strictAcl); + throw new IllegalStateException(path + " did not have the correct ACL found " + + foundAcl + " expected " + strictAcl); } } } catch (KeeperException.NoNodeException ne) { @@ -308,7 +354,8 @@ public static void main(String[] args) throws Exception { if ("-f".equals(a) || "--fixup".equals(a)) { fixUp = true; } else { - throw new IllegalArgumentException("Unsupported argument " + arg + " only -f or --fixup is supported."); + throw new IllegalArgumentException("Unsupported argument " + arg + + " only -f or --fixup is supported."); } } verifyAcls(conf, fixUp); diff --git a/storm-server/src/main/java/org/apache/storm/zookeeper/LeaderElectorImp.java b/storm-server/src/main/java/org/apache/storm/zookeeper/LeaderElectorImp.java index 7e887fbb431..2011dd3bee2 100644 --- a/storm-server/src/main/java/org/apache/storm/zookeeper/LeaderElectorImp.java +++ b/storm-server/src/main/java/org/apache/storm/zookeeper/LeaderElectorImp.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -38,12 +44,14 @@ public class LeaderElectorImp implements ILeaderElector { private final LeaderListenerCallbackFactory leaderListenerCallbackFactory; private final StormTimer timer; - public LeaderElectorImp(CuratorFramework zk, String id, LeaderListenerCallbackFactory leaderListenerCallbackFactory) { + public LeaderElectorImp(CuratorFramework zk, String id, + LeaderListenerCallbackFactory leaderListenerCallbackFactory) { this.zk = zk; this.id = id; this.leaderLatch = new AtomicReference<>(new LeaderLatch(zk, leaderLockPath, id)); this.leaderListenerCallbackFactory = leaderListenerCallbackFactory; - this.timer = new StormTimer("leader-elector-timer", Utils.createDefaultUncaughtExceptionHandler()); + this.timer = new StormTimer("leader-elector-timer", Utils + .createDefaultUncaughtExceptionHandler()); } @Override @@ -59,7 +67,8 @@ public void addToLeaderLockQueue() throws Exception { latch.addListener(leaderListenerCallbackFactory.create(this)); latch.start(); leaderLatch.set(latch); - LOG.info("LeaderLatch was in closed state. Reset the leaderLatch, and queued for leader lock."); + LOG.info("LeaderLatch was in closed state. Reset the leaderLatch, and queued for " + + "leader lock."); } // If the latch is not started yet, start it if (LeaderLatch.State.LATENT.equals(leaderLatch.get().getState())) { @@ -80,7 +89,7 @@ public void quitElectionFor(int delayMs) throws Exception { } catch (Exception e) { throw Utils.wrapInRuntime(e); } - }, false, 0); //Don't error if timer is shut down, happens when the elector is closed. + }, false, 0); // Don't error if timer is shut down, happens when the elector is closed. } private void removeFromLeaderLockQueue() throws Exception { diff --git a/storm-server/src/main/java/org/apache/storm/zookeeper/LeaderListenerCallbackFactory.java b/storm-server/src/main/java/org/apache/storm/zookeeper/LeaderListenerCallbackFactory.java index 5582c060359..5cc933ce2a2 100644 --- a/storm-server/src/main/java/org/apache/storm/zookeeper/LeaderListenerCallbackFactory.java +++ b/storm-server/src/main/java/org/apache/storm/zookeeper/LeaderListenerCallbackFactory.java @@ -47,7 +47,8 @@ public class LeaderListenerCallbackFactory { private final StormMetricsRegistry metricsRegistry; private final Object submitLock; - public LeaderListenerCallbackFactory(Map conf, CuratorFramework zk, BlobStore blobStore, TopoCache tc, + public LeaderListenerCallbackFactory(Map conf, CuratorFramework zk, + BlobStore blobStore, TopoCache tc, IStormClusterState clusterState, List acls, StormMetricsRegistry metricsRegistry, Object submitLock) { this.conf = conf; this.zk = zk; @@ -60,7 +61,8 @@ public LeaderListenerCallbackFactory(Map conf, CuratorFramework } public LeaderLatchListener create(ILeaderElector elector) throws UnknownHostException { - final LeaderListenerCallback callback = new LeaderListenerCallback(conf, zk, blobStore, elector, + final LeaderListenerCallback callback = new LeaderListenerCallback(conf, zk, blobStore, + elector, tc, clusterState, acls, metricsRegistry); final String hostName = InetAddress.getLocalHost().getCanonicalHostName(); return new LeaderLatchListener() { @@ -74,7 +76,7 @@ public void isLeader() { @Override public void notLeader() { LOG.info("{} lost leadership.", hostName); - //Just to be sure + // Just to be sure callback.notLeaderCallback(); } }; diff --git a/storm-server/src/main/java/org/apache/storm/zookeeper/Zookeeper.java b/storm-server/src/main/java/org/apache/storm/zookeeper/Zookeeper.java index a5a8ecac8c4..a1416175834 100644 --- a/storm-server/src/main/java/org/apache/storm/zookeeper/Zookeeper.java +++ b/storm-server/src/main/java/org/apache/storm/zookeeper/Zookeeper.java @@ -38,7 +38,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; - public class Zookeeper { // A singleton instance allows us to mock delegated static methods in our // tests by subclassing. @@ -47,7 +46,8 @@ public class Zookeeper { private static Zookeeper instance = INSTANCE; /** - * Provide an instance of this class for delegates to use. To mock out delegated methods, provide an instance of a subclass that + * Provide an instance of this class for delegates to use. To mock out delegated methods, + * provide an instance of a subclass that * overrides the implementation of the delegated method. * * @param u a Zookeeper instance @@ -57,14 +57,16 @@ public static void setInstance(Zookeeper u) { } /** - * Resets the singleton instance to the default. This is helpful to reset the class to its original functionality when mocking is no + * Resets the singleton instance to the default. This is helpful to reset the class to its + * original functionality when mocking is no * longer desired. */ public static void resetInstance() { instance = INSTANCE; } - public static NIOServerCnxnFactory mkInprocessZookeeper(String localdir, Integer port) throws Exception { + public static NIOServerCnxnFactory mkInprocessZookeeper(String localdir, + Integer port) throws Exception { NIOServerCnxnFactory factory = null; int report = 2000; int limitPort = 65535; @@ -80,7 +82,8 @@ public static NIOServerCnxnFactory mkInprocessZookeeper(String localdir, Integer } catch (BindException e) { report++; if (report > limitPort) { - throw new RuntimeException("No port is available to launch an inprocess zookeeper"); + throw new RuntimeException("No port is available to launch an inprocess " + + "zookeeper"); } } } @@ -98,7 +101,8 @@ public static void shutdownInprocessZookeeper(NIOServerCnxnFactory handle) { public static NimbusInfo toNimbusInfo(Participant participant) { String id = participant.getId(); if (StringUtils.isBlank(id)) { - throw new RuntimeException("No nimbus leader participant host found, have you started your nimbus hosts?"); + throw new RuntimeException("No nimbus leader participant host found, have you started " + + "your nimbus hosts?"); } NimbusInfo nimbusInfo = NimbusInfo.parse(id); nimbusInfo.setLeader(participant.isLeader()); @@ -109,25 +113,30 @@ public static NimbusInfo toNimbusInfo(Participant participant) { * Get master leader elector. * * @param conf Config. - * @param zkClient ZkClient, the client must have a default Config.STORM_ZOOKEEPER_ROOT as root path. + * @param zkClient ZkClient, the client must have a default Config.STORM_ZOOKEEPER_ROOT as root + * path. * @param blobStore {@link BlobStore} * @param tc {@link TopoCache} * @param clusterState {@link IStormClusterState} * @param acls ACLs * @return Instance of {@link ILeaderElector} */ - public static ILeaderElector zkLeaderElector(Map conf, CuratorFramework zkClient, BlobStore blobStore, + public static ILeaderElector zkLeaderElector(Map conf, + CuratorFramework zkClient, BlobStore blobStore, final TopoCache tc, IStormClusterState clusterState, List acls, StormMetricsRegistry metricsRegistry, Object submitLock) { - return instance.zkLeaderElectorImpl(conf, zkClient, blobStore, tc, clusterState, acls, metricsRegistry, submitLock); + return instance.zkLeaderElectorImpl(conf, zkClient, blobStore, tc, clusterState, acls, + metricsRegistry, submitLock); } - protected ILeaderElector zkLeaderElectorImpl(Map conf, CuratorFramework zk, BlobStore blobStore, + protected ILeaderElector zkLeaderElectorImpl(Map conf, CuratorFramework zk, + BlobStore blobStore, final TopoCache tc, IStormClusterState clusterState, List acls, StormMetricsRegistry metricsRegistry, Object submitLock) { String id = NimbusInfo.fromConf(conf).toHostPortString(); return new LeaderElectorImp(zk, id, - new LeaderListenerCallbackFactory(conf, zk, blobStore, tc, clusterState, acls, metricsRegistry, submitLock)); + new LeaderListenerCallbackFactory(conf, zk, blobStore, tc, clusterState, acls, + metricsRegistry, submitLock)); } } diff --git a/storm-server/src/test/java/org/apache/storm/AssertLoop.java b/storm-server/src/test/java/org/apache/storm/AssertLoop.java index 89edc600bdf..e517de74008 100644 --- a/storm-server/src/test/java/org/apache/storm/AssertLoop.java +++ b/storm-server/src/test/java/org/apache/storm/AssertLoop.java @@ -34,7 +34,8 @@ public static void assertLoop(Predicate condition, Object... conditionPa Awaitility.with() .pollInterval(1, TimeUnit.MILLISECONDS) .atMost(Testing.TEST_TIMEOUT_MS, TimeUnit.MILLISECONDS) - .untilAsserted(() -> assertThat(Arrays.asList(conditionParams), everyItem(matchesPredicate(condition)))); + .untilAsserted(() -> assertThat(Arrays.asList(conditionParams), + everyItem(matchesPredicate(condition)))); } catch (ConditionTimeoutException e) { throw new AssertionError(e.getMessage()); } diff --git a/storm-server/src/test/java/org/apache/storm/DaemonConfigTest.java b/storm-server/src/test/java/org/apache/storm/DaemonConfigTest.java index a6bd97afcc3..74c29c8bffa 100644 --- a/storm-server/src/test/java/org/apache/storm/DaemonConfigTest.java +++ b/storm-server/src/test/java/org/apache/storm/DaemonConfigTest.java @@ -1,29 +1,35 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.LinkedList; import java.util.Map; - import org.apache.storm.utils.ConfigUtils; import org.apache.storm.validation.ConfigValidation; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.*; - public class DaemonConfigTest { private void stringOrStringListTest(String key) { @@ -40,7 +46,7 @@ private void stringOrStringListTest(String key) { Integer[] wrongStuff = { 1, 2, 3 }; failCases.add(Arrays.asList(wrongStuff)); - //worker.childopts validates + // worker.childopts validates for (Object value : passCases) { conf.put(key, value); ConfigValidation.validateFields(conf); @@ -48,7 +54,8 @@ private void stringOrStringListTest(String key) { for (Object value : failCases) { conf.put(key, value); - assertThrows(IllegalArgumentException.class, () -> ConfigValidation.validateFields(conf)); + assertThrows(IllegalArgumentException.class, () -> ConfigValidation + .validateFields(conf)); } } diff --git a/storm-server/src/test/java/org/apache/storm/LocalStateTest.java b/storm-server/src/test/java/org/apache/storm/LocalStateTest.java index 4b7651ddce8..04f219886ea 100644 --- a/storm-server/src/test/java/org/apache/storm/LocalStateTest.java +++ b/storm-server/src/test/java/org/apache/storm/LocalStateTest.java @@ -1,17 +1,27 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + import java.io.File; import java.io.FileOutputStream; import java.io.IOException; @@ -23,10 +33,6 @@ import org.apache.storm.utils.LocalState; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; - public class LocalStateTest { @Test diff --git a/storm-server/src/test/java/org/apache/storm/MessagingTest.java b/storm-server/src/test/java/org/apache/storm/MessagingTest.java index 72cf7b8fe8a..a49d0422837 100644 --- a/storm-server/src/test/java/org/apache/storm/MessagingTest.java +++ b/storm-server/src/test/java/org/apache/storm/MessagingTest.java @@ -1,17 +1,25 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm; +import static org.junit.jupiter.api.Assertions.assertEquals; + import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -26,8 +34,6 @@ import org.apache.storm.topology.TopologyBuilder; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; - public class MessagingTest { @Test @@ -37,7 +43,8 @@ public void testLocalTransport() throws Exception { topoConf.put(Config.STORM_MESSAGING_TRANSPORT, "org.apache.storm.messaging.netty.Context"); try (ILocalCluster cluster = new LocalCluster.Builder().withSimulatedTime() - .withSupervisors(1).withPortsPerSupervisor(2) + .withSupervisors(1) + .withPortsPerSupervisor(2) .withDaemonConf(topoConf).build()) { TopologyBuilder builder = new TopologyBuilder(); @@ -55,21 +62,23 @@ public void testLocalTransport() throws Exception { MockedSources mockedSources = new MockedSources(data); CompleteTopologyParam completeTopologyParam = new CompleteTopologyParam(); completeTopologyParam.setMockedSources(mockedSources); - Map> results = Testing.completeTopology(cluster, stormTopology, completeTopologyParam); + Map> results = Testing.completeTopology(cluster, stormTopology, + completeTopologyParam); assertEquals(6 * 4, Testing.readTuples(results, "2").size()); } } @Test public void testRemoteTransportWithManyTasksInReceivingExecutor() throws Exception { - //STORM-3141 regression test - //Verify that remote worker can handle many tasks in one executor + // STORM-3141 regression test + // Verify that remote worker can handle many tasks in one executor Config topoConf = new Config(); topoConf.put(Config.TOPOLOGY_WORKERS, 2); topoConf.put(Config.STORM_MESSAGING_TRANSPORT, "org.apache.storm.messaging.netty.Context"); try (ILocalCluster cluster = new LocalCluster.Builder().withSimulatedTime() - .withSupervisors(1).withPortsPerSupervisor(2) + .withSupervisors(1) + .withPortsPerSupervisor(2) .withDaemonConf(topoConf).build()) { TopologyBuilder builder = new TopologyBuilder(); @@ -89,7 +98,8 @@ public void testRemoteTransportWithManyTasksInReceivingExecutor() throws Excepti MockedSources mockedSources = new MockedSources(data); CompleteTopologyParam completeTopologyParam = new CompleteTopologyParam(); completeTopologyParam.setMockedSources(mockedSources); - Map> results = Testing.completeTopology(cluster, stormTopology, completeTopologyParam); + Map> results = Testing.completeTopology(cluster, stormTopology, + completeTopologyParam); assertEquals(6 * 4, Testing.readTuples(results, "2").size()); } } diff --git a/storm-server/src/test/java/org/apache/storm/MockAutoCred.java b/storm-server/src/test/java/org/apache/storm/MockAutoCred.java index 87a63ad6f43..348525258ad 100644 --- a/storm-server/src/test/java/org/apache/storm/MockAutoCred.java +++ b/storm-server/src/test/java/org/apache/storm/MockAutoCred.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -19,8 +25,10 @@ import org.apache.storm.security.auth.ICredentialsRenewer; /** - * Mock implementation of INimbusCredentialPlugin, IAutoCredentials and ICredentialsRenewer for testing only. - * Duplicated from storm-core test sources since storm-server cannot depend on storm-core test classes. + * Mock implementation of INimbusCredentialPlugin, IAutoCredentials and ICredentialsRenewer for + * testing only. + * Duplicated from storm-core test sources since storm-server cannot depend on storm-core test + * classes. */ public class MockAutoCred implements INimbusCredentialPlugin, IAutoCredentials, ICredentialsRenewer { public static final String NIMBUS_CRED_KEY = "nimbusCredTestKey"; @@ -52,7 +60,8 @@ public void updateSubject(Subject subject, Map credentials) { } @Override - public void renew(Map credentials, Map topologyConf, String ownerPrincipal) { + public void renew(Map credentials, Map topologyConf, + String ownerPrincipal) { credentials.put(NIMBUS_CRED_KEY, NIMBUS_CRED_RENEW_VAL); credentials.put(GATEWAY_CRED_KEY, GATEWAY_CRED_RENEW_VAL); } diff --git a/storm-server/src/test/java/org/apache/storm/PacemakerTest.java b/storm-server/src/test/java/org/apache/storm/PacemakerTest.java index 9fd5505f8af..9d4dd6d1f39 100644 --- a/storm-server/src/test/java/org/apache/storm/PacemakerTest.java +++ b/storm-server/src/test/java/org/apache/storm/PacemakerTest.java @@ -1,17 +1,28 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + import java.io.UnsupportedEncodingException; import java.nio.charset.StandardCharsets; import java.util.List; @@ -27,11 +38,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; - @SuppressWarnings("deprecation") public class PacemakerTest { @@ -106,12 +112,14 @@ public void testServerSendPulseGetPulse() { HBMessage response = handler.handleMessage(hbMessage, true); assertEquals(mid, response.get_message_id()); assertEquals(HBServerMessageType.GET_PULSE_RESPONSE, response.get_type()); - assertEquals(dataString, new String(response.get_data().get_pulse().get_details(), StandardCharsets.UTF_8)); + assertEquals(dataString, new String(response.get_data().get_pulse().get_details(), + StandardCharsets.UTF_8)); } @Test public void testServerGetAllPulseForPath() { - messageWithRandId(HBServerMessageType.GET_ALL_PULSE_FOR_PATH, HBMessageData.path("/testpath")); + messageWithRandId(HBServerMessageType.GET_ALL_PULSE_FOR_PATH, HBMessageData + .path("/testpath")); HBMessage badResponse = handler.handleMessage(hbMessage, false); HBMessage goodResponse = handler.handleMessage(hbMessage, true); assertEquals(mid, badResponse.get_message_id()); @@ -128,7 +136,8 @@ public void testServerGetAllNodesForPath() throws UnsupportedEncodingException { makeNode(handler, "/some-root-path/bar"); makeNode(handler, "/some-root-path/baz"); makeNode(handler, "/some-root-path/boo"); - messageWithRandId(HBServerMessageType.GET_ALL_NODES_FOR_PATH, HBMessageData.path("/some-root-path")); + messageWithRandId(HBServerMessageType.GET_ALL_NODES_FOR_PATH, HBMessageData + .path("/some-root-path")); HBMessage badResponse = handler.handleMessage(hbMessage, false); HBMessage goodResponse = handler.handleMessage(hbMessage, true); List pulseIds = goodResponse.get_data().get_nodes().get_pulseIds(); @@ -147,7 +156,8 @@ public void testServerGetAllNodesForPath() throws UnsupportedEncodingException { makeNode(handler, "/some/deeper/path/foo"); makeNode(handler, "/some/deeper/path/bar"); makeNode(handler, "/some/deeper/path/baz"); - messageWithRandId(HBServerMessageType.GET_ALL_NODES_FOR_PATH, HBMessageData.path("/some/deeper/path")); + messageWithRandId(HBServerMessageType.GET_ALL_NODES_FOR_PATH, HBMessageData + .path("/some/deeper/path")); badResponse = handler.handleMessage(hbMessage, false); goodResponse = handler.handleMessage(hbMessage, true); pulseIds = goodResponse.get_data().get_nodes().get_pulseIds(); @@ -166,7 +176,8 @@ public void testServerGetAllNodesForPath() throws UnsupportedEncodingException { @Test public void testServerGetPulse() throws UnsupportedEncodingException { makeNode(handler, "/some-root/GET_PULSE"); - messageWithRandId(HBServerMessageType.GET_PULSE, HBMessageData.path("/some-root/GET_PULSE")); + messageWithRandId(HBServerMessageType.GET_PULSE, HBMessageData + .path("/some-root/GET_PULSE")); HBMessage badResponse = handler.handleMessage(hbMessage, false); HBMessage goodResponse = handler.handleMessage(hbMessage, true); HBPulse goodPulse = goodResponse.get_data().get_pulse(); @@ -187,13 +198,15 @@ public void testServerDeletePath() throws UnsupportedEncodingException { makeNode(handler, "/some-root/DELETE_PATH/baz"); makeNode(handler, "/some-root/DELETE_PATH/boo"); - messageWithRandId(HBServerMessageType.DELETE_PATH, HBMessageData.path("/some-root/DELETE_PATH")); + messageWithRandId(HBServerMessageType.DELETE_PATH, HBMessageData + .path("/some-root/DELETE_PATH")); HBMessage response = handler.handleMessage(hbMessage, true); assertEquals(mid, response.get_message_id()); assertEquals(HBServerMessageType.DELETE_PATH_RESPONSE, response.get_type()); assertNull(response.get_data()); - messageWithRandId(HBServerMessageType.GET_ALL_NODES_FOR_PATH, HBMessageData.path("/some-root/DELETE_PATH")); + messageWithRandId(HBServerMessageType.GET_ALL_NODES_FOR_PATH, HBMessageData + .path("/some-root/DELETE_PATH")); response = handler.handleMessage(hbMessage, true); List pulseIds = response.get_data().get_nodes().get_pulseIds(); assertEquals(mid, response.get_message_id()); @@ -208,13 +221,15 @@ public void testServerDeletePulseId() throws UnsupportedEncodingException { makeNode(handler, "/some-root/DELETE_PULSE_ID/baz"); makeNode(handler, "/some-root/DELETE_PULSE_ID/boo"); - messageWithRandId(HBServerMessageType.DELETE_PULSE_ID, HBMessageData.path("/some-root/DELETE_PULSE_ID/foo")); + messageWithRandId(HBServerMessageType.DELETE_PULSE_ID, HBMessageData + .path("/some-root/DELETE_PULSE_ID/foo")); HBMessage response = handler.handleMessage(hbMessage, true); assertEquals(mid, response.get_message_id()); assertEquals(HBServerMessageType.DELETE_PULSE_ID_RESPONSE, response.get_type()); assertNull(response.get_data()); - messageWithRandId(HBServerMessageType.GET_ALL_NODES_FOR_PATH, HBMessageData.path("/some-root/DELETE_PULSE_ID")); + messageWithRandId(HBServerMessageType.GET_ALL_NODES_FOR_PATH, HBMessageData + .path("/some-root/DELETE_PULSE_ID")); response = handler.handleMessage(hbMessage, true); List pulseIds = response.get_data().get_nodes().get_pulseIds(); assertEquals(mid, response.get_message_id()); @@ -232,7 +247,8 @@ private HBMessage makeNode(Pacemaker handler, String path) throws UnsupportedEnc HBPulse hbPulse = new HBPulse(); hbPulse.set_id(path); hbPulse.set_details("nothing".getBytes(StandardCharsets.UTF_8)); - HBMessage message = new HBMessage(HBServerMessageType.SEND_PULSE, HBMessageData.pulse(hbPulse)); + HBMessage message = new HBMessage(HBServerMessageType.SEND_PULSE, HBMessageData + .pulse(hbPulse)); return handler.handleMessage(message, true); } } diff --git a/storm-server/src/test/java/org/apache/storm/TestCgroups.java b/storm-server/src/test/java/org/apache/storm/TestCgroups.java index 9becdeda2e7..e041408154f 100644 --- a/storm-server/src/test/java/org/apache/storm/TestCgroups.java +++ b/storm-server/src/test/java/org/apache/storm/TestCgroups.java @@ -1,17 +1,28 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + import java.io.File; import java.io.IOException; import java.nio.file.Files; @@ -24,29 +35,29 @@ import org.apache.storm.utils.Utils; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assumptions.assumeTrue; - /** - * Unit tests for CGroups + * Unit tests for CGroups. */ public class TestCgroups { /** - * Test whether cgroups are setup up correctly for use. Also tests whether Cgroups produces the right command to + * Test whether cgroups are setup up correctly for use. Also tests whether Cgroups produces the + * right command to * start a worker and cleans up correctly after the worker is shutdown */ @Test public void testSetupAndTearDown() throws IOException { Config config = new Config(); config.putAll(Utils.readDefaultConfig()); - //We don't want to run the test is CGroups are not setup - assumeTrue(((boolean) config.get(DaemonConfig.STORM_RESOURCE_ISOLATION_PLUGIN_ENABLE)) == true,"Check if CGroups are setup"); + // We don't want to run the test is CGroups are not setup + assumeTrue(((boolean) config + .get(DaemonConfig.STORM_RESOURCE_ISOLATION_PLUGIN_ENABLE)) == true, "Check if " + + "CGroups are setup"); - assertTrue(stormCgroupHierarchyExists(config), "Check if STORM_CGROUP_HIERARCHY_DIR exists"); - assertTrue(stormCgroupSupervisorRootDirExists(config), "Check if STORM_SUPERVISOR_CGROUP_ROOTDIR exists"); + assertTrue(stormCgroupHierarchyExists(config), + "Check if STORM_CGROUP_HIERARCHY_DIR exists"); + assertTrue(stormCgroupSupervisorRootDirExists(config), + "Check if STORM_SUPERVISOR_CGROUP_ROOTDIR exists"); CgroupManager manager = new CgroupManager(); manager.prepare(config); @@ -59,15 +70,22 @@ public void testSetupAndTearDown() throws IOException { for (String entry : commandList) { command.append(entry).append(" "); } - String correctCommand1 = config.get(DaemonConfig.STORM_CGROUP_CGEXEC_CMD) + " -g memory,cpu:/" - + config.get(DaemonConfig.STORM_SUPERVISOR_CGROUP_ROOTDIR) + "/" + workerId + " "; - String correctCommand2 = config.get(DaemonConfig.STORM_CGROUP_CGEXEC_CMD) + " -g cpu,memory:/" - + config.get(DaemonConfig.STORM_SUPERVISOR_CGROUP_ROOTDIR) + "/" + workerId + " "; - assertTrue(command.toString().equals(correctCommand1) || command.toString().equals(correctCommand2), + String correctCommand1 = config.get(DaemonConfig.STORM_CGROUP_CGEXEC_CMD) + + " -g memory,cpu:/" + + config.get(DaemonConfig.STORM_SUPERVISOR_CGROUP_ROOTDIR) + "/" + + workerId + " "; + String correctCommand2 = config.get(DaemonConfig.STORM_CGROUP_CGEXEC_CMD) + + " -g cpu,memory:/" + + config.get(DaemonConfig.STORM_SUPERVISOR_CGROUP_ROOTDIR) + "/" + + workerId + " "; + assertTrue(command.toString().equals(correctCommand1) || command.toString() + .equals(correctCommand2), "Check if cgroup launch command is correct"); String pathToWorkerCgroupDir = config.get(Config.STORM_CGROUP_HIERARCHY_DIR) - + "/" + config.get(DaemonConfig.STORM_SUPERVISOR_CGROUP_ROOTDIR) + "/" + workerId; + + "/" + config + .get(DaemonConfig.STORM_SUPERVISOR_CGROUP_ROOTDIR) + "/" + + workerId; assertTrue(dirExists(pathToWorkerCgroupDir), "Check if cgroup directory exists for worker"); @@ -75,13 +93,15 @@ public void testSetupAndTearDown() throws IOException { String pathToCpuShares = pathToWorkerCgroupDir + "/cpu.shares"; assertTrue(fileExists(pathToCpuShares), "Check if cpu.shares file exists"); - assertEquals("200", readFileAll(pathToCpuShares), "Check if the correct value is written into cpu.shares"); + assertEquals("200", readFileAll(pathToCpuShares), + "Check if the correct value is written into cpu.shares"); /* validate memory settings */ String pathTomemoryLimitInBytes = pathToWorkerCgroupDir + "/memory.limit_in_bytes"; - assertTrue(fileExists(pathTomemoryLimitInBytes), "Check if memory.limit_in_bytes file exists"); + assertTrue(fileExists(pathTomemoryLimitInBytes), + "Check if memory.limit_in_bytes file exists"); assertEquals(String.valueOf(1024 * 1024 * 1024), readFileAll(pathTomemoryLimitInBytes), "Check if the correct value is written into memory.limit_in_bytes"); @@ -100,7 +120,8 @@ private boolean stormCgroupHierarchyExists(Map config) { private boolean stormCgroupSupervisorRootDirExists(Map config) { String pathTostormCgroupSupervisorRootDir = config.get(Config.STORM_CGROUP_HIERARCHY_DIR) - + "/" + config.get(DaemonConfig.STORM_SUPERVISOR_CGROUP_ROOTDIR); + + "/" + config + .get(DaemonConfig.STORM_SUPERVISOR_CGROUP_ROOTDIR); return dirExists(pathTostormCgroupSupervisorRootDir); } diff --git a/storm-server/src/test/java/org/apache/storm/TestDaemonConfigValidate.java b/storm-server/src/test/java/org/apache/storm/TestDaemonConfigValidate.java index 483bd148a4b..05aea7cdde5 100644 --- a/storm-server/src/test/java/org/apache/storm/TestDaemonConfigValidate.java +++ b/storm-server/src/test/java/org/apache/storm/TestDaemonConfigValidate.java @@ -18,6 +18,8 @@ package org.apache.storm; +import static org.junit.jupiter.api.Assertions.assertThrows; + import java.util.Arrays; import java.util.Collection; import java.util.HashMap; @@ -26,8 +28,6 @@ import org.apache.storm.validation.ConfigValidation; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertThrows; - public class TestDaemonConfigValidate { @Test @@ -77,7 +77,7 @@ public void testSupervisorSlotsPorts() { passCases.add(Arrays.asList(test2)); String[] test3 = { "1233", "1234", "1235" }; - //duplicate case + // duplicate case Integer[] test4 = { 1233, 1233, 1235 }; failCases.add(test3); failCases.add(test4); diff --git a/storm-server/src/test/java/org/apache/storm/TestRebalance.java b/storm-server/src/test/java/org/apache/storm/TestRebalance.java index f365e2b9d49..21e3a375be8 100644 --- a/storm-server/src/test/java/org/apache/storm/TestRebalance.java +++ b/storm-server/src/test/java/org/apache/storm/TestRebalance.java @@ -1,26 +1,33 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

      - * http://www.apache.org/licenses/LICENSE-2.0 - *

      - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

      http://www.apache.org/licenses/LICENSE-2.0 + * + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + import java.util.HashMap; import java.util.Map; - import net.minidev.json.JSONObject; import net.minidev.json.parser.JSONParser; - import org.apache.storm.generated.ClusterSummary; -import org.apache.storm.generated.RebalanceOptions; import org.apache.storm.generated.NotAliveException; +import org.apache.storm.generated.RebalanceOptions; import org.apache.storm.generated.StormTopology; import org.apache.storm.generated.TopologySummary; import org.apache.storm.scheduler.resource.ResourceAwareScheduler; @@ -36,21 +43,17 @@ import org.apache.storm.topology.SpoutDeclarer; import org.apache.storm.topology.TopologyBuilder; import org.apache.storm.utils.Utils; - import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - public class TestRebalance { private static final Class[] strategyClasses = { - DefaultResourceAwareStrategy.class, - DefaultResourceAwareStrategyOld.class, - RoundRobinResourceAwareStrategy.class, - GenericResourceAwareStrategy.class, - GenericResourceAwareStrategyOld.class, + DefaultResourceAwareStrategy.class, + DefaultResourceAwareStrategyOld.class, + RoundRobinResourceAwareStrategy.class, + GenericResourceAwareStrategy.class, + GenericResourceAwareStrategyOld.class, }; static final int SLEEP_TIME_BETWEEN_RETRY = 1000; @@ -75,7 +78,8 @@ public void testRebalanceTopologyResourcesAndConfigs() Config conf = new Config(); conf.put(DaemonConfig.STORM_SCHEDULER, ResourceAwareScheduler.class.getName()); - conf.put(DaemonConfig.RESOURCE_AWARE_SCHEDULER_PRIORITY_STRATEGY, DefaultSchedulingPriorityStrategy.class.getName()); + conf.put(DaemonConfig.RESOURCE_AWARE_SCHEDULER_PRIORITY_STRATEGY, + DefaultSchedulingPriorityStrategy.class.getName()); conf.put(Config.TOPOLOGY_SCHEDULER_STRATEGY, strategyClass.getName()); conf.put(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT, 10.0); conf.put(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB, 10.0); @@ -88,11 +92,14 @@ public void testRebalanceTopologyResourcesAndConfigs() try (ILocalCluster cluster = new LocalCluster.Builder().withDaemonConf(conf).build()) { TopologyBuilder builder = new TopologyBuilder(); - SpoutDeclarer s1 = builder.setSpout("spout-1", new TestUtilsForResourceAwareScheduler.TestSpout(), + SpoutDeclarer s1 = builder.setSpout("spout-1", + new TestUtilsForResourceAwareScheduler.TestSpout(), 2); - BoltDeclarer b1 = builder.setBolt("bolt-1", new TestUtilsForResourceAwareScheduler.TestBolt(), + BoltDeclarer b1 = builder.setBolt("bolt-1", new TestUtilsForResourceAwareScheduler + .TestBolt(), 2).shuffleGrouping("spout-1"); - BoltDeclarer b2 = builder.setBolt("bolt-2", new TestUtilsForResourceAwareScheduler.TestBolt(), + BoltDeclarer b2 = builder.setBolt("bolt-2", new TestUtilsForResourceAwareScheduler + .TestBolt(), 2).shuffleGrouping("bolt-1"); StormTopology stormTopology = builder.createTopology(); @@ -105,9 +112,11 @@ public void testRebalanceTopologyResourcesAndConfigs() RebalanceOptions opts = new RebalanceOptions(); - Map> resources = new HashMap>(); + Map> resources = + new HashMap>(); resources.put("spout-1", new HashMap()); - resources.get("spout-1").put(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB, 120.0); + resources.get("spout-1").put(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB, + 120.0); resources.get("spout-1").put(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT, 25.0); resources.get("spout-1").put("gpu.count", 5.0); @@ -140,33 +149,43 @@ public void testRebalanceTopologyResourcesAndConfigs() } } - StormTopology readStormTopology = cluster.getTopology(topoNameToId(topoName, cluster)); - String componentConfRaw = readStormTopology.get_spouts().get("spout-1").get_common().get_json_conf(); + StormTopology readStormTopology = cluster.getTopology(topoNameToId(topoName, + cluster)); + String componentConfRaw = readStormTopology.get_spouts().get("spout-1").get_common() + .get_json_conf(); JSONObject readTopologyConf = (JSONObject) parser.parse(componentConfRaw); - Map componentResources = (Map) readTopologyConf.get(Config.TOPOLOGY_COMPONENT_RESOURCES_MAP); + Map componentResources = (Map) readTopologyConf + .get(Config.TOPOLOGY_COMPONENT_RESOURCES_MAP); assertTrue(topologyUpdated, "Topology has been updated"); - assertEquals(25.0, componentResources.get(Constants.COMMON_CPU_RESOURCE_NAME), 0.001, "Updated CPU correct"); - assertEquals(120.0, componentResources.get(Constants.COMMON_ONHEAP_MEMORY_RESOURCE_NAME), 0.001, "Updated Memory correct"); - assertEquals(5.0, componentResources.get("gpu.count"), 0.001, "Updated Generic resource correct"); + assertEquals(25.0, componentResources.get(Constants.COMMON_CPU_RESOURCE_NAME), + 0.001, "Updated CPU correct"); + assertEquals(120.0, componentResources + .get(Constants.COMMON_ONHEAP_MEMORY_RESOURCE_NAME), 0.001, "Updated " + + "Memory correct"); + assertEquals(5.0, componentResources.get("gpu.count"), 0.001, + "Updated Generic resource correct"); } } } - public void waitTopologyScheduled(String topoName, ILocalCluster cluster, int retryAttempts) throws TException { + public void waitTopologyScheduled(String topoName, ILocalCluster cluster, + int retryAttempts) throws TException { for (int i = 0; i < retryAttempts; i++) { if (checkTopologyScheduled(topoName, cluster)) { - //sleep to prevent race conditions + // sleep to prevent race conditions Utils.sleep(SLEEP_TIME_BETWEEN_RETRY); return; } Utils.sleep(SLEEP_TIME_BETWEEN_RETRY); } - throw new RuntimeException("Error: Wait for topology " + topoName + " to be ACTIVE has timed out!"); + throw new RuntimeException("Error: Wait for topology " + topoName + + " to be ACTIVE has timed out!"); } - public boolean checkTopologyScheduled(String topoName, ILocalCluster cluster) throws TException { + public boolean checkTopologyScheduled(String topoName, + ILocalCluster cluster) throws TException { if (checkTopologyUp(topoName, cluster)) { TopologySummary topoSum = cluster.getTopologySummaryByName(topoName); String status = topoSum.get_status(); @@ -181,9 +200,9 @@ public boolean checkTopologyScheduled(String topoName, ILocalCluster cluster) th public boolean checkTopologyUp(String topoName, ILocalCluster cluster) throws TException { ClusterSummary sum = cluster.getClusterInfo(); TopologySummary topoSum = cluster.getTopologySummaryByName(topoName); - if (topoSum != null) { - return true; - } + if (topoSum != null) { + return true; + } return false; } } diff --git a/storm-server/src/test/java/org/apache/storm/TestingTest.java b/storm-server/src/test/java/org/apache/storm/TestingTest.java index 769fc28a881..c136a890ead 100644 --- a/storm-server/src/test/java/org/apache/storm/TestingTest.java +++ b/storm-server/src/test/java/org/apache/storm/TestingTest.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -31,6 +37,7 @@ import org.apache.storm.tuple.Fields; import org.apache.storm.tuple.Values; import org.junit.jupiter.api.Test; + /** * Test that the testing class does what it should do. */ @@ -41,11 +48,11 @@ public class TestingTest { TopologyBuilder tb = new TopologyBuilder(); tb.setSpout("spout", new TestWordSpout(true), 3); tb.setBolt("2", new TestWordCounter(), 4) - .fieldsGrouping("spout", new Fields("word")); + .fieldsGrouping("spout", new Fields("word")); tb.setBolt("3", new TestGlobalCount()) - .globalGrouping("spout"); + .globalGrouping("spout"); tb.setBolt("4", new TestAggregatesCounter()) - .globalGrouping("2"); + .globalGrouping("2"); MockedSources mocked = new MockedSources(); mocked.addMockData("spout", @@ -61,26 +68,34 @@ public class TestingTest { ctp.setMockedSources(mocked); ctp.setStormConf(topoConf); - Map> results = Testing.completeTopology(cluster, tb.createTopology(), ctp); + Map> results = Testing.completeTopology(cluster, tb + .createTopology(), ctp); List> spoutTuples = Testing.readTuples(results, "spout"); - List> expectedSpoutTuples = Arrays.asList(Arrays.asList("nathan"), Arrays.asList("bob"), Arrays.asList("joey"), + List> expectedSpoutTuples = Arrays.asList(Arrays.asList("nathan"), Arrays + .asList("bob"), Arrays.asList("joey"), Arrays.asList("nathan")); - assertTrue(Testing.multiseteq(expectedSpoutTuples, spoutTuples), expectedSpoutTuples + " expected, but found " + spoutTuples); + assertTrue(Testing.multiseteq(expectedSpoutTuples, spoutTuples), expectedSpoutTuples + + " expected, but found " + spoutTuples); List> twoTuples = Testing.readTuples(results, "2"); - List> expectedTwoTuples = Arrays.asList(Arrays.asList("nathan", 1), Arrays.asList("nathan", 2), - Arrays.asList("bob", 1), Arrays.asList("joey", 1)); - assertTrue(Testing.multiseteq(expectedTwoTuples, twoTuples), expectedTwoTuples + " expected, but found " + twoTuples); + List> expectedTwoTuples = Arrays.asList(Arrays.asList("nathan", 1), Arrays + .asList("nathan", 2), + Arrays.asList("bob", 1), Arrays + .asList("joey", 1)); + assertTrue(Testing.multiseteq(expectedTwoTuples, twoTuples), expectedTwoTuples + + " expected, but found " + twoTuples); List> threeTuples = Testing.readTuples(results, "3"); List> expectedThreeTuples = Arrays.asList(Arrays.asList(1), Arrays.asList(2), Arrays.asList(3), Arrays.asList(4)); - assertTrue(Testing.multiseteq(expectedThreeTuples, threeTuples), expectedThreeTuples + " expected, but found " + threeTuples); + assertTrue(Testing.multiseteq(expectedThreeTuples, threeTuples), expectedThreeTuples + + " expected, but found " + threeTuples); List> fourTuples = Testing.readTuples(results, "4"); List> expectedFourTuples = Arrays.asList(Arrays.asList(1), Arrays.asList(2), Arrays.asList(3), Arrays.asList(4)); - assertTrue(Testing.multiseteq(expectedFourTuples, fourTuples), expectedFourTuples + " expected, but found " + fourTuples); + assertTrue(Testing.multiseteq(expectedFourTuples, fourTuples), expectedFourTuples + + " expected, but found " + fourTuples); }; @Test diff --git a/storm-server/src/test/java/org/apache/storm/TickTupleTest.java b/storm-server/src/test/java/org/apache/storm/TickTupleTest.java index c36d96df32a..7e5cff165aa 100644 --- a/storm-server/src/test/java/org/apache/storm/TickTupleTest.java +++ b/storm-server/src/test/java/org/apache/storm/TickTupleTest.java @@ -1,17 +1,26 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertNull; + import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -30,23 +39,21 @@ import org.apache.storm.tuple.Values; import org.apache.storm.utils.Time; import org.apache.storm.utils.TupleUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.awaitility.Awaitility; import org.awaitility.core.ConditionTimeoutException; import org.hamcrest.Matchers; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.junit.jupiter.api.Assertions.assertNull; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class TickTupleTest { - private final static Logger LOG = LoggerFactory.getLogger(TickTupleTest.class); + private static final Logger LOG = LoggerFactory.getLogger(TickTupleTest.class); private static final AtomicInteger tickTupleCount = new AtomicInteger(); private static final AtomicReference nonTickTuple = new AtomicReference<>(null); private static final AtomicBoolean receivedAnyTuple = new AtomicBoolean(); - //This needs to be appropriately large to drown out any time advances performed during topology boot + // This needs to be appropriately large to drown out any time advances performed during topology + // boot private static final int TICK_INTERVAL_SECS = 30; @AfterEach @@ -74,13 +81,16 @@ public void testTickTupleWorksWithSystemBolt() throws Exception { Config topoConf = new Config(); topoConf.put(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, TICK_INTERVAL_SECS); - try (ILocalTopology ignored = cluster.submitTopology("test", topoConf, builder.createTopology())) { - //Use a bootstrap tuple to wait for topology to be running + try (ILocalTopology ignored = cluster.submitTopology("test", topoConf, builder + .createTopology())) { + // Use a bootstrap tuple to wait for topology to be running feeder.feed(new Values("val"), 1); AssertLoop.assertAcked(tracker, 1); /* - * Verify that some ticks are received. The interval between ticks is validated by the bolt. - * Too few and the checks will time out. Too many and the bolt may crash (not reliably, but the test should become flaky). + * Verify that some ticks are received. The interval between ticks is validated by + * the bolt. + * Too few and the checks will time out. Too many and the bolt may crash (not + * reliably, but the test should become flaky). */ try { cluster.advanceClusterTime(TICK_INTERVAL_SECS); @@ -92,7 +102,8 @@ public void testTickTupleWorksWithSystemBolt() throws Exception { } catch (ConditionTimeoutException e) { throw new AssertionError(e.getMessage()); } - assertNull(nonTickTuple.get(), "The bolt got a tuple that is not a tick tuple " + nonTickTuple.get()); + assertNull(nonTickTuple.get(), "The bolt got a tuple that is not a tick tuple " + + nonTickTuple.get()); } } } @@ -102,7 +113,8 @@ private void waitForTicks(int minTicks) { Awaitility.with() .pollInterval(1, TimeUnit.MILLISECONDS) .atMost(Testing.TEST_TIMEOUT_MS, TimeUnit.MILLISECONDS) - .untilAsserted(() -> assertThat(tickTupleCount.get(), Matchers.greaterThanOrEqualTo(minTicks))); + .untilAsserted(() -> assertThat(tickTupleCount.get(), Matchers + .greaterThanOrEqualTo(minTicks))); } catch (ConditionTimeoutException e) { throw new AssertionError(e.getMessage()); } @@ -112,7 +124,8 @@ private static class NoopBolt extends BaseRichBolt { private OutputCollector collector; @Override - public void prepare(Map conf, TopologyContext topologyContext, OutputCollector outputCollector) { + public void prepare(Map conf, TopologyContext topologyContext, + OutputCollector outputCollector) { collector = outputCollector; } @@ -120,8 +133,10 @@ public void prepare(Map conf, TopologyContext topologyContext, O public void execute(Tuple tuple) { LOG.info("GOT {} at time {}", tuple, Time.currentTimeMillis()); if (!receivedAnyTuple.get() && Time.currentTimeSecs() > TICK_INTERVAL_SECS) { - throw new RuntimeException("Simulated time was higher than " + TICK_INTERVAL_SECS + " at start of test." - + " Increase the interval until this no longer occurs, but keep an eye on Storm's timeouts for e.g. worker heartbeat."); + throw new RuntimeException("Simulated time was higher than " + TICK_INTERVAL_SECS + + " at start of test." + + " Increase the interval until this no longer occurs, but keep an eye on " + + "Storm's timeouts for e.g. worker heartbeat."); } receivedAnyTuple.set(true); if (tickTupleCount.get() > 3) { @@ -140,7 +155,7 @@ public void execute(Tuple tuple) { } @Override - public void cleanup() { } + public void cleanup() {} @Override public void declareOutputFields(OutputFieldsDeclarer ofd) {} diff --git a/storm-server/src/test/java/org/apache/storm/blobstore/BlobStoreUtilsTest.java b/storm-server/src/test/java/org/apache/storm/blobstore/BlobStoreUtilsTest.java index 24fb1b59bab..d6364c8126a 100644 --- a/storm-server/src/test/java/org/apache/storm/blobstore/BlobStoreUtilsTest.java +++ b/storm-server/src/test/java/org/apache/storm/blobstore/BlobStoreUtilsTest.java @@ -1,24 +1,46 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.blobstore; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyMap; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + import java.io.IOException; import java.io.InputStream; import java.util.Collections; import java.util.List; import java.util.Map; - -import org.apache.storm.generated.*; +import javax.security.auth.Subject; +import org.apache.storm.generated.BeginDownloadResult; +import org.apache.storm.generated.KeyAlreadyExistsException; +import org.apache.storm.generated.Nimbus; +import org.apache.storm.generated.ReadableBlobMeta; +import org.apache.storm.generated.SettableBlobMeta; import org.apache.storm.nimbus.NimbusInfo; import org.apache.storm.thrift.TException; import org.apache.storm.utils.NimbusClient; @@ -26,22 +48,6 @@ import org.mockito.MockedStatic; import org.mockito.Mockito; -import javax.security.auth.Subject; - -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyInt; -import static org.mockito.ArgumentMatchers.anyMap; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.when; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.atLeastOnce; - - - public class BlobStoreUtilsTest { private static final String KEY = "key"; @@ -72,7 +78,8 @@ public void testUpdateKeyForBlobStore_nullNimbusInfo() { @Test public void testUpdateKeyForBlobStore_missingNode() { zkClientBuilder.withExists(BLOBSTORE_KEY, false); - BlobStoreUtils.updateKeyForBlobStore(conf, blobStore, zkClientBuilder.build(), KEY, nimbusDetails); + BlobStoreUtils.updateKeyForBlobStore(conf, blobStore, zkClientBuilder.build(), KEY, + nimbusDetails); zkClientBuilder.verifyExists(true); zkClientBuilder.verifyGetChildren(false); @@ -88,7 +95,8 @@ public void testUpdateKeyForBlobStore_missingNode() { public void testUpdateKeyForBlobStore_nodeWithNullChildren() { zkClientBuilder.withExists(BLOBSTORE_KEY, true); zkClientBuilder.withGetChildren(BLOBSTORE_KEY, (List) null); - BlobStoreUtils.updateKeyForBlobStore(conf, blobStore, zkClientBuilder.build(), KEY, nimbusDetails); + BlobStoreUtils.updateKeyForBlobStore(conf, blobStore, zkClientBuilder.build(), KEY, + nimbusDetails); zkClientBuilder.verifyExists(true); zkClientBuilder.verifyGetChildren(); @@ -103,7 +111,8 @@ public void testUpdateKeyForBlobStore_nodeWithNullChildren() { public void testUpdateKeyForBlobStore_nodeWithEmptyChildren() { zkClientBuilder.withExists(BLOBSTORE_KEY, true); zkClientBuilder.withGetChildren(BLOBSTORE_KEY); - BlobStoreUtils.updateKeyForBlobStore(conf, blobStore, zkClientBuilder.build(), KEY, nimbusDetails); + BlobStoreUtils.updateKeyForBlobStore(conf, blobStore, zkClientBuilder.build(), KEY, + nimbusDetails); zkClientBuilder.verifyExists(true); zkClientBuilder.verifyGetChildren(); @@ -120,7 +129,8 @@ public void testUpdateKeyForBlobStore_hostsMatch() { zkClientBuilder.withExists(BLOBSTORE_KEY, true); zkClientBuilder.withGetChildren(BLOBSTORE_KEY, "localhost:1111-1"); when(nimbusDetails.getHost()).thenReturn("localhost"); - BlobStoreUtils.updateKeyForBlobStore(conf, blobStore, zkClientBuilder.build(), KEY, nimbusDetails); + BlobStoreUtils.updateKeyForBlobStore(conf, blobStore, zkClientBuilder.build(), KEY, + nimbusDetails); zkClientBuilder.verifyExists(true); zkClientBuilder.verifyGetChildren(2); @@ -137,7 +147,8 @@ public void testUpdateKeyForBlobStore_noMatch() { zkClientBuilder.withExists(BLOBSTORE_KEY, true); zkClientBuilder.withGetChildren(BLOBSTORE_KEY, "localhost:1111-1"); when(nimbusDetails.getHost()).thenReturn("no match"); - BlobStoreUtils.updateKeyForBlobStore(conf, blobStore, zkClientBuilder.build(), KEY, nimbusDetails); + BlobStoreUtils.updateKeyForBlobStore(conf, blobStore, zkClientBuilder.build(), KEY, + nimbusDetails); zkClientBuilder.verifyExists(true); zkClientBuilder.verifyGetChildren(2); @@ -155,11 +166,13 @@ public void testDownloadMissingBlob_KeyAkreadyExists() throws TException, IOExce Nimbus.Iface iface = mock(Nimbus.Iface.class); - try (MockedStatic mockedNimbusClient = Mockito.mockStatic(NimbusClient.Builder.class)) { + try (MockedStatic mockedNimbusClient = Mockito + .mockStatic(NimbusClient.Builder.class)) { - mockedNimbusClient.when(() ->NimbusClient.Builder.withConf(anyMap())).thenReturn(builder1); + mockedNimbusClient.when(() -> NimbusClient.Builder.withConf(anyMap())) + .thenReturn(builder1); when(builder1.forDaemon()).thenReturn(builder2); - when(builder2.buildWithNimbusHostPort(anyString(),anyInt())).thenReturn(client); + when(builder2.buildWithNimbusHostPort(anyString(), anyInt())).thenReturn(client); when(client.getClient()).thenReturn(iface); when(iface.getBlobMeta(anyString())).thenReturn(readableBlobMeta); when(readableBlobMeta.get_settable()).thenReturn(new SettableBlobMeta()); @@ -167,9 +180,11 @@ public void testDownloadMissingBlob_KeyAkreadyExists() throws TException, IOExce when(nimbusDetails.getHost()).thenReturn("localhost"); when(nimbusDetails.getPort()).thenReturn(1234); - doThrow(new KeyAlreadyExistsException()).when(blobStore).createBlob(anyString(),any(InputStream.class),any(SettableBlobMeta.class),any(Subject.class)); + doThrow(new KeyAlreadyExistsException()).when(blobStore).createBlob(anyString(), + any(InputStream.class), any(SettableBlobMeta.class), any(Subject.class)); - assertTrue((BlobStoreUtils.downloadMissingBlob(conf, blobStore, "testKey", Collections.singleton(nimbusDetails)))); + assertTrue((BlobStoreUtils.downloadMissingBlob(conf, blobStore, "testKey", Collections + .singleton(nimbusDetails)))); } } diff --git a/storm-server/src/test/java/org/apache/storm/blobstore/KeySequenceNumberTest.java b/storm-server/src/test/java/org/apache/storm/blobstore/KeySequenceNumberTest.java index 8b4e8a08ba5..e9b0a8f810d 100644 --- a/storm-server/src/test/java/org/apache/storm/blobstore/KeySequenceNumberTest.java +++ b/storm-server/src/test/java/org/apache/storm/blobstore/KeySequenceNumberTest.java @@ -38,19 +38,23 @@ class KeySequenceNumberTest { private static final NimbusInfo PEER = new NimbusInfo("nimbus-2", 6627, false); /** - * Replays the blob store state changes behind the nimbus logs reported on STORM-3871. A non-leader that downloaded - * a blob while the leader deleted it registered the key again as new, and every other nimbus, the leader included, + * Replays the blob store state changes behind the nimbus logs reported on STORM-3871. A + * non-leader that downloaded + * a blob while the leader deleted it registered the key again as new, and every other nimbus, + * the leader included, * then downloaded the blob back from it. */ @Test void aNonLeaderCannotRegisterAKeyAgainThatWasDeletedWhileItDownloadedIt() throws Exception { try (InProcessZookeeper zk = new InProcessZookeeper(); CuratorFramework zkClient = newClient(zk)) { - // the client uploads the dependency: createBlob, then createStateInZookeeper when it closes the stream + // the client uploads the dependency: createBlob, then createStateInZookeeper when it + // closes the stream assertEquals(1, register(zkClient, LEADER, true)); assertEquals(2, register(zkClient, LEADER, true)); - // the topology is cleaned up and the leader deletes the blob, as LocalFsBlobStore#deleteBlob does + // the topology is cleaned up and the leader deletes the blob, as + // LocalFsBlobStore#deleteBlob does zkClient.delete().deletingChildrenIfNeeded().forPath(KEY_PATH); zkClient.delete().deletingChildrenIfNeeded().forPath(MAX_SEQUENCE_PATH); @@ -81,8 +85,10 @@ private static CuratorFramework newClient(InProcessZookeeper zk) { /** * Do what IStormClusterState#setupBlob does with the version KeySequenceNumber hands out. */ - private static int register(CuratorFramework zkClient, NimbusInfo nimbus, boolean mayCreateKey) throws Exception { - int version = new KeySequenceNumber(KEY, nimbus).getKeySequenceNumber(zkClient, mayCreateKey); + private static int register(CuratorFramework zkClient, NimbusInfo nimbus, + boolean mayCreateKey) throws Exception { + int version = new KeySequenceNumber(KEY, nimbus).getKeySequenceNumber(zkClient, + mayCreateKey); if (zkClient.checkExists().forPath(KEY_PATH) != null) { for (String child : zkClient.getChildren().forPath(KEY_PATH)) { if (child.startsWith(nimbus.toHostPortString())) { @@ -90,7 +96,8 @@ private static int register(CuratorFramework zkClient, NimbusInfo nimbus, boolea } } } - zkClient.create().creatingParentsIfNeeded().forPath(KEY_PATH + "/" + nimbus.toHostPortString() + "-" + version); + zkClient.create().creatingParentsIfNeeded().forPath(KEY_PATH + "/" + nimbus + .toHostPortString() + "-" + version); return version; } } diff --git a/storm-server/src/test/java/org/apache/storm/blobstore/LocalFsBlobStoreFileTest.java b/storm-server/src/test/java/org/apache/storm/blobstore/LocalFsBlobStoreFileTest.java index 2faae2a5185..aa0ec4f19f6 100644 --- a/storm-server/src/test/java/org/apache/storm/blobstore/LocalFsBlobStoreFileTest.java +++ b/storm-server/src/test/java/org/apache/storm/blobstore/LocalFsBlobStoreFileTest.java @@ -1,29 +1,34 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.blobstore; -import org.apache.commons.io.FileUtils; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.nio.file.Files; import java.util.zip.CRC32C; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotEquals; +import org.apache.commons.io.FileUtils; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; class LocalFsBlobStoreFileTest { @@ -38,14 +43,15 @@ public void setUp() throws IOException { fs.write("Content for checksum".getBytes()); } blobStoreFile = new LocalFsBlobStoreFile(tempFile.getParentFile(), tempFile.getName()); - checksumAlgorithm= new CRC32C(); + checksumAlgorithm = new CRC32C(); } @Test void testGetVersion() throws IOException { long expectedVersion = FileUtils.checksum(tempFile, checksumAlgorithm).getValue(); long actualVersion = blobStoreFile.getVersion(); - assertEquals(expectedVersion, actualVersion, "The version should match the expected checksum value."); + assertEquals(expectedVersion, actualVersion, + "The version should match the expected checksum value."); } @Test @@ -55,13 +61,15 @@ void testGetVersion_Mismatch() throws IOException { fs.write("Different content".getBytes()); } long actualVersion = blobStoreFile.getVersion(); - assertNotEquals(expectedVersion, actualVersion, "The version shouldn't match the checksum value of different content."); + assertNotEquals(expectedVersion, actualVersion, + "The version shouldn't match the checksum value of different content."); } @Test void testGetModTime() throws IOException { long expectedModTime = tempFile.lastModified(); long actualModTime = blobStoreFile.getModTime(); - assertEquals(expectedModTime, actualModTime, "The modification time should match the expected value."); + assertEquals(expectedModTime, actualModTime, + "The modification time should match the expected value."); } } diff --git a/storm-server/src/test/java/org/apache/storm/blobstore/LocalFsBlobStoreSynchronizerTest.java b/storm-server/src/test/java/org/apache/storm/blobstore/LocalFsBlobStoreSynchronizerTest.java index f1f9473958e..66b47c1724f 100644 --- a/storm-server/src/test/java/org/apache/storm/blobstore/LocalFsBlobStoreSynchronizerTest.java +++ b/storm-server/src/test/java/org/apache/storm/blobstore/LocalFsBlobStoreSynchronizerTest.java @@ -1,17 +1,26 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.blobstore; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + import java.io.File; import java.io.IOException; import java.util.ArrayList; @@ -29,19 +38,16 @@ import org.apache.storm.shade.org.apache.curator.framework.CuratorFramework; import org.apache.storm.shade.org.apache.curator.framework.CuratorFrameworkFactory; import org.apache.storm.shade.org.apache.curator.retry.ExponentialBackoffRetry; -import org.apache.storm.utils.Utils; import org.apache.storm.testing.InProcessZookeeper; +import org.apache.storm.utils.Utils; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - /** * Unit tests for most of the testable utility methods - * and LocalFsBlobStoreSynchronizer class methods + * and LocalFsBlobStoreSynchronizer class methods. */ public class LocalFsBlobStoreSynchronizerTest { private static Map conf = new HashMap<>(); @@ -73,52 +79,60 @@ private LocalFsBlobStore initLocalFs() { Map conf = new HashMap<>(this.conf); conf.putAll(Utils.readStormConfig()); conf.put(Config.STORM_LOCAL_DIR, baseFile.getAbsolutePath()); - conf.put(Config.STORM_PRINCIPAL_TO_LOCAL_PLUGIN, "org.apache.storm.security.auth.DefaultPrincipalToLocal"); + conf.put(Config.STORM_PRINCIPAL_TO_LOCAL_PLUGIN, + "org.apache.storm.security.auth.DefaultPrincipalToLocal"); conf.put(Config.STORM_ZOOKEEPER_PORT, zk.getPort()); store.prepare(conf, null, null, null); return store; } - @Test + @Test public void testBlobSynchronizerForKeysToDownload() { - BlobStore store = initLocalFs(); - LocalFsBlobStoreSynchronizer sync = new LocalFsBlobStoreSynchronizer(store, conf); - // test for keylist to download - Set zkSet = new HashSet<>(); - zkSet.add("key1"); - Set blobStoreSet = new HashSet<>(); - blobStoreSet.add("key1"); - Set resultSet = sync.getKeySetToDownload(blobStoreSet, zkSet); - assertTrue(resultSet.isEmpty(), "Not Empty"); - zkSet.add("key1"); - blobStoreSet.add("key2"); - resultSet = sync.getKeySetToDownload(blobStoreSet, zkSet); - assertTrue(resultSet.isEmpty(), "Not Empty"); - blobStoreSet.remove("key1"); - blobStoreSet.remove("key2"); - zkSet.add("key1"); - resultSet = sync.getKeySetToDownload(blobStoreSet, zkSet); - assertTrue((resultSet.size() == 1) && (resultSet.contains("key1")), "Unexpected keys to download"); - } + BlobStore store = initLocalFs(); + LocalFsBlobStoreSynchronizer sync = new LocalFsBlobStoreSynchronizer(store, conf); + // test for keylist to download + Set zkSet = new HashSet<>(); + zkSet.add("key1"); + Set blobStoreSet = new HashSet<>(); + blobStoreSet.add("key1"); + Set resultSet = sync.getKeySetToDownload(blobStoreSet, zkSet); + assertTrue(resultSet.isEmpty(), "Not Empty"); + zkSet.add("key1"); + blobStoreSet.add("key2"); + resultSet = sync.getKeySetToDownload(blobStoreSet, zkSet); + assertTrue(resultSet.isEmpty(), "Not Empty"); + blobStoreSet.remove("key1"); + blobStoreSet.remove("key2"); + zkSet.add("key1"); + resultSet = sync.getKeySetToDownload(blobStoreSet, zkSet); + assertTrue((resultSet.size() == 1) && (resultSet.contains("key1")), + "Unexpected keys to download"); + } @Test public void testGetLatestSequenceNumber() { List stateInfoList = new ArrayList<>(); stateInfoList.add("nimbus1:8000-2"); stateInfoList.add("nimbus-1:8000-4"); - assertEquals(4, BlobStoreUtils.getLatestSequenceNumber(stateInfoList), "Failed to get the latest version"); + assertEquals(4, BlobStoreUtils.getLatestSequenceNumber(stateInfoList), + "Failed to get the latest version"); } @Test public void testNimbodesWithLatestVersionOfBlob() throws Exception { - try (TestingServer server = new TestingServer(); CuratorFramework zkClient = CuratorFrameworkFactory + try (TestingServer server = new TestingServer(); CuratorFramework zkClient = + CuratorFrameworkFactory .newClient(server.getConnectString(), new ExponentialBackoffRetry(1000, 3))) { zkClient.start(); // Creating nimbus hosts containing the latest version of blob - zkClient.create().creatingParentContainersIfNeeded().forPath("/blobstore/key1/nimbus1:7800-1"); - zkClient.create().creatingParentContainersIfNeeded().forPath("/blobstore/key1/nimbus2:7800-2"); - Set set = BlobStoreUtils.getNimbodesWithLatestSequenceNumberOfBlob(zkClient, "key1"); - assertEquals("nimbus2", (set.iterator().next()).getHost(), "Failed to get the correct nimbus hosts with latest blob version"); + zkClient.create().creatingParentContainersIfNeeded() + .forPath("/blobstore/key1/nimbus1:7800-1"); + zkClient.create().creatingParentContainersIfNeeded() + .forPath("/blobstore/key1/nimbus2:7800-2"); + Set set = BlobStoreUtils.getNimbodesWithLatestSequenceNumberOfBlob(zkClient, + "key1"); + assertEquals("nimbus2", (set.iterator().next()).getHost(), + "Failed to get the correct nimbus hosts with latest blob version"); zkClient.delete().deletingChildrenIfNeeded().forPath("/blobstore/key1/nimbus1:7800-1"); zkClient.delete().deletingChildrenIfNeeded().forPath("/blobstore/key1/nimbus2:7800-2"); } @@ -126,10 +140,12 @@ public void testNimbodesWithLatestVersionOfBlob() throws Exception { @Test public void testNormalizeVersionInfo() { - BlobKeySequenceInfo info1 = BlobStoreUtils.normalizeNimbusHostPortSequenceNumberInfo("nimbus1:7800-1"); + BlobKeySequenceInfo info1 = BlobStoreUtils + .normalizeNimbusHostPortSequenceNumberInfo("nimbus1:7800-1"); assertEquals("nimbus1:7800", info1.getNimbusHostPort()); assertEquals("1", info1.getSequenceNumber()); - BlobKeySequenceInfo info2 = BlobStoreUtils.normalizeNimbusHostPortSequenceNumberInfo("nimbus-1:7800-1"); + BlobKeySequenceInfo info2 = BlobStoreUtils + .normalizeNimbusHostPortSequenceNumberInfo("nimbus-1:7800-1"); assertEquals("nimbus-1:7800", info2.getNimbusHostPort()); assertEquals("1", info2.getSequenceNumber()); } diff --git a/storm-server/src/test/java/org/apache/storm/blobstore/LocalFsBlobStoreTest.java b/storm-server/src/test/java/org/apache/storm/blobstore/LocalFsBlobStoreTest.java index 51f392725a6..e2ff1f3607b 100644 --- a/storm-server/src/test/java/org/apache/storm/blobstore/LocalFsBlobStoreTest.java +++ b/storm-server/src/test/java/org/apache/storm/blobstore/LocalFsBlobStoreTest.java @@ -1,17 +1,28 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.blobstore; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; +import static org.mockito.Mockito.spy; + import java.io.File; import java.io.IOException; import java.io.InputStream; @@ -43,103 +54,99 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; -import static org.mockito.Mockito.spy; - public class LocalFsBlobStoreTest { - private static final Logger LOG = LoggerFactory.getLogger(LocalFsBlobStoreTest.class); - URI base; - File baseFile; - private static Map conf = new HashMap<>(); - private InProcessZookeeper zk; + private static final Logger LOG = LoggerFactory.getLogger(LocalFsBlobStoreTest.class); + URI base; + File baseFile; + private static Map conf = new HashMap<>(); + private InProcessZookeeper zk; - @BeforeEach + @BeforeEach public void init() { - initializeConfigs(); - baseFile = new File("target/blob-store-test-"+UUID.randomUUID()); - base = baseFile.toURI(); - try { - zk = new InProcessZookeeper(); - } catch (Exception e) { - throw new RuntimeException(e); + initializeConfigs(); + baseFile = new File("target/blob-store-test-" + UUID.randomUUID()); + base = baseFile.toURI(); + try { + zk = new InProcessZookeeper(); + } catch (Exception e) { + throw new RuntimeException(e); + } } - } - @AfterEach + @AfterEach public void cleanup() throws IOException { - FileUtils.deleteDirectory(baseFile); - try { - zk.close(); - } catch (Exception e) { - throw new RuntimeException(e); + FileUtils.deleteDirectory(baseFile); + try { + zk.close(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + // Method which initializes nimbus admin + public static void initializeConfigs() { + conf.put(Config.NIMBUS_ADMINS, "admin"); + conf.put(Config.NIMBUS_SUPERVISOR_USERS, "supervisor"); + } + + private LocalFsBlobStore initLocalFs() { + LocalFsBlobStore store = new LocalFsBlobStore(); + // Spy object that tries to mock the real object store + LocalFsBlobStore spy = spy(store); + Mockito.doNothing().when(spy).checkForBlobUpdate("test"); + Mockito.doNothing().when(spy).checkForBlobUpdate("other"); + Mockito.doNothing().when(spy).checkForBlobUpdate("test-empty-subject-WE"); + Mockito.doNothing().when(spy).checkForBlobUpdate("test-empty-subject-DEF"); + Mockito.doNothing().when(spy).checkForBlobUpdate("test-empty-acls"); + Map conf = Utils.readStormConfig(); + conf.put(Config.STORM_ZOOKEEPER_PORT, zk.getPort()); + conf.put(Config.STORM_LOCAL_DIR, baseFile.getAbsolutePath()); + conf.put(Config.STORM_PRINCIPAL_TO_LOCAL_PLUGIN, + "org.apache.storm.security.auth.DefaultPrincipalToLocal"); + NimbusInfo nimbusInfo = new NimbusInfo("localhost", 0, false); + spy.prepare(conf, null, nimbusInfo, null); + return spy; } - } - - // Method which initializes nimbus admin - public static void initializeConfigs() { - conf.put(Config.NIMBUS_ADMINS,"admin"); - conf.put(Config.NIMBUS_SUPERVISOR_USERS,"supervisor"); - } - - private LocalFsBlobStore initLocalFs() { - LocalFsBlobStore store = new LocalFsBlobStore(); - // Spy object that tries to mock the real object store - LocalFsBlobStore spy = spy(store); - Mockito.doNothing().when(spy).checkForBlobUpdate("test"); - Mockito.doNothing().when(spy).checkForBlobUpdate("other"); - Mockito.doNothing().when(spy).checkForBlobUpdate("test-empty-subject-WE"); - Mockito.doNothing().when(spy).checkForBlobUpdate("test-empty-subject-DEF"); - Mockito.doNothing().when(spy).checkForBlobUpdate("test-empty-acls"); - Map conf = Utils.readStormConfig(); - conf.put(Config.STORM_ZOOKEEPER_PORT, zk.getPort()); - conf.put(Config.STORM_LOCAL_DIR, baseFile.getAbsolutePath()); - conf.put(Config.STORM_PRINCIPAL_TO_LOCAL_PLUGIN,"org.apache.storm.security.auth.DefaultPrincipalToLocal"); - NimbusInfo nimbusInfo = new NimbusInfo("localhost", 0, false); - spy.prepare(conf, null, nimbusInfo, null); - return spy; - } - - @Test + + @Test public void testLocalFsWithAuth() throws Exception { - testWithAuthentication(initLocalFs()); - } + testWithAuthentication(initLocalFs()); + } - @Test + @Test public void testBasicLocalFs() throws Exception { - testBasic(initLocalFs()); - } + testBasic(initLocalFs()); + } - @Test + @Test public void testMultipleLocalFs() throws Exception { - testMultiple(initLocalFs()); - } + testMultiple(initLocalFs()); + } - @Test - public void testDeleteAfterFailedCreate() throws Exception{ - //Check that a blob can be deleted when a temporary file exists in the blob directory - LocalFsBlobStore store = initLocalFs(); + @Test + public void testDeleteAfterFailedCreate() throws Exception { + // Check that a blob can be deleted when a temporary file exists in the blob directory + LocalFsBlobStore store = initLocalFs(); - String key = "test"; - SettableBlobMeta metadata = new SettableBlobMeta(BlobStoreAclHandler + String key = "test"; + SettableBlobMeta metadata = new SettableBlobMeta(BlobStoreAclHandler .WORLD_EVERYTHING); - try (AtomicOutputStream out = store.createBlob(key, metadata, null)) { - out.write(1); - File blobDir = store.getKeyDataDir(key); - Files.createFile(blobDir.toPath().resolve("tempFile.tmp")); - } + try (AtomicOutputStream out = store.createBlob(key, metadata, null)) { + out.write(1); + File blobDir = store.getKeyDataDir(key); + Files.createFile(blobDir.toPath().resolve("tempFile.tmp")); + } - store.deleteBlob("test",null); + store.deleteBlob("test", null); - } + } - public Subject getSubject(String name) { - Subject subject = new Subject(); - SingleUserPrincipal user = new SingleUserPrincipal(name); - subject.getPrincipals().add(user); - return subject; - } + public Subject getSubject(String name) { + Subject subject = new Subject(); + SingleUserPrincipal user = new SingleUserPrincipal(name); + subject.getPrincipals().add(user); + return subject; + } // Gets Nimbus Subject with NimbusPrincipal set on it public static Subject getNimbusSubject() { @@ -148,7 +155,8 @@ public static Subject getNimbusSubject() { return nimbus; } - // Overloading the assertStoreHasExactly method accomodate Subject in order to check for authorization + // Overloading the assertStoreHasExactly method accomodate Subject in order to check for + // authorization public static void assertStoreHasExactly(BlobStore store, Subject who, String... keys) { Set expected = new HashSet<>(Arrays.asList(keys)); Set found = new HashSet<>(); @@ -169,7 +177,8 @@ public static void assertStoreHasExactly(BlobStore store, String... keys) { assertStoreHasExactly(store, null, keys); } - // Overloading the readInt method accomodate Subject in order to check for authorization (security turned on) + // Overloading the readInt method accomodate Subject in order to check for authorization + // (security turned on) public static int readInt(BlobStore store, Subject who, String key) throws IOException, KeyNotFoundException, AuthorizationException { try (InputStream in = store.getBlob(key, who)) { @@ -195,7 +204,7 @@ public void readAssertEqualsWithAuth(BlobStore store, Subject who, String key, i // Check for Blobstore with authentication public void testWithAuthentication(BlobStore store) throws Exception { - //Test for Nimbus Admin + // Test for Nimbus Admin Subject admin = getSubject("admin"); assertStoreHasExactly(store); SettableBlobMeta metadata = new SettableBlobMeta(BlobStoreAclHandler.DEFAULT); @@ -205,7 +214,7 @@ public void testWithAuthentication(BlobStore store) throws Exception { } store.deleteBlob("test", admin); - //Test for Supervisor Admin + // Test for Supervisor Admin Subject supervisor = getSubject("supervisor"); assertStoreHasExactly(store); metadata = new SettableBlobMeta(BlobStoreAclHandler.DEFAULT); @@ -215,7 +224,7 @@ public void testWithAuthentication(BlobStore store) throws Exception { } store.deleteBlob("test", supervisor); - //Test for Nimbus itself as a user + // Test for Nimbus itself as a user Subject nimbus = getNimbusSubject(); assertStoreHasExactly(store); metadata = new SettableBlobMeta(BlobStoreAclHandler.DEFAULT); @@ -237,7 +246,8 @@ public void testWithAuthentication(BlobStore store) throws Exception { } assertStoreHasExactly(store, "test"); // Testing whether acls are set to WORLD_EVERYTHING - assertTrue(metadata.toString().contains("AccessControl(type:OTHER, access:7)"), "ACL does not contain WORLD_EVERYTHING"); + assertTrue(metadata.toString().contains("AccessControl(type:OTHER, access:7)"), + "ACL does not contain WORLD_EVERYTHING"); readAssertEqualsWithAuth(store, who, "test", 1); LOG.info("Deleting test"); @@ -252,10 +262,13 @@ public void testWithAuthentication(BlobStore store) throws Exception { out.write(2); } assertStoreHasExactly(store, "test"); - // Testing whether acls are set to WORLD_EVERYTHING. Here the acl should not contain WORLD_EVERYTHING because - // the subject is neither null nor empty. The ACL should however contain USER_EVERYTHING as user needs to have + // Testing whether acls are set to WORLD_EVERYTHING. Here the acl should not contain + // WORLD_EVERYTHING because + // the subject is neither null nor empty. The ACL should however contain USER_EVERYTHING as + // user needs to have // complete access to the blob - assertTrue(!metadata.toString().contains("AccessControl(type:OTHER, access:7)"), "ACL does not contain WORLD_EVERYTHING"); + assertTrue(!metadata.toString().contains("AccessControl(type:OTHER, access:7)"), + "ACL does not contain WORLD_EVERYTHING"); readAssertEqualsWithAuth(store, who, "test", 2); LOG.info("Updating test"); @@ -284,7 +297,8 @@ public void testWithAuthentication(BlobStore store) throws Exception { } assertStoreHasExactly(store, "test-empty-subject-WE", "test"); // Testing whether acls are set to WORLD_EVERYTHING - assertTrue(metadata.toString().contains("AccessControl(type:OTHER, access:7)"), "ACL does not contain WORLD_EVERYTHING"); + assertTrue(metadata.toString().contains("AccessControl(type:OTHER, access:7)"), + "ACL does not contain WORLD_EVERYTHING"); readAssertEqualsWithAuth(store, who, "test-empty-subject-WE", 2); // Test for subject with no principals and acls set to DEFAULT @@ -297,7 +311,8 @@ public void testWithAuthentication(BlobStore store) throws Exception { } assertStoreHasExactly(store, "test-empty-subject-DEF", "test", "test-empty-subject-WE"); // Testing whether acls are set to WORLD_EVERYTHING - assertTrue(metadata.toString().contains("AccessControl(type:OTHER, access:7)"), "ACL does not contain WORLD_EVERYTHING"); + assertTrue(metadata.toString().contains("AccessControl(type:OTHER, access:7)"), + "ACL does not contain WORLD_EVERYTHING"); readAssertEqualsWithAuth(store, who, "test-empty-subject-DEF", 2); if (store instanceof LocalFsBlobStore) { @@ -319,7 +334,8 @@ public void testBasic(BlobStore store) throws Exception { } assertStoreHasExactly(store, "test"); // Testing whether acls are set to WORLD_EVERYTHING - assertTrue(metadata.toString().contains("AccessControl(type:OTHER, access:7)"), "ACL does not contain WORLD_EVERYTHING"); + assertTrue(metadata.toString().contains("AccessControl(type:OTHER, access:7)"), + "ACL does not contain WORLD_EVERYTHING"); readAssertEquals(store, "test", 1); LOG.info("Deleting test"); @@ -335,7 +351,8 @@ public void testBasic(BlobStore store) throws Exception { } assertStoreHasExactly(store, "test"); if (store instanceof LocalFsBlobStore) { - assertTrue(metadata.toString().contains("AccessControl(type:OTHER, access:7)"), "ACL does not contain WORLD_EVERYTHING"); + assertTrue(metadata.toString().contains("AccessControl(type:OTHER, access:7)"), + "ACL does not contain WORLD_EVERYTHING"); } readAssertEquals(store, "test", 2); LOG.info("Updating test"); @@ -363,10 +380,13 @@ public void testBasic(BlobStore store) throws Exception { out.write(2); } assertStoreHasExactly(store, "test-empty-acls", "test"); - // Testing whether acls are set to WORLD_EVERYTHING, Here we are testing only for LocalFsBlobstore - // as the HdfsBlobstore gets the subject information of the local system user and behaves as it is + // Testing whether acls are set to WORLD_EVERYTHING, Here we are testing only for + // LocalFsBlobstore + // as the HdfsBlobstore gets the subject information of the local system user and + // behaves as it is // always authenticated. - assertTrue(metadata.get_acl().toString().contains("OTHER"), "ACL does not contain WORLD_EVERYTHING"); + assertTrue(metadata.get_acl().toString().contains("OTHER"), + "ACL does not contain WORLD_EVERYTHING"); LOG.info("Deleting test-empty-acls"); store.deleteBlob("test-empty-acls", null); @@ -379,12 +399,12 @@ public void testBasic(BlobStore store) throws Exception { } } - public void testMultiple(BlobStore store) throws Exception { assertStoreHasExactly(store); LOG.info("Creating test"); - try (AtomicOutputStream out = store.createBlob("test", new SettableBlobMeta(BlobStoreAclHandler + try (AtomicOutputStream out = store.createBlob("test", + new SettableBlobMeta(BlobStoreAclHandler .WORLD_EVERYTHING), null)) { out.write(1); } @@ -392,7 +412,8 @@ public void testMultiple(BlobStore store) throws Exception { readAssertEquals(store, "test", 1); LOG.info("Creating other"); - try (AtomicOutputStream out = store.createBlob("other", new SettableBlobMeta(BlobStoreAclHandler.WORLD_EVERYTHING), + try (AtomicOutputStream out = store.createBlob("other", + new SettableBlobMeta(BlobStoreAclHandler.WORLD_EVERYTHING), null)) { out.write(2); } @@ -414,7 +435,8 @@ public void testMultiple(BlobStore store) throws Exception { readAssertEquals(store, "other", 5); LOG.info("Creating test again"); - try (AtomicOutputStream out = store.createBlob("test", new SettableBlobMeta(BlobStoreAclHandler.WORLD_EVERYTHING), + try (AtomicOutputStream out = store.createBlob("test", + new SettableBlobMeta(BlobStoreAclHandler.WORLD_EVERYTHING), null)) { out.write(2); } @@ -463,7 +485,8 @@ public void testMultiple(BlobStore store) throws Exception { public void testGetFileLength() throws AuthorizationException, KeyNotFoundException, KeyAlreadyExistsException, IOException { LocalFsBlobStore store = initLocalFs(); - try (AtomicOutputStream out = store.createBlob("test", new SettableBlobMeta(BlobStoreAclHandler + try (AtomicOutputStream out = store.createBlob("test", + new SettableBlobMeta(BlobStoreAclHandler .WORLD_EVERYTHING), null)) { out.write(1); } diff --git a/storm-server/src/test/java/org/apache/storm/blobstore/MockZookeeperClientBuilder.java b/storm-server/src/test/java/org/apache/storm/blobstore/MockZookeeperClientBuilder.java index 19cdfed25d8..b89f67c7115 100644 --- a/storm-server/src/test/java/org/apache/storm/blobstore/MockZookeeperClientBuilder.java +++ b/storm-server/src/test/java/org/apache/storm/blobstore/MockZookeeperClientBuilder.java @@ -1,17 +1,28 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.blobstore; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + import java.util.Arrays; import java.util.List; import org.apache.log4j.Logger; @@ -21,11 +32,6 @@ import org.apache.storm.shade.org.apache.curator.framework.api.Pathable; import org.apache.storm.shade.org.apache.zookeeper.data.Stat; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - public class MockZookeeperClientBuilder { private static final Logger LOG = Logger.getLogger(MockZookeeperClientBuilder.class); diff --git a/storm-server/src/test/java/org/apache/storm/cluster/ClusterStateTest.java b/storm-server/src/test/java/org/apache/storm/cluster/ClusterStateTest.java index fff2fdf4bb4..8f7b1ab194e 100644 --- a/storm-server/src/test/java/org/apache/storm/cluster/ClusterStateTest.java +++ b/storm-server/src/test/java/org/apache/storm/cluster/ClusterStateTest.java @@ -18,6 +18,13 @@ package org.apache.storm.cluster; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; @@ -26,8 +33,8 @@ import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; - import org.apache.storm.Config; +import org.apache.storm.callback.WatcherCallBack; import org.apache.storm.callback.ZKStateChangedCallback; import org.apache.storm.generated.Assignment; import org.apache.storm.generated.Credentials; @@ -39,34 +46,31 @@ import org.apache.storm.generated.TopologyStatus; import org.apache.storm.generated.WorkerResources; import org.apache.storm.nimbus.NimbusInfo; +import org.apache.storm.shade.org.apache.curator.framework.CuratorFramework; +import org.apache.storm.shade.org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.storm.shade.org.apache.curator.framework.api.BackgroundVersionable; +import org.apache.storm.shade.org.apache.curator.framework.api.DeleteBuilder; +import org.apache.storm.shade.org.apache.curator.framework.api.ExistsBuilder; +import org.apache.storm.shade.org.apache.zookeeper.KeeperException; import org.apache.storm.shade.org.apache.zookeeper.Watcher; import org.apache.storm.shade.org.apache.zookeeper.ZooDefs; import org.apache.storm.shade.org.apache.zookeeper.data.ACL; +import org.apache.storm.shade.org.apache.zookeeper.data.Stat; import org.apache.storm.testing.InProcessZookeeper; import org.apache.storm.utils.CuratorUtils; import org.apache.storm.utils.Time; import org.apache.storm.utils.Utils; import org.apache.storm.utils.ZookeeperAuthInfo; -import org.apache.storm.shade.org.apache.curator.framework.CuratorFramework; -import org.apache.storm.shade.org.apache.curator.framework.CuratorFrameworkFactory; -import org.apache.storm.callback.WatcherCallBack; -import org.apache.storm.shade.org.apache.curator.framework.api.BackgroundVersionable; -import org.apache.storm.shade.org.apache.curator.framework.api.DeleteBuilder; -import org.apache.storm.shade.org.apache.curator.framework.api.ExistsBuilder; -import org.apache.storm.shade.org.apache.zookeeper.KeeperException; -import org.apache.storm.shade.org.apache.zookeeper.data.Stat; import org.apache.storm.zookeeper.ClientZookeeper; -import org.junit.jupiter.api.Test; import org.awaitility.Awaitility; +import org.junit.jupiter.api.Test; import org.mockito.Mockito; -import static org.junit.jupiter.api.Assertions.*; - /** * Tests for IStateStorage (ZKStateStorage) and IStormClusterState (StormClusterStateImpl) * using an in-process ZooKeeper. * - * Ported from storm-core/test/clj/org/apache/storm/cluster_test.clj + *

      Ported from storm-core/test/clj/org/apache/storm/cluster_test.clj */ public class ClusterStateTest { @@ -122,7 +126,8 @@ public void testBasics() throws Exception { state.mkdirs("/lalala", OPEN_ACL); assertEquals(List.of(), state.get_children("/lalala", false)); - assertEquals(Set.of("root", "a", "lalala"), new HashSet<>(state.get_children("/", false))); + assertEquals(Set.of("root", "a", "lalala"), new HashSet<>(state.get_children("/", + false))); state.delete_node("/a"); assertEquals(Set.of("root", "lalala"), new HashSet<>(state.get_children("/", false))); @@ -217,7 +222,8 @@ public void testCallbacks() throws Exception { assertNull(cb2.lastEvent.get()); state2.set_data("/root", barr(2), OPEN_ACL); - assertEquals(event(Watcher.Event.EventType.NodeDataChanged, "/root"), cb2.readAndReset()); + assertEquals(event(Watcher.Event.EventType.NodeDataChanged, "/root"), cb2 + .readAndReset()); assertNull(cb1.lastEvent.get()); // no watch set, so no callback @@ -239,12 +245,14 @@ public void testCallbacks() throws Exception { state1.get_children("/", true); state2.set_data("/a", barr(9), OPEN_ACL); assertNull(cb2.lastEvent.get()); - assertEquals(event(Watcher.Event.EventType.NodeChildrenChanged, "/"), cb1.readAndReset()); + assertEquals(event(Watcher.Event.EventType.NodeChildrenChanged, "/"), cb1 + .readAndReset()); // ephemeral node data change state2.get_data("/root", true); state1.set_ephemeral_node("/root", barr(1, 2), OPEN_ACL); - assertEquals(event(Watcher.Event.EventType.NodeDataChanged, "/root"), cb2.readAndReset()); + assertEquals(event(Watcher.Event.EventType.NodeDataChanged, "/root"), cb2 + .readAndReset()); // children + data creation state1.mkdirs("/ccc", OPEN_ACL); @@ -252,7 +260,8 @@ public void testCallbacks() throws Exception { state2.get_data("/ccc/b", true); state2.set_data("/ccc/b", barr(8), OPEN_ACL); assertEquals(event(Watcher.Event.EventType.NodeCreated, "/ccc/b"), cb2.readAndReset()); - assertEquals(event(Watcher.Event.EventType.NodeChildrenChanged, "/ccc"), cb1.readAndReset()); + assertEquals(event(Watcher.Event.EventType.NodeChildrenChanged, "/ccc"), cb1 + .readAndReset()); // closing state1 removes its ephemeral nodes state2.get_data("/root", true); @@ -304,8 +313,10 @@ public void testStormClusterStateBasics() throws Exception { NimbusInfo nimbusInfo1 = new NimbusInfo("nimbus1", 6667, false); NimbusInfo nimbusInfo2 = new NimbusInfo("nimbus2", 6667, false); - NimbusSummary nimbusSummary1 = new NimbusSummary("nimbus1", 6667, Time.currentTimeSecs(), false, "v1"); - NimbusSummary nimbusSummary2 = new NimbusSummary("nimbus2", 6667, Time.currentTimeSecs(), false, "v2"); + NimbusSummary nimbusSummary1 = new NimbusSummary("nimbus1", 6667, Time + .currentTimeSecs(), false, "v1"); + NimbusSummary nimbusSummary2 = new NimbusSummary("nimbus2", 6667, Time + .currentTimeSecs(), false, "v2"); StormBase base1 = mkStormBase("/tmp/storm1", 1, TopologyStatus.ACTIVE, 2); StormBase base2 = mkStormBase("/tmp/storm2", 2, TopologyStatus.ACTIVE, 2); @@ -350,9 +361,11 @@ public void testStormClusterStateBasics() throws Exception { assertEquals(List.of(), state.blobstoreInfo("")); state.setupBlob("key1", nimbusInfo1, 1); assertEquals(List.of("key1"), state.blobstoreInfo("")); - assertEquals(List.of(nimbusInfo1.toHostPortString() + "-1"), state.blobstoreInfo("key1")); + assertEquals(List.of(nimbusInfo1.toHostPortString() + "-1"), state + .blobstoreInfo("key1")); state.setupBlob("key1", nimbusInfo2, 1); - assertEquals(Set.of(nimbusInfo1.toHostPortString() + "-1", nimbusInfo2.toHostPortString() + "-1"), + assertEquals(Set.of(nimbusInfo1.toHostPortString() + "-1", nimbusInfo2 + .toHostPortString() + "-1"), new HashSet<>(state.blobstoreInfo("key1"))); state.removeBlobstoreKey("key1"); assertEquals(List.of(), state.blobstoreInfo("")); @@ -395,7 +408,8 @@ public void testStormClusterStateErrors() throws Exception { Time.advanceTimeSecs(2); } validateErrors(state, "a", "2", - "IllegalArgumentException", "IllegalArgumentException", "IllegalArgumentException", + "IllegalArgumentException", "IllegalArgumentException", "IllegalArgumentExcept" + + "ion", "IllegalArgumentException", "IllegalArgumentException", "RuntimeException", "RuntimeException", "RuntimeException", "RuntimeException", "RuntimeException"); @@ -404,17 +418,20 @@ public void testStormClusterStateErrors() throws Exception { } } - private void validateErrors(IStormClusterState state, String stormId, String component, String... expectedErrors) { + private void validateErrors(IStormClusterState state, String stormId, String component, + String... expectedErrors) { List errors = state.errors(stormId, component); assertEquals(expectedErrors.length, errors.size(), "Expected " + expectedErrors.length + " errors but got " + errors.size()); for (int i = 0; i < expectedErrors.length; i++) { assertTrue(errors.get(i).get_error().contains(expectedErrors[i]), - "Error " + i + " should contain '" + expectedErrors[i] + "' but was: " + errors.get(i).get_error()); + "Error " + i + " should contain '" + expectedErrors[i] + "' but was: " + errors + .get(i).get_error()); } } - private static SupervisorInfo mkSupervisorInfo(long timeSecs, String hostname, String assignmentId, + private static SupervisorInfo mkSupervisorInfo(long timeSecs, String hostname, + String assignmentId, List usedPorts, List meta, Map schedulerMeta, long uptimeSecs, String version, @@ -466,7 +483,8 @@ public void testSupervisorState() throws Exception { @Test public void testClusterAuthentication() throws Exception { try (InProcessZookeeper zk = new InProcessZookeeper()) { - CuratorFrameworkFactory.Builder builder = Mockito.mock(CuratorFrameworkFactory.Builder.class); + CuratorFrameworkFactory.Builder builder = Mockito + .mock(CuratorFrameworkFactory.Builder.class); Map conf = mkConfig(zk.getPort()); conf.put(Config.STORM_ZOOKEEPER_CONNECTION_TIMEOUT, 10); conf.put(Config.STORM_ZOOKEEPER_SESSION_TIMEOUT, 10); @@ -561,7 +579,8 @@ public void testDeleteNodeSwallowsConcurrentDelete() throws Exception { BackgroundVersionable childrenDeletable = Mockito.mock(BackgroundVersionable.class); Mockito.when(zk.delete()).thenReturn(deleteBuilder); Mockito.when(deleteBuilder.deletingChildrenIfNeeded()).thenReturn(childrenDeletable); - Mockito.when(childrenDeletable.forPath("/race")).thenThrow(new KeeperException.NoNodeException()); + Mockito.when(childrenDeletable.forPath("/race")).thenThrow(new KeeperException + .NoNodeException()); assertDoesNotThrow(() -> ClientZookeeper.deleteNode(zk, "/race")); // ensure to delete was actually attempted (the catch branch was exercised, not skipped) diff --git a/storm-server/src/test/java/org/apache/storm/container/docker/DockerExecCommandTest.java b/storm-server/src/test/java/org/apache/storm/container/docker/DockerExecCommandTest.java index 7c9a41588be..27e6073c94a 100644 --- a/storm-server/src/test/java/org/apache/storm/container/docker/DockerExecCommandTest.java +++ b/storm-server/src/test/java/org/apache/storm/container/docker/DockerExecCommandTest.java @@ -1,12 +1,17 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-server/src/test/java/org/apache/storm/container/docker/DockerInspectCommandTest.java b/storm-server/src/test/java/org/apache/storm/container/docker/DockerInspectCommandTest.java index 5e3a259f44c..95c0f7832d4 100644 --- a/storm-server/src/test/java/org/apache/storm/container/docker/DockerInspectCommandTest.java +++ b/storm-server/src/test/java/org/apache/storm/container/docker/DockerInspectCommandTest.java @@ -1,12 +1,17 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -14,7 +19,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; - import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -39,4 +43,4 @@ public void getCommandWithArguments() { assertEquals("inspect --format='{{.State.Status}}' container_name", dockerInspectCommand.getCommandWithArguments()); } -} \ No newline at end of file +} diff --git a/storm-server/src/test/java/org/apache/storm/container/docker/DockerPsCommandTest.java b/storm-server/src/test/java/org/apache/storm/container/docker/DockerPsCommandTest.java index 4e72f2f5184..a19628e7a2e 100644 --- a/storm-server/src/test/java/org/apache/storm/container/docker/DockerPsCommandTest.java +++ b/storm-server/src/test/java/org/apache/storm/container/docker/DockerPsCommandTest.java @@ -20,7 +20,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; - import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -45,4 +44,4 @@ public void getCommandWithArguments() { assertEquals("ps --filter=name=container_name --quiet=true", dockerPsCommand.getCommandWithArguments()); } -} \ No newline at end of file +} diff --git a/storm-server/src/test/java/org/apache/storm/container/docker/DockerRmCommandTest.java b/storm-server/src/test/java/org/apache/storm/container/docker/DockerRmCommandTest.java index 9dd8b0664cf..85053ada3c9 100644 --- a/storm-server/src/test/java/org/apache/storm/container/docker/DockerRmCommandTest.java +++ b/storm-server/src/test/java/org/apache/storm/container/docker/DockerRmCommandTest.java @@ -1,12 +1,17 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -14,7 +19,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; - import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -38,4 +42,4 @@ public void getCommandWithArguments() { assertEquals("rm --force container_name", dockerRmCommand.getCommandWithArguments()); } -} \ No newline at end of file +} diff --git a/storm-server/src/test/java/org/apache/storm/container/docker/DockerRunCommandTest.java b/storm-server/src/test/java/org/apache/storm/container/docker/DockerRunCommandTest.java index 5436bdaf44e..8858e8321a5 100644 --- a/storm-server/src/test/java/org/apache/storm/container/docker/DockerRunCommandTest.java +++ b/storm-server/src/test/java/org/apache/storm/container/docker/DockerRunCommandTest.java @@ -1,12 +1,17 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -14,7 +19,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; - import java.io.IOException; import java.util.Arrays; import java.util.List; @@ -51,4 +55,4 @@ public void getCommandOption() throws IOException { + "image_name bash launch_command", dockerRunCommand.getCommandWithArguments()); } -} \ No newline at end of file +} diff --git a/storm-server/src/test/java/org/apache/storm/container/docker/DockerStopCommandTest.java b/storm-server/src/test/java/org/apache/storm/container/docker/DockerStopCommandTest.java index 1d243cfc120..787509c2f70 100644 --- a/storm-server/src/test/java/org/apache/storm/container/docker/DockerStopCommandTest.java +++ b/storm-server/src/test/java/org/apache/storm/container/docker/DockerStopCommandTest.java @@ -1,12 +1,17 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -14,7 +19,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; - import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -39,4 +43,4 @@ public void getCommandWithArguments() { assertEquals("stop --time=3 container_name", dockerStopCommand.getCommandWithArguments()); } -} \ No newline at end of file +} diff --git a/storm-server/src/test/java/org/apache/storm/container/docker/DockerWaitCommandTest.java b/storm-server/src/test/java/org/apache/storm/container/docker/DockerWaitCommandTest.java index de5b4f9fb38..cdc426bff8c 100644 --- a/storm-server/src/test/java/org/apache/storm/container/docker/DockerWaitCommandTest.java +++ b/storm-server/src/test/java/org/apache/storm/container/docker/DockerWaitCommandTest.java @@ -1,12 +1,17 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -14,7 +19,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; - import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -37,4 +41,4 @@ public void getCommandWithArguments() { assertEquals("wait container_name", dockerWaitCommand.getCommandWithArguments()); } -} \ No newline at end of file +} diff --git a/storm-server/src/test/java/org/apache/storm/container/oci/OciUtilsTest.java b/storm-server/src/test/java/org/apache/storm/container/oci/OciUtilsTest.java index 69deeafcc55..bb47837e26a 100644 --- a/storm-server/src/test/java/org/apache/storm/container/oci/OciUtilsTest.java +++ b/storm-server/src/test/java/org/apache/storm/container/oci/OciUtilsTest.java @@ -18,6 +18,10 @@ package org.apache.storm.container.oci; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -28,17 +32,13 @@ import org.apache.storm.utils.WrappedInvalidTopologyException; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertThrows; - public class OciUtilsTest { @Test public void validateImageInDaemonConfSkipped() { Map conf = new HashMap<>(); conf.put(DaemonConfig.STORM_OCI_IMAGE, "storm/rhel7:dev_test"); - //this is essentially a no-op + // this is essentially a no-op OciUtils.validateImageInDaemonConf(conf); } @@ -79,7 +79,7 @@ public void validateImageInDaemonConfWithNullDefault() { allowedImages.add("storm/rhel7:dev_test"); conf.put(DaemonConfig.STORM_OCI_ALLOWED_IMAGES, allowedImages); - conf.put(DaemonConfig.STORM_OCI_IMAGE, null); //or not set + conf.put(DaemonConfig.STORM_OCI_IMAGE, null); // or not set OciUtils.validateImageInDaemonConf(conf); }); } @@ -100,40 +100,41 @@ public void validateImageInDaemonConfWrongPattern() { @Test public void adjustImageConfigForTopoTest() throws InvalidTopologyException { Map conf = new HashMap<>(); - conf.put(DaemonConfig.STORM_OCI_ALLOWED_IMAGES, null); //or not set + conf.put(DaemonConfig.STORM_OCI_ALLOWED_IMAGES, null); // or not set Map topoConf = new HashMap<>(); String topoId = "topo1"; - //case 1: nothing is not; nothing will happen + // case 1: nothing is not; nothing will happen OciUtils.adjustImageConfigForTopo(conf, topoConf, topoId); String image1 = "storm/rhel7:dev_test"; String defaultImage = "storm/rhel7:dev_current"; - //case 2: allowed list is not set; topology oci image will be set to null + // case 2: allowed list is not set; topology oci image will be set to null topoConf.put(Config.TOPOLOGY_OCI_IMAGE, image1); OciUtils.adjustImageConfigForTopo(conf, topoConf, topoId); - assertNull(topoConf.get(Config.TOPOLOGY_OCI_IMAGE), Config.TOPOLOGY_OCI_IMAGE + " is not removed"); + assertNull(topoConf.get(Config.TOPOLOGY_OCI_IMAGE), Config.TOPOLOGY_OCI_IMAGE + + " is not removed"); - //set up daemon conf properly + // set up daemon conf properly List allowedImages = new ArrayList<>(); allowedImages.add(image1); allowedImages.add(defaultImage); conf.put(DaemonConfig.STORM_OCI_ALLOWED_IMAGES, allowedImages); conf.put(DaemonConfig.STORM_OCI_IMAGE, defaultImage); - //case 3: configs are set properly; nothing will happen + // case 3: configs are set properly; nothing will happen topoConf.put(Config.TOPOLOGY_OCI_IMAGE, image1); OciUtils.adjustImageConfigForTopo(conf, topoConf, topoId); assertEquals(image1, topoConf.get(Config.TOPOLOGY_OCI_IMAGE)); - //case 4: topology oci image is not set; will be set to default image + // case 4: topology oci image is not set; will be set to default image topoConf.remove(Config.TOPOLOGY_OCI_IMAGE); OciUtils.adjustImageConfigForTopo(conf, topoConf, topoId); assertEquals(defaultImage, topoConf.get(Config.TOPOLOGY_OCI_IMAGE)); - //case 5: any topology oci image is allowed + // case 5: any topology oci image is allowed allowedImages.add("*"); String image2 = "storm/rhel7:dev_wow"; topoConf.put(Config.TOPOLOGY_OCI_IMAGE, image2); @@ -158,4 +159,4 @@ public void adjustImageConfigForTopoNotInAllowedList() { assertThrows(WrappedInvalidTopologyException.class, () -> OciUtils.adjustImageConfigForTopo(conf, topoConf, topoId)); } -} \ No newline at end of file +} diff --git a/storm-server/src/test/java/org/apache/storm/daemon/drpc/DRPCTest.java b/storm-server/src/test/java/org/apache/storm/daemon/drpc/DRPCTest.java index a9fd7f1d1b6..fa4be28feba 100644 --- a/storm-server/src/test/java/org/apache/storm/daemon/drpc/DRPCTest.java +++ b/storm-server/src/test/java/org/apache/storm/daemon/drpc/DRPCTest.java @@ -18,6 +18,11 @@ package org.apache.storm.daemon.drpc; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -37,24 +42,18 @@ import org.apache.storm.generated.DRPCExceptionType; import org.apache.storm.generated.DRPCExecutionException; import org.apache.storm.generated.DRPCRequest; +import org.apache.storm.metric.StormMetricsRegistry; import org.apache.storm.security.auth.DefaultPrincipalToLocal; import org.apache.storm.security.auth.ReqContext; import org.apache.storm.security.auth.SingleUserPrincipal; -import org.apache.storm.security.auth.authorizer.DRPCSimpleACLAuthorizer; import org.apache.storm.security.auth.authorizer.DRPCSimpleACLAuthorizer.AclFunctionEntry; +import org.apache.storm.security.auth.authorizer.DRPCSimpleACLAuthorizer; import org.apache.storm.security.auth.authorizer.DenyAuthorizer; import org.apache.storm.utils.Time; import org.awaitility.Awaitility; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; - -import org.apache.storm.metric.StormMetricsRegistry; - public class DRPCTest { private static final ExecutorService exec = Executors.newCachedThreadPool(); @@ -63,7 +62,8 @@ private static void assertThrows(ThrowStuff t, Class expect t.run(); fail("Expected " + t + " to throw " + expected + " didn't throw at all..."); } catch (Exception e) { - assertTrue(expected.isInstance(e), "Expected " + t + " to throw " + expected + " but threw " + e); + assertTrue(expected.isInstance(e), "Expected " + t + " to throw " + expected + + " but threw " + e); } } @@ -79,7 +79,8 @@ public static DRPCRequest getNextAvailableRequest(DRPC server, String func) { .pollInterval(1, TimeUnit.MILLISECONDS) .until(() -> { DRPCRequest req = server.fetchRequest(func); - if (req != null && req.get_request_id() != null && !req.get_request_id().isEmpty()) { + if (req != null && req.get_request_id() != null && !req.get_request_id() + .isEmpty()) { result.set(req); return true; } @@ -117,8 +118,9 @@ public void testFailedBlocking() throws Exception { } catch (ExecutionException e) { Throwable t = e.getCause(); assertTrue(t instanceof DRPCExecutionException); - //Don't know a better way to validate that it failed. - assertEquals(DRPCExceptionType.FAILED_REQUEST, ((DRPCExecutionException) t).get_type()); + // Don't know a better way to validate that it failed. + assertEquals(DRPCExceptionType.FAILED_REQUEST, ((DRPCExecutionException) t) + .get_type()); } } } @@ -147,13 +149,13 @@ public void testDequeueAfterTimeout() throws Exception { @Test public void testQueuesAreRemovedWhenEmpty() throws Exception { try (DRPC server = new DRPC(new StormMetricsRegistry(), null, 1000)) { - //Fetching for a function nothing was ever submitted for must not leave state behind + // Fetching for a function nothing was ever submitted for must not leave state behind DRPCRequest nothing = server.fetchRequest("never-registered"); assertNotNull(nothing); assertEquals("", nothing.get_request_id()); assertEquals(0, server.getNumTrackedFunctions()); - //A registered function is still served repeatedly, and is not left behind once idle + // A registered function is still served repeatedly, and is not left behind once idle for (int i = 0; i < 3; i++) { Future found = exec.submit(() -> server.executeBlocking("testing", "test")); DRPCRequest request = getNextAvailableRequest(server, "testing"); @@ -163,7 +165,7 @@ public void testQueuesAreRemovedWhenEmpty() throws Exception { } assertEquals(0, server.getNumTrackedFunctions()); - //Nor is a function whose only request timed out. The timer thread fails the request + // Nor is a function whose only request timed out. The timer thread fails the request // before it drops the queue, so the caller can return first; wait for the drop instead // of racing it, with a hard timeout so a real leak still fails the test. try { @@ -181,7 +183,8 @@ public void testQueuesAreRemovedWhenEmpty() throws Exception { @Test public void testConcurrentExecuteAndFetchLosesNoRequests() throws Exception { - //A bounded pool of 16 threads is what keeps this cheap: executeBlocking() parks its caller, + // A bounded pool of 16 threads is what keeps this cheap: executeBlocking() parks its + // caller, // so an unbounded pool would need one live thread per outstanding request. The request // count costs no threads at all, and is what gives the stress test its power. Measured // against a fetchRequest() whose poll/remove escapes the per-function compute lock, the @@ -190,7 +193,7 @@ public void testConcurrentExecuteAndFetchLosesNoRequests() throws Exception { final int numRequests = 5000; final int numThreads = 16; final long deadlineMs = 30_000; - //A timeout far beyond the test deadline, so the cleanup timer never reaps a live request. + // A timeout far beyond the test deadline, so the cleanup timer never reaps a live request. try (DRPC server = new DRPC(new StormMetricsRegistry(), null, 300_000)) { ExecutorService submitters = Executors.newFixedThreadPool(numThreads); try { @@ -212,7 +215,7 @@ public void testConcurrentExecuteAndFetchLosesNoRequests() throws Exception { assertNotNull(req); String id = req.get_request_id(); if (id.isEmpty()) { - //Nothing to serve right now. Spin at first, so fetches keep interleaving + // Nothing to serve right now. Spin at first, so fetches keep interleaving // tightly with the submitting threads, and only back off if this goes on // for a long time (a regression, which the deadline above then fails). if (++emptyFetches > 10_000) { @@ -231,13 +234,15 @@ public void testConcurrentExecuteAndFetchLosesNoRequests() throws Exception { for (Future f : futures) { long left = deadline - Time.currentTimeMillis(); assertTrue(left > 0, "Ran out of time waiting for the blocked callers"); - assertTrue(results.add(f.get(left, TimeUnit.MILLISECONDS)), "Duplicate result returned"); + assertTrue(results.add(f.get(left, TimeUnit.MILLISECONDS)), + "Duplicate result returned"); } assertEquals(numRequests, results.size()); for (String id : servedIds) { - assertTrue(results.contains("tested-" + id), "No caller got the result for " + id); + assertTrue(results.contains("tested-" + id), "No caller got the result for " + + id); } - //Nothing is waiting any more, so no per-function queue may be left behind + // Nothing is waiting any more, so no per-function queue may be left behind assertEquals(0, server.getNumTrackedFunctions()); } finally { submitters.shutdownNow(); @@ -248,7 +253,8 @@ public void testConcurrentExecuteAndFetchLosesNoRequests() throws Exception { @Test public void testDeny() { try (DRPC server = new DRPC(new StormMetricsRegistry(), new DenyAuthorizer(), 100)) { - assertThrows(() -> server.executeBlocking("testing", "test"), AuthorizationException.class); + assertThrows(() -> server.executeBlocking("testing", "test"), + AuthorizationException.class); assertThrows(() -> server.fetchRequest("testing"), AuthorizationException.class); } } @@ -268,7 +274,8 @@ public void testStrict() throws Exception { other.subject().getPrincipals().add(otherUser); Map acl = new HashMap<>(); - acl.put("jump", new AclFunctionEntry(Collections.singletonList(jumpClient.getName()), jumpTopo.getName())); + acl.put("jump", new AclFunctionEntry(Collections.singletonList(jumpClient.getName()), + jumpTopo.getName())); Map conf = new HashMap<>(); conf.put(Config.DRPC_AUTHORIZER_ACL_STRICT, true); conf.put(Config.STORM_PRINCIPAL_TO_LOCAL_PLUGIN, DefaultPrincipalToLocal.class.getName()); @@ -279,31 +286,46 @@ protected Map readAclFromConfig() { } }; auth.prepare(conf); - //JUMP + // JUMP DRPC.checkAuthorization(jt, auth, "fetchRequest", "jump"); - assertThrows(() -> DRPC.checkAuthorization(jc, auth, "fetchRequest", "jump"), AuthorizationException.class); - assertThrows(() -> DRPC.checkAuthorization(other, auth, "fetchRequest", "jump"), AuthorizationException.class); + assertThrows(() -> DRPC.checkAuthorization(jc, auth, "fetchRequest", "jump"), + AuthorizationException.class); + assertThrows(() -> DRPC.checkAuthorization(other, auth, "fetchRequest", "jump"), + AuthorizationException.class); DRPC.checkAuthorization(jt, auth, "result", "jump"); - assertThrows(() -> DRPC.checkAuthorization(jc, auth, "result", "jump"), AuthorizationException.class); - assertThrows(() -> DRPC.checkAuthorization(other, auth, "result", "jump"), AuthorizationException.class); + assertThrows(() -> DRPC.checkAuthorization(jc, auth, "result", "jump"), + AuthorizationException.class); + assertThrows(() -> DRPC.checkAuthorization(other, auth, "result", "jump"), + AuthorizationException.class); - assertThrows(() -> DRPC.checkAuthorization(jt, auth, "execute", "jump"), AuthorizationException.class); + assertThrows(() -> DRPC.checkAuthorization(jt, auth, "execute", "jump"), + AuthorizationException.class); DRPC.checkAuthorization(jc, auth, "execute", "jump"); - assertThrows(() -> DRPC.checkAuthorization(other, auth, "execute", "jump"), AuthorizationException.class); - - //not_jump (closed in strict mode) - assertThrows(() -> DRPC.checkAuthorization(jt, auth, "fetchRequest", "not_jump"), AuthorizationException.class); - assertThrows(() -> DRPC.checkAuthorization(jc, auth, "fetchRequest", "not_jump"), AuthorizationException.class); - assertThrows(() -> DRPC.checkAuthorization(other, auth, "fetchRequest", "not_jump"), AuthorizationException.class); - - assertThrows(() -> DRPC.checkAuthorization(jt, auth, "result", "not_jump"), AuthorizationException.class); - assertThrows(() -> DRPC.checkAuthorization(jc, auth, "result", "not_jump"), AuthorizationException.class); - assertThrows(() -> DRPC.checkAuthorization(other, auth, "result", "not_jump"), AuthorizationException.class); - - assertThrows(() -> DRPC.checkAuthorization(jt, auth, "execute", "not_jump"), AuthorizationException.class); - assertThrows(() -> DRPC.checkAuthorization(jc, auth, "execute", "not_jump"), AuthorizationException.class); - assertThrows(() -> DRPC.checkAuthorization(other, auth, "execute", "not_jump"), AuthorizationException.class); + assertThrows(() -> DRPC.checkAuthorization(other, auth, "execute", "jump"), + AuthorizationException.class); + + // not_jump (closed in strict mode) + assertThrows(() -> DRPC.checkAuthorization(jt, auth, "fetchRequest", "not_jump"), + AuthorizationException.class); + assertThrows(() -> DRPC.checkAuthorization(jc, auth, "fetchRequest", "not_jump"), + AuthorizationException.class); + assertThrows(() -> DRPC.checkAuthorization(other, auth, "fetchRequest", "not_jump"), + AuthorizationException.class); + + assertThrows(() -> DRPC.checkAuthorization(jt, auth, "result", "not_jump"), + AuthorizationException.class); + assertThrows(() -> DRPC.checkAuthorization(jc, auth, "result", "not_jump"), + AuthorizationException.class); + assertThrows(() -> DRPC.checkAuthorization(other, auth, "result", "not_jump"), + AuthorizationException.class); + + assertThrows(() -> DRPC.checkAuthorization(jt, auth, "execute", "not_jump"), + AuthorizationException.class); + assertThrows(() -> DRPC.checkAuthorization(jc, auth, "execute", "not_jump"), + AuthorizationException.class); + assertThrows(() -> DRPC.checkAuthorization(other, auth, "execute", "not_jump"), + AuthorizationException.class); } @Test @@ -321,7 +343,8 @@ public void testNotStrict() throws Exception { other.subject().getPrincipals().add(otherUser); Map acl = new HashMap<>(); - acl.put("jump", new AclFunctionEntry(Collections.singletonList(jumpClient.getName()), jumpTopo.getName())); + acl.put("jump", new AclFunctionEntry(Collections.singletonList(jumpClient.getName()), + jumpTopo.getName())); Map conf = new HashMap<>(); conf.put(Config.DRPC_AUTHORIZER_ACL_STRICT, false); conf.put(Config.STORM_PRINCIPAL_TO_LOCAL_PLUGIN, DefaultPrincipalToLocal.class.getName()); @@ -332,20 +355,26 @@ protected Map readAclFromConfig() { } }; auth.prepare(conf); - //JUMP + // JUMP DRPC.checkAuthorization(jt, auth, "fetchRequest", "jump"); - assertThrows(() -> DRPC.checkAuthorization(jc, auth, "fetchRequest", "jump"), AuthorizationException.class); - assertThrows(() -> DRPC.checkAuthorization(other, auth, "fetchRequest", "jump"), AuthorizationException.class); + assertThrows(() -> DRPC.checkAuthorization(jc, auth, "fetchRequest", "jump"), + AuthorizationException.class); + assertThrows(() -> DRPC.checkAuthorization(other, auth, "fetchRequest", "jump"), + AuthorizationException.class); DRPC.checkAuthorization(jt, auth, "result", "jump"); - assertThrows(() -> DRPC.checkAuthorization(jc, auth, "result", "jump"), AuthorizationException.class); - assertThrows(() -> DRPC.checkAuthorization(other, auth, "result", "jump"), AuthorizationException.class); + assertThrows(() -> DRPC.checkAuthorization(jc, auth, "result", "jump"), + AuthorizationException.class); + assertThrows(() -> DRPC.checkAuthorization(other, auth, "result", "jump"), + AuthorizationException.class); - assertThrows(() -> DRPC.checkAuthorization(jt, auth, "execute", "jump"), AuthorizationException.class); + assertThrows(() -> DRPC.checkAuthorization(jt, auth, "execute", "jump"), + AuthorizationException.class); DRPC.checkAuthorization(jc, auth, "execute", "jump"); - assertThrows(() -> DRPC.checkAuthorization(other, auth, "execute", "jump"), AuthorizationException.class); + assertThrows(() -> DRPC.checkAuthorization(other, auth, "execute", "jump"), + AuthorizationException.class); - //not_jump (open in not strict mode) + // not_jump (open in not strict mode) DRPC.checkAuthorization(jt, auth, "fetchRequest", "not_jump"); DRPC.checkAuthorization(jc, auth, "fetchRequest", "not_jump"); DRPC.checkAuthorization(other, auth, "fetchRequest", "not_jump"); diff --git a/storm-server/src/test/java/org/apache/storm/daemon/metrics/MetricsUtilsTest.java b/storm-server/src/test/java/org/apache/storm/daemon/metrics/MetricsUtilsTest.java index 6c02ed43410..e5a30dd037a 100644 --- a/storm-server/src/test/java/org/apache/storm/daemon/metrics/MetricsUtilsTest.java +++ b/storm-server/src/test/java/org/apache/storm/daemon/metrics/MetricsUtilsTest.java @@ -1,12 +1,17 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -41,7 +46,8 @@ public void getPreparableReporters() { assertEquals(1, reporters.size()); assertTrue(reporters.get(0) instanceof JmxPreparableReporter); - List reporterPlugins = Arrays.asList("org.apache.storm.daemon.metrics.reporters.ConsolePreparableReporter", + List reporterPlugins = Arrays + .asList("org.apache.storm.daemon.metrics.reporters.ConsolePreparableReporter", "org.apache.storm.daemon.metrics.reporters.CsvPreparableReporter"); daemonConf.put(DaemonConfig.STORM_DAEMON_METRICS_REPORTER_PLUGINS, reporterPlugins); diff --git a/storm-server/src/test/java/org/apache/storm/daemon/nimbus/HeartbeatCacheTest.java b/storm-server/src/test/java/org/apache/storm/daemon/nimbus/HeartbeatCacheTest.java index 5ae1ee31546..85aeb5be376 100644 --- a/storm-server/src/test/java/org/apache/storm/daemon/nimbus/HeartbeatCacheTest.java +++ b/storm-server/src/test/java/org/apache/storm/daemon/nimbus/HeartbeatCacheTest.java @@ -18,13 +18,15 @@ package org.apache.storm.daemon.nimbus; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; - import org.apache.storm.generated.Assignment; import org.apache.storm.generated.ExecutorInfo; import org.apache.storm.generated.NodeInfo; @@ -33,9 +35,6 @@ import org.apache.storm.utils.Time; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - class HeartbeatCacheTest { private static final String TOPO_ID = "test-topology-1"; private static final int TIMEOUT_SECS = 30; @@ -57,15 +56,20 @@ void testExecutorRemainsAliveWhenHeartbeatTimestampDoesNotAdvance() { cache.updateHeartbeat(mkWorkerHeartbeat(TOPO_ID, 100, 1, 1), TIMEOUT_SECS); // Advance just 2 more seconds: now t = TIMEOUT_SECS + 1, which is past the original - // timeout window (rooted at t=0) but well within the refreshed window (rooted at t=TIMEOUT_SECS-1). + // timeout window (rooted at t=0) but well within the refreshed window (rooted at + // t=TIMEOUT_SECS-1). Time.advanceTimeSecs(2); // Simulate the scheduling cycle timeout check cache.timeoutOldHeartbeats(TOPO_ID, TIMEOUT_SECS); - // Executor should still be alive because a fresh heartbeat was received at t=(TIMEOUT_SECS-1) - Set> alive = cache.getAliveExecutors(TOPO_ID, allExecutors, assignment, TIMEOUT_SECS); - assertFalse(alive.isEmpty(), "Executor should be alive after receiving a recent heartbeat even if TIME_SECS did not advance"); + // Executor should still be alive because a fresh heartbeat was received at + // t=(TIMEOUT_SECS-1) + Set> alive = cache.getAliveExecutors(TOPO_ID, allExecutors, assignment, + TIMEOUT_SECS); + assertFalse(alive.isEmpty(), + "Executor should be alive after receiving a recent heartbeat even if " + + "TIME_SECS did not advance"); } } @@ -85,8 +89,10 @@ void testExecutorTimesOutWhenNoHeartbeatReceived() { // Simulate the scheduling cycle timeout check cache.timeoutOldHeartbeats(TOPO_ID, TIMEOUT_SECS); - Set> alive = cache.getAliveExecutors(TOPO_ID, allExecutors, assignment, TIMEOUT_SECS); - assertTrue(alive.isEmpty(), "Executor should be timed out after no heartbeat for longer than timeout"); + Set> alive = cache.getAliveExecutors(TOPO_ID, allExecutors, assignment, + TIMEOUT_SECS); + assertTrue(alive.isEmpty(), + "Executor should be timed out after no heartbeat for longer than timeout"); } } @@ -99,15 +105,18 @@ void testExecutorAliveWithRegularHeartbeats() { // Send heartbeats every second for 60 seconds for (int t = 0; t < 60; t++) { - cache.updateHeartbeat(mkWorkerHeartbeat(TOPO_ID, Time.currentTimeSecs(), 1, 1), TIMEOUT_SECS); + cache.updateHeartbeat(mkWorkerHeartbeat(TOPO_ID, Time.currentTimeSecs(), 1, 1), + TIMEOUT_SECS); Time.advanceTimeSecs(1); } // Simulate the scheduling cycle timeout check cache.timeoutOldHeartbeats(TOPO_ID, TIMEOUT_SECS); - Set> alive = cache.getAliveExecutors(TOPO_ID, allExecutors, assignment, TIMEOUT_SECS); - assertFalse(alive.isEmpty(), "Executor should be alive when receiving regular heartbeats"); + Set> alive = cache.getAliveExecutors(TOPO_ID, allExecutors, assignment, + TIMEOUT_SECS); + assertFalse(alive.isEmpty(), + "Executor should be alive when receiving regular heartbeats"); } } @@ -120,21 +129,26 @@ void testZkExecutorTimesOutWhenTimeSecsStopsAdvancing() { // Heartbeats with advancing TIME_SECS — executor is healthy for (int t = 0; t < 5; t++) { - cache.updateFromZkHeartbeat(TOPO_ID, mkZkExecutorBeats(1, 1, t * 10), allExecutors, TIMEOUT_SECS); + cache.updateFromZkHeartbeat(TOPO_ID, mkZkExecutorBeats(1, 1, t * 10), allExecutors, + TIMEOUT_SECS); Time.advanceTimeSecs(1); } // TIME_SECS freezes — zombie executor keeps sending heartbeats but stats are stuck int frozenTimeSecs = 40; for (int t = 0; t < TIMEOUT_SECS + 1; t++) { - cache.updateFromZkHeartbeat(TOPO_ID, mkZkExecutorBeats(1, 1, frozenTimeSecs), allExecutors, TIMEOUT_SECS); + cache.updateFromZkHeartbeat(TOPO_ID, mkZkExecutorBeats(1, 1, frozenTimeSecs), + allExecutors, TIMEOUT_SECS); Time.advanceTimeSecs(1); } cache.timeoutOldHeartbeats(TOPO_ID, TIMEOUT_SECS); - Set> alive = cache.getAliveExecutors(TOPO_ID, allExecutors, assignment, TIMEOUT_SECS); - assertTrue(alive.isEmpty(), "ZK executor should be timed out when TIME_SECS stops advancing (zombie detection)"); + Set> alive = cache.getAliveExecutors(TOPO_ID, allExecutors, assignment, + TIMEOUT_SECS); + assertTrue(alive.isEmpty(), + "ZK executor should be timed out when TIME_SECS stops advancing (zombie " + + "detection)"); } } @@ -147,18 +161,22 @@ void testZkExecutorAliveWhenTimeSecsAdvances() { // Heartbeats with advancing TIME_SECS every second for (int t = 0; t < 60; t++) { - cache.updateFromZkHeartbeat(TOPO_ID, mkZkExecutorBeats(1, 1, t), allExecutors, TIMEOUT_SECS); + cache.updateFromZkHeartbeat(TOPO_ID, mkZkExecutorBeats(1, 1, t), allExecutors, + TIMEOUT_SECS); Time.advanceTimeSecs(1); } cache.timeoutOldHeartbeats(TOPO_ID, TIMEOUT_SECS); - Set> alive = cache.getAliveExecutors(TOPO_ID, allExecutors, assignment, TIMEOUT_SECS); - assertFalse(alive.isEmpty(), "ZK executor should be alive when TIME_SECS advances regularly"); + Set> alive = cache.getAliveExecutors(TOPO_ID, allExecutors, assignment, + TIMEOUT_SECS); + assertFalse(alive.isEmpty(), + "ZK executor should be alive when TIME_SECS advances regularly"); } } - private SupervisorWorkerHeartbeat mkWorkerHeartbeat(String topoId, int timeSecs, int... executors) { + private SupervisorWorkerHeartbeat mkWorkerHeartbeat(String topoId, int timeSecs, + int... executors) { SupervisorWorkerHeartbeat hb = new SupervisorWorkerHeartbeat(); hb.set_storm_id(topoId); hb.set_time_secs(timeSecs); @@ -171,8 +189,8 @@ private SupervisorWorkerHeartbeat mkWorkerHeartbeat(String topoId, int timeSecs, return hb; } - - private Map, Map> mkZkExecutorBeats(int taskStart, int taskEnd, int timeSecs) { + private Map, Map> mkZkExecutorBeats(int taskStart, int taskEnd, + int timeSecs) { Map beat = new HashMap<>(); beat.put(ClientStatsUtil.TIME_SECS, timeSecs); return Collections.singletonMap(Arrays.asList(taskStart, taskEnd), beat); diff --git a/storm-server/src/test/java/org/apache/storm/daemon/nimbus/NimbusClojurePortTest.java b/storm-server/src/test/java/org/apache/storm/daemon/nimbus/NimbusClojurePortTest.java index 0975e52a57f..676c93f074c 100644 --- a/storm-server/src/test/java/org/apache/storm/daemon/nimbus/NimbusClojurePortTest.java +++ b/storm-server/src/test/java/org/apache/storm/daemon/nimbus/NimbusClojurePortTest.java @@ -18,6 +18,16 @@ package org.apache.storm.daemon.nimbus; +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.awaitility.Awaitility.await; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + import java.io.File; import java.io.IOException; import java.util.ArrayList; @@ -31,9 +41,8 @@ import java.util.Set; import java.util.function.UnaryOperator; import java.util.stream.Collectors; - import javax.security.auth.Subject; - +import net.minidev.json.JSONValue; import org.apache.commons.io.FileUtils; import org.apache.storm.Config; import org.apache.storm.DaemonConfig; @@ -75,22 +84,10 @@ import org.apache.storm.utils.ConfigUtils; import org.apache.storm.utils.Time; import org.apache.storm.utils.Utils; - -import net.minidev.json.JSONValue; import org.junit.jupiter.api.Test; import org.mockito.ArgumentMatchers; import org.mockito.Mockito; -import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static java.util.concurrent.TimeUnit.SECONDS; -import static org.awaitility.Awaitility.await; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; - /** * Tests ported from storm-core/test/clj/org/apache/storm/nimbus_test.clj. * Batch 7a: Simple/unit-like tests. @@ -111,7 +108,8 @@ public void testBogusId() throws Exception { assertThrows(NotAliveException.class, () -> nimbus.getTopology("bogus-id")); assertThrows(NotAliveException.class, () -> nimbus.getUserTopology("bogus-id")); assertThrows(NotAliveException.class, () -> nimbus.getTopologyInfo("bogus-id")); - assertThrows(NotAliveException.class, () -> nimbus.uploadNewCredentials("bogus-id", new Credentials())); + assertThrows(NotAliveException.class, () -> nimbus.uploadNewCredentials("bogus-id", + new Credentials())); } } @@ -119,8 +117,10 @@ public void testBogusId() throws Exception { public void testNimbusIfaceSubmitTopologyWithOptsChecksAuthorization() throws Exception { try (LocalCluster cluster = new LocalCluster.Builder() .withDaemonConf(Map.of( - DaemonConfig.NIMBUS_AUTHORIZER, "org.apache.storm.security.auth.authorizer.DenyAuthorizer", - DaemonConfig.SUPERVISOR_AUTHORIZER, "org.apache.storm.security.auth.authorizer.DenyAuthorizer")) + DaemonConfig.NIMBUS_AUTHORIZER, "org.apache.storm.security.auth.authorizer.Den" + + "yAuthorizer", + DaemonConfig.SUPERVISOR_AUTHORIZER, "org.apache.storm.security.auth.authorizer" + + ".DenyAuthorizer")) .build()) { StormTopology topology = Thrift.buildTopology(Map.of(), Map.of()); assertThrows(AuthorizationException.class, () -> @@ -140,8 +140,10 @@ public void testNimbusIfaceMethodsCheckAuthorization() throws Exception { .withBlobStore(blobStore) .withTopoCache(tc) .withDaemonConf(Map.of( - DaemonConfig.NIMBUS_AUTHORIZER, "org.apache.storm.security.auth.authorizer.DenyAuthorizer", - DaemonConfig.SUPERVISOR_AUTHORIZER, "org.apache.storm.security.auth.authorizer.DenyAuthorizer")) + DaemonConfig.NIMBUS_AUTHORIZER, "org.apache.storm.security.auth.authorizer.Den" + + "yAuthorizer", + DaemonConfig.SUPERVISOR_AUTHORIZER, "org.apache.storm.security.auth.authorizer" + + ".DenyAuthorizer")) .build()) { Nimbus nimbus = cluster.getNimbus(); String topologyName = "test"; @@ -164,16 +166,20 @@ public void testNimbusIfaceGetTopologyMethodsThrowCorrectly() throws Exception { Nimbus nimbus = cluster.getNimbus(); String id = "bogus ID"; - NotAliveException e1 = assertThrows(NotAliveException.class, () -> nimbus.getTopology(id)); + NotAliveException e1 = assertThrows(NotAliveException.class, () -> nimbus + .getTopology(id)); assertEquals(id, e1.get_msg()); - NotAliveException e2 = assertThrows(NotAliveException.class, () -> nimbus.getTopologyConf(id)); + NotAliveException e2 = assertThrows(NotAliveException.class, () -> nimbus + .getTopologyConf(id)); assertEquals(id, e2.get_msg()); - NotAliveException e3 = assertThrows(NotAliveException.class, () -> nimbus.getTopologyInfo(id)); + NotAliveException e3 = assertThrows(NotAliveException.class, () -> nimbus + .getTopologyInfo(id)); assertEquals(id, e3.get_msg()); - NotAliveException e4 = assertThrows(NotAliveException.class, () -> nimbus.getUserTopology(id)); + NotAliveException e4 = assertThrows(NotAliveException.class, () -> nimbus + .getUserTopology(id)); assertEquals(id, e4.get_msg()); } } @@ -220,10 +226,13 @@ public void testNimbusIfaceGetClusterInfoFiltersToposWithoutBases() throws Excep topology.set_bolts(Map.of()); topology.set_state_spouts(Map.of()); - Mockito.when(clusterState.stormBase(Mockito.any(String.class), ArgumentMatchers.any())).thenReturn(new StormBase()); + Mockito.when(clusterState.stormBase(Mockito.any(String.class), ArgumentMatchers.any())) + .thenReturn(new StormBase()); Mockito.when(clusterState.topologyBases()).thenReturn(bogusBasesMap); - Mockito.when(tc.readTopoConf(Mockito.any(String.class), Mockito.any(Subject.class))).thenReturn(topoConf); - Mockito.when(tc.readTopology(Mockito.any(String.class), Mockito.any(Subject.class))).thenReturn(topology); + Mockito.when(tc.readTopoConf(Mockito.any(String.class), Mockito.any(Subject.class))) + .thenReturn(topoConf); + Mockito.when(tc.readTopology(Mockito.any(String.class), Mockito.any(Subject.class))) + .thenReturn(topology); List topos = cluster.getNimbus().getClusterInfo().get_topologies(); // Only topologies with non-null bases should be present @@ -252,8 +261,10 @@ public void testValidateTopoConfigOnSubmit() throws Exception { .withBlobStore(blobStore) .withTopoCache(tc) .withDaemonConf(Map.of( - DaemonConfig.NIMBUS_AUTHORIZER, "org.apache.storm.security.auth.authorizer.NoopAuthorizer", - DaemonConfig.SUPERVISOR_AUTHORIZER, "org.apache.storm.security.auth.authorizer.NoopAuthorizer")) + DaemonConfig.NIMBUS_AUTHORIZER, "org.apache.storm.security.auth.authorizer.Noo" + + "pAuthorizer", + DaemonConfig.SUPERVISOR_AUTHORIZER, "org.apache.storm.security.auth.authorizer" + + ".NoopAuthorizer")) .build()) { Mockito.when(clusterState.getTopoId("test")).thenReturn(Optional.empty()); StormTopology topology = Thrift.buildTopology(Map.of(), Map.of()); @@ -299,8 +310,10 @@ public void emptySaveConfigResultsInAllUnchangedActions() throws Exception { .withBlobStore(blobStore) .withTopoCache(tc) .withDaemonConf(Map.of( - DaemonConfig.NIMBUS_AUTHORIZER, "org.apache.storm.security.auth.authorizer.NoopAuthorizer", - DaemonConfig.SUPERVISOR_AUTHORIZER, "org.apache.storm.security.auth.authorizer.NoopAuthorizer")) + DaemonConfig.NIMBUS_AUTHORIZER, "org.apache.storm.security.auth.authorizer.Noo" + + "pAuthorizer", + DaemonConfig.SUPERVISOR_AUTHORIZER, "org.apache.storm.security.auth.authorizer" + + ".NoopAuthorizer")) .build()) { Nimbus nimbus = cluster.getNimbus(); @@ -316,8 +329,10 @@ public void emptySaveConfigResultsInAllUnchangedActions() throws Exception { expectedLevel.set_action(LogLevelAction.UNCHANGED); expectedConfig.put_to_named_logger_level("test", expectedLevel); - Mockito.when(tc.readTopoConf(Mockito.any(String.class), ArgumentMatchers.any())).thenReturn(Map.of()); - Mockito.when(clusterState.topologyLogConfig(Mockito.any(String.class), ArgumentMatchers.any())).thenReturn(previousConfig); + Mockito.when(tc.readTopoConf(Mockito.any(String.class), ArgumentMatchers.any())) + .thenReturn(Map.of()); + Mockito.when(clusterState.topologyLogConfig(Mockito.any(String.class), ArgumentMatchers + .any())).thenReturn(previousConfig); LogConfig emptyConfig = new LogConfig(); nimbus.setLogConfig("foo", emptyConfig); @@ -338,8 +353,10 @@ public void logLevelUpdateMergesAndFlagsExistentLogLevel() throws Exception { .withBlobStore(blobStore) .withTopoCache(tc) .withDaemonConf(Map.of( - DaemonConfig.NIMBUS_AUTHORIZER, "org.apache.storm.security.auth.authorizer.NoopAuthorizer", - DaemonConfig.SUPERVISOR_AUTHORIZER, "org.apache.storm.security.auth.authorizer.NoopAuthorizer")) + DaemonConfig.NIMBUS_AUTHORIZER, "org.apache.storm.security.auth.authorizer.Noo" + + "pAuthorizer", + DaemonConfig.SUPERVISOR_AUTHORIZER, "org.apache.storm.security.auth.authorizer" + + ".NoopAuthorizer")) .build()) { Nimbus nimbus = cluster.getNimbus(); @@ -372,8 +389,10 @@ public void logLevelUpdateMergesAndFlagsExistentLogLevel() throws Exception { expectedLevel2.set_action(LogLevelAction.UNCHANGED); expectedConfig.put_to_named_logger_level("other-test", expectedLevel2); - Mockito.when(tc.readTopoConf(Mockito.any(String.class), ArgumentMatchers.any())).thenReturn(Map.of()); - Mockito.when(clusterState.topologyLogConfig(Mockito.any(String.class), ArgumentMatchers.any())).thenReturn(previousConfig); + Mockito.when(tc.readTopoConf(Mockito.any(String.class), ArgumentMatchers.any())) + .thenReturn(Map.of()); + Mockito.when(clusterState.topologyLogConfig(Mockito.any(String.class), ArgumentMatchers + .any())).thenReturn(previousConfig); nimbus.setLogConfig("foo", mockConfig); @@ -384,10 +403,12 @@ public void logLevelUpdateMergesAndFlagsExistentLogLevel() throws Exception { @Test public void cleanupStormIdsReturnsInactiveTopos() { - IStormClusterState mockState = mockClusterState(List.of("topo1"), List.of("topo1", "topo2", "topo3")); + IStormClusterState mockState = mockClusterState(List.of("topo1"), List.of("topo1", "topo2", + "topo3")); BlobStore store = Mockito.mock(BlobStore.class); Mockito.when(store.storedTopoIds()).thenReturn(Set.of()); - Map conf = Map.of(DaemonConfig.NIMBUS_TOPOLOGY_BLOBSTORE_DELETION_DELAY_MS, 0); + Map conf = Map.of(DaemonConfig.NIMBUS_TOPOLOGY_BLOBSTORE_DELETION_DELAY_MS, + 0); assertEquals(Set.of("topo2", "topo3"), Nimbus.topoIdsToClean(mockState, store, conf)); } @@ -398,21 +419,25 @@ public void cleanupStormIdsPerformsUnionOfStormIdsWithActiveZnodes() { List hbTopos = List.of("hb1", "hb2", "hb3"); List errorTopos = List.of("e1", "e2", "e3"); List bpTopos = List.of("bp1", "bp2", "bp3"); - IStormClusterState mockState = mockClusterState(activeTopos, hbTopos, errorTopos, bpTopos, null); + IStormClusterState mockState = mockClusterState(activeTopos, hbTopos, errorTopos, bpTopos, + null); BlobStore store = Mockito.mock(BlobStore.class); Mockito.when(store.storedTopoIds()).thenReturn(Set.of()); assertEquals(Set.of("hb2", "hb3", "e1", "e3", "bp1", "bp2"), - Nimbus.topoIdsToClean(mockState, store, Map.of(DaemonConfig.NIMBUS_TOPOLOGY_BLOBSTORE_DELETION_DELAY_MS, 0))); + Nimbus.topoIdsToClean(mockState, store, Map + .of(DaemonConfig.NIMBUS_TOPOLOGY_BLOBSTORE_DELETION_DELAY_MS, 0))); } @Test public void cleanupStormIdsReturnsEmptySetWhenAllToposAreActive() { - List activeTopos = List.of("hb1", "hb2", "hb3", "e1", "e2", "e3", "bp1", "bp2", "bp3"); + List activeTopos = List.of("hb1", "hb2", "hb3", "e1", "e2", "e3", "bp1", "bp2", + "bp3"); List hbTopos = List.of("hb1", "hb2", "hb3"); List errorTopos = List.of("e1", "e2", "e3"); List bpTopos = List.of("bp1", "bp2", "bp3"); - IStormClusterState mockState = mockClusterState(activeTopos, hbTopos, errorTopos, bpTopos, null); + IStormClusterState mockState = mockClusterState(activeTopos, hbTopos, errorTopos, bpTopos, + null); BlobStore store = Mockito.mock(BlobStore.class); Mockito.when(store.storedTopoIds()).thenReturn(Set.of()); @@ -434,7 +459,8 @@ public void doCleanupRemovesInactiveZnodes() throws Exception { mockLeaderElector(), null, new StormMetricsRegistry())); nimbus.getHeartbeatsCache().addEmptyTopoForTests("topo2"); nimbus.getHeartbeatsCache().addEmptyTopoForTests("topo3"); - Mockito.when(mockBlobStore.storedTopoIds()).thenReturn(new HashSet<>(List.of("topo2", "topo3"))); + Mockito.when(mockBlobStore.storedTopoIds()).thenReturn(new HashSet<>(List.of("topo2", + "topo3"))); nimbus.doCleanup(); @@ -534,7 +560,8 @@ public void userTopologiesForSupervisorWithUnauthorizedUser() throws Exception { List.of(1L, 1L), new NodeInfo("super1", Set.of(2L)), List.of(2L, 2L), new NodeInfo("super2", Set.of(2L)))); - Map assignments = Map.of("topo1", assignment, "authorized", assignment2); + Map assignments = Map.of("topo1", assignment, "authorized", + assignment2); IStormClusterState mockState = mockClusterState(null, null); BlobStore mockBlobStore = Mockito.mock(BlobStore.class); @@ -555,7 +582,8 @@ public void userTopologiesForSupervisorWithUnauthorizedUser() throws Exception { public void prepare(Map conf) {} @Override - public boolean permit(org.apache.storm.security.auth.ReqContext context, String operation, Map topoConf) { + public boolean permit(org.apache.storm.security.auth.ReqContext context, + String operation, Map topoConf) { return "authorized".equals(topoConf.get(Config.TOPOLOGY_NAME)); } }); @@ -607,21 +635,24 @@ public void testSubmitInvalid() throws Exception { .build()) { // Invalid topology name with slash StormTopology topology = Thrift.buildTopology( - Map.of("1", Thrift.prepareSpoutDetails(new TestPlannerSpout(true), 1, Map.of(Config.TOPOLOGY_TASKS, 1))), + Map.of("1", Thrift.prepareSpoutDetails(new TestPlannerSpout(true), 1, Map + .of(Config.TOPOLOGY_TASKS, 1))), Map.of()); assertThrows(InvalidTopologyException.class, () -> cluster.submitTopology("test/aaa", Map.of(), topology)); // Too many executors StormTopology topology2 = Thrift.buildTopology( - Map.of("1", Thrift.prepareSpoutDetails(new TestPlannerSpout(true), 16, Map.of(Config.TOPOLOGY_TASKS, 16))), + Map.of("1", Thrift.prepareSpoutDetails(new TestPlannerSpout(true), 16, Map + .of(Config.TOPOLOGY_TASKS, 16))), Map.of()); assertThrows(InvalidTopologyException.class, () -> cluster.submitTopology("test", Map.of(Config.TOPOLOGY_WORKERS, 3), topology2)); // Too many workers StormTopology topology3 = Thrift.buildTopology( - Map.of("1", Thrift.prepareSpoutDetails(new TestPlannerSpout(true), 5, Map.of(Config.TOPOLOGY_TASKS, 5))), + Map.of("1", Thrift.prepareSpoutDetails(new TestPlannerSpout(true), 5, Map + .of(Config.TOPOLOGY_TASKS, 5))), Map.of()); assertThrows(InvalidTopologyException.class, () -> cluster.submitTopology("test", Map.of(Config.TOPOLOGY_WORKERS, 16), topology3)); @@ -645,23 +676,29 @@ public void testAssignment() throws Exception { StormTopology topology = Thrift.buildTopology( Map.of("1", Thrift.prepareSpoutDetails(new TestPlannerSpout(false), 3)), Map.of("2", Thrift.prepareBoltDetails( - Map.of(Utils.getGlobalStreamId("1", null), Thrift.prepareNoneGrouping()), + Map.of(Utils.getGlobalStreamId("1", null), Thrift + .prepareNoneGrouping()), new TestPlannerBolt(), 4), "3", Thrift.prepareBoltDetails( - Map.of(Utils.getGlobalStreamId("2", null), Thrift.prepareNoneGrouping()), + Map.of(Utils.getGlobalStreamId("2", null), Thrift + .prepareNoneGrouping()), new TestPlannerBolt()))); StormTopology topology2 = Thrift.buildTopology( Map.of("1", Thrift.prepareSpoutDetails(new TestPlannerSpout(true), 12)), Map.of("2", Thrift.prepareBoltDetails( - Map.of(Utils.getGlobalStreamId("1", null), Thrift.prepareNoneGrouping()), + Map.of(Utils.getGlobalStreamId("1", null), Thrift + .prepareNoneGrouping()), new TestPlannerBolt(), 6), "3", Thrift.prepareBoltDetails( - Map.of(Utils.getGlobalStreamId("1", null), Thrift.prepareGlobalGrouping()), + Map.of(Utils.getGlobalStreamId("1", null), Thrift + .prepareGlobalGrouping()), new TestPlannerBolt(), 8), "4", Thrift.prepareBoltDetails( - Map.of(Utils.getGlobalStreamId("1", null), Thrift.prepareGlobalGrouping(), - Utils.getGlobalStreamId("2", null), Thrift.prepareNoneGrouping()), + Map.of(Utils.getGlobalStreamId("1", null), Thrift + .prepareGlobalGrouping(), + Utils.getGlobalStreamId("2", null), Thrift + .prepareNoneGrouping()), new TestPlannerBolt(), 4))); cluster.submitTopology("mystorm", Map.of(Config.TOPOLOGY_WORKERS, 4), topology); @@ -706,10 +743,12 @@ public void testZeroExecutorOrTasks() throws Exception { Map.of("1", Thrift.prepareSpoutDetails(new TestPlannerSpout(false), 3, Map.of(Config.TOPOLOGY_TASKS, 0))), Map.of("2", Thrift.prepareBoltDetails( - Map.of(Utils.getGlobalStreamId("1", null), Thrift.prepareNoneGrouping()), + Map.of(Utils.getGlobalStreamId("1", null), Thrift + .prepareNoneGrouping()), new TestPlannerBolt(), 1, Map.of(Config.TOPOLOGY_TASKS, 2)), "3", Thrift.prepareBoltDetails( - Map.of(Utils.getGlobalStreamId("2", null), Thrift.prepareNoneGrouping()), + Map.of(Utils.getGlobalStreamId("2", null), Thrift + .prepareNoneGrouping()), new TestPlannerBolt(), null, Map.of(Config.TOPOLOGY_TASKS, 5)))); cluster.submitTopology("mystorm", Map.of(Config.TOPOLOGY_WORKERS, 4), topology); @@ -741,13 +780,16 @@ public void testOverParallelismAssignment() throws Exception { StormTopology topology = Thrift.buildTopology( Map.of("1", Thrift.prepareSpoutDetails(new TestPlannerSpout(true), 21)), Map.of("2", Thrift.prepareBoltDetails( - Map.of(Utils.getGlobalStreamId("1", null), Thrift.prepareNoneGrouping()), + Map.of(Utils.getGlobalStreamId("1", null), Thrift + .prepareNoneGrouping()), new TestPlannerBolt(), 9), "3", Thrift.prepareBoltDetails( - Map.of(Utils.getGlobalStreamId("1", null), Thrift.prepareNoneGrouping()), + Map.of(Utils.getGlobalStreamId("1", null), Thrift + .prepareNoneGrouping()), new TestPlannerBolt(), 2), "4", Thrift.prepareBoltDetails( - Map.of(Utils.getGlobalStreamId("1", null), Thrift.prepareNoneGrouping()), + Map.of(Utils.getGlobalStreamId("1", null), Thrift + .prepareNoneGrouping()), new TestPlannerBolt(), 10))); cluster.submitTopology("test", Map.of(Config.TOPOLOGY_WORKERS, 7), topology); @@ -858,10 +900,12 @@ public void testAutoCredentials() throws Exception { StormTopology topology = Thrift.buildTopology( Map.of("1", Thrift.prepareSpoutDetails(new TestPlannerSpout(false), 3)), Map.of("2", Thrift.prepareBoltDetails( - Map.of(Utils.getGlobalStreamId("1", null), Thrift.prepareNoneGrouping()), + Map.of(Utils.getGlobalStreamId("1", null), Thrift + .prepareNoneGrouping()), new TestPlannerBolt(), 4), "3", Thrift.prepareBoltDetails( - Map.of(Utils.getGlobalStreamId("2", null), Thrift.prepareNoneGrouping()), + Map.of(Utils.getGlobalStreamId("2", null), Thrift + .prepareNoneGrouping()), new TestPlannerBolt()))); cluster.submitTopologyWithOpts(topologyName, Map.of( @@ -889,12 +933,15 @@ public void testAutoCredentials() throws Exception { @SuppressWarnings("unchecked") private static Map fromJson(String str) { - if (str == null) return null; + if (str == null) { + return null; + } return (Map) JSONValue.parse(str); } @SuppressWarnings("unchecked") - private static Map> stormComponentToTaskInfo(LocalCluster cluster, String stormName) throws Exception { + private static Map> stormComponentToTaskInfo(LocalCluster cluster, + String stormName) throws Exception { IStormClusterState state = cluster.getClusterState(); String stormId = state.getTopoId(stormName).get(); Nimbus nimbus = cluster.getNimbus(); @@ -905,7 +952,8 @@ private static Map> stormComponentToTaskInfo(LocalCluster // reverse: component -> list of task ids Map> result = new HashMap<>(); for (Map.Entry entry : taskToComponent.entrySet()) { - result.computeIfAbsent(entry.getValue(), k -> new java.util.ArrayList<>()).add(entry.getKey()); + result.computeIfAbsent(entry.getValue(), k -> new java.util.ArrayList<>()).add(entry + .getKey()); } return result; } @@ -921,11 +969,14 @@ private static Map getCredentials(LocalCluster cluster, String s IStormClusterState state = cluster.getClusterState(); String stormId = state.getTopoId(stormName).get(); Credentials creds = state.credentials(stormId, null); - if (creds == null) return null; + if (creds == null) { + return null; + } return new HashMap<>(creds.get_creds()); } - private static void checkConsistency(LocalCluster cluster, String stormName, boolean shouldBeAssigned) throws Exception { + private static void checkConsistency(LocalCluster cluster, String stormName, + boolean shouldBeAssigned) throws Exception { IStormClusterState state = cluster.getClusterState(); String stormId = state.getTopoId(stormName).get(); Nimbus nimbus = cluster.getNimbus(); @@ -974,7 +1025,8 @@ private static void checkConsistency(LocalCluster cluster, String stormName, boo } } - private static void createFile(String dirLocation, String name, int secondsAgo) throws IOException { + private static void createFile(String dirLocation, String name, + int secondsAgo) throws IOException { File f = new File(dirLocation + "/" + name); FileUtils.touch(f); long t = Time.currentTimeMillis() - (secondsAgo * 1000L); @@ -1002,7 +1054,20 @@ private static ILeaderElector mockLeaderElector() { return elector; } - private static IStormClusterState mockClusterState(List activeTopos, List inactiveTopos) { + private static ILeaderElector mockLeaderElector(boolean isLeader) { + ILeaderElector elector = Mockito.mock(ILeaderElector.class); + try { + Mockito.when(elector.isLeader()).thenReturn(isLeader); + Mockito.when(elector.getLeader()).thenReturn(new NimbusInfo("test-host", 9999, false)); + Mockito.when(elector.getAllNimbuses()).thenReturn(List.of()); + } catch (Exception e) { + throw new RuntimeException(e); + } + return elector; + } + + private static IStormClusterState mockClusterState(List activeTopos, + List inactiveTopos) { return mockClusterState(activeTopos, inactiveTopos, inactiveTopos, inactiveTopos, null); } @@ -1020,18 +1085,6 @@ private static IStormClusterState mockClusterState( return state; } - private static ILeaderElector mockLeaderElector(boolean isLeader) { - ILeaderElector elector = Mockito.mock(ILeaderElector.class); - try { - Mockito.when(elector.isLeader()).thenReturn(isLeader); - Mockito.when(elector.getLeader()).thenReturn(new NimbusInfo("test-host", 9999, false)); - Mockito.when(elector.getAllNimbuses()).thenReturn(List.of()); - } catch (Exception e) { - throw new RuntimeException(e); - } - return elector; - } - // --- Additional helpers for batch 7d/7e --- private static Set topologyNodes(IStormClusterState state, String stormName) { @@ -1055,7 +1108,8 @@ private static Set topologySlots(IStormClusterState state, String stor } /** Returns a map like {nodeCount: numberOfNodesWithThatCount}. */ - private static Map topologyNodeDistribution(IStormClusterState state, String stormName) { + private static Map topologyNodeDistribution(IStormClusterState state, + String stormName) { String stormId = state.getTopoId(stormName).get(); Assignment assignment = state.assignmentInfo(stormId, null); Set slots = new HashSet<>(assignment.get_executor_node_port().values()); @@ -1070,7 +1124,8 @@ private static Map topologyNodeDistribution(IStormClusterState return distribution; } - private static NodeInfo executorAssignment(LocalCluster cluster, String stormId, List executorId) { + private static NodeInfo executorAssignment(LocalCluster cluster, String stormId, + List executorId) { IStormClusterState state = cluster.getClusterState(); Assignment assignment = state.assignmentInfo(stormId, null); return assignment.get_executor_node_port().get(executorId); @@ -1090,19 +1145,23 @@ private static List> topologyExecutors(LocalCluster cluster, String s /** Reverse map: NodeInfo -> List> (slot -> executors). */ @SuppressWarnings("unchecked") - private static Map>> slotAssignments(LocalCluster cluster, String stormId) { + private static Map>> slotAssignments(LocalCluster cluster, + String stormId) { IStormClusterState state = cluster.getClusterState(); Assignment assignment = state.assignmentInfo(stormId, null); Map>> result = new HashMap<>(); - for (Map.Entry, NodeInfo> entry : assignment.get_executor_node_port().entrySet()) { + for (Map.Entry, NodeInfo> entry : assignment.get_executor_node_port() + .entrySet()) { result.computeIfAbsent(entry.getValue(), k -> new ArrayList<>()).add(entry.getKey()); } return result; } - private static void doExecutorHeartbeat(LocalCluster cluster, String stormId, List executor) throws Exception { + private static void doExecutorHeartbeat(LocalCluster cluster, String stormId, + List executor) throws Exception { IStormClusterState state = cluster.getClusterState(); - Map, NodeInfo> executorNodePort = state.assignmentInfo(stormId, null).get_executor_node_port(); + Map, NodeInfo> executorNodePort = state.assignmentInfo(stormId, null) + .get_executor_node_port(); NodeInfo np = executorNodePort.get(executor); String node = np.get_node(); long port = np.get_port().iterator().next(); @@ -1113,7 +1172,8 @@ private static void doExecutorHeartbeat(LocalCluster cluster, String stormId, Li Map, org.apache.storm.generated.ExecutorStats> stats = new HashMap<>(); stats.put(org.apache.storm.stats.ClientStatsUtil.convertExecutor(executor), new org.apache.storm.stats.BoltExecutorStats(20, - ((Number) ConfigUtils.readStormConfig().getOrDefault(Config.NUM_STAT_BUCKETS, 20)).intValue() + ((Number) ConfigUtils.readStormConfig().getOrDefault(Config.NUM_STAT_BUCKETS, 20)) + .intValue() ).renderStats()); state.workerHeartbeat(stormId, node, port, @@ -1123,18 +1183,21 @@ private static void doExecutorHeartbeat(LocalCluster cluster, String stormId, Li org.apache.storm.stats.StatsUtil.thriftifyRpcWorkerHb(stormId, executor)); } - private static void checkDistribution(Collection> items, List expectedDistribution) { + private static void checkDistribution(Collection> items, + List expectedDistribution) { List counts = new ArrayList<>(); for (Collection item : items) { counts.add((long) item.size()); } - List expected = expectedDistribution.stream().map(Long::valueOf).collect(Collectors.toList()); + List expected = expectedDistribution.stream().map(Long::valueOf).collect(Collectors + .toList()); java.util.Collections.sort(counts); java.util.Collections.sort(expected); assertEquals(expected, counts); } - private static void checkExecutorDistribution(Map>> slotExecutors, List distribution) { + private static void checkExecutorDistribution(Map>> slotExecutors, + List distribution) { checkDistribution(slotExecutors.values(), distribution); } @@ -1155,7 +1218,8 @@ private static void checkForCollisions(IStormClusterState state) { Set nodePorts = new HashSet<>(executorNodePort.values()); Map> nodeToPorts = new HashMap<>(); for (NodeInfo np : nodePorts) { - nodeToPorts.computeIfAbsent(np.get_node(), k -> new HashSet<>()).addAll(np.get_port()); + nodeToPorts.computeIfAbsent(np.get_node(), k -> new HashSet<>()).addAll(np + .get_port()); } idToNodeToPorts.put(id, nodeToPorts); } @@ -1165,14 +1229,16 @@ private static void checkForCollisions(IStormClusterState state) { for (Map.Entry> entry : nodeToPorts.entrySet()) { Set existing = combined.computeIfAbsent(entry.getKey(), k -> new HashSet<>()); for (Long port : entry.getValue()) { - assertTrue(existing.add(port), "Port collision on node " + entry.getKey() + " port " + port); + assertTrue(existing.add(port), "Port collision on node " + entry.getKey() + + " port " + port); } } } } @SuppressWarnings("unchecked") - private static Map>> stormComponentToExecutorInfo(LocalCluster cluster, String stormName) throws Exception { + private static Map>> stormComponentToExecutorInfo(LocalCluster cluster, + String stormName) throws Exception { IStormClusterState state = cluster.getClusterState(); String stormId = state.getTopoId(stormName).get(); Nimbus nimbus = cluster.getNimbus(); @@ -1232,20 +1298,26 @@ public void testIsolatedAssignment() throws Exception { @Override public void prepare(Map topoConf, String schedulerLocalDir) { standalone.prepare(topoConf, schedulerLocalDir); } + @Override public Collection allSlotsAvailableForScheduling( Collection supervisors, org.apache.storm.scheduler.Topologies topologies, Set topologiesMissingAssignments) { - return standalone.allSlotsAvailableForScheduling(supervisors, topologies, topologiesMissingAssignments); + return standalone.allSlotsAvailableForScheduling(supervisors, topologies, + topologiesMissingAssignments); } + @Override public void assignSlots(org.apache.storm.scheduler.Topologies topologies, Map> newSlotsByTopologyId) { standalone.assignSlots(topologies, newSlotsByTopologyId); } - @Override public String getHostName(Map supervisors, + + @Override public String getHostName(Map supervisors, String nodeId) { return nodeId; } + @Override public org.apache.storm.scheduler.IScheduler getForcedScheduler() { return standalone.getForcedScheduler(); } @@ -1268,10 +1340,12 @@ public void testIsolatedAssignment() throws Exception { StormTopology topology = Thrift.buildTopology( Map.of("1", Thrift.prepareSpoutDetails(new TestPlannerSpout(false), 3)), Map.of("2", Thrift.prepareBoltDetails( - Map.of(Utils.getGlobalStreamId("1", null), Thrift.prepareNoneGrouping()), + Map.of(Utils.getGlobalStreamId("1", null), Thrift + .prepareNoneGrouping()), new TestPlannerBolt(), 5), "3", Thrift.prepareBoltDetails( - Map.of(Utils.getGlobalStreamId("2", null), Thrift.prepareNoneGrouping()), + Map.of(Utils.getGlobalStreamId("2", null), Thrift + .prepareNoneGrouping()), new TestPlannerBolt(), null))); cluster.submitTopology("noniso", Map.of(Config.TOPOLOGY_WORKERS, 4), topology); @@ -1324,19 +1398,23 @@ public void testExecutorAssignments() throws Exception { .build()) { StormTopology topology = Thrift.buildTopology( - Map.of("1", Thrift.prepareSpoutDetails(new TestPlannerSpout(true), 3, Map.of(Config.TOPOLOGY_TASKS, 5))), + Map.of("1", Thrift.prepareSpoutDetails(new TestPlannerSpout(true), 3, Map + .of(Config.TOPOLOGY_TASKS, 5))), Map.of("2", Thrift.prepareBoltDetails( - Map.of(Utils.getGlobalStreamId("1", null), Thrift.prepareNoneGrouping()), + Map.of(Utils.getGlobalStreamId("1", null), Thrift + .prepareNoneGrouping()), new TestPlannerBolt(), 8, Map.of(Config.TOPOLOGY_TASKS, 2)), "3", Thrift.prepareBoltDetails( - Map.of(Utils.getGlobalStreamId("2", null), Thrift.prepareNoneGrouping()), + Map.of(Utils.getGlobalStreamId("2", null), Thrift + .prepareNoneGrouping()), new TestPlannerBolt(), 3))); cluster.submitTopology("mystorm", Map.of(Config.TOPOLOGY_WORKERS, 4), topology); cluster.advanceClusterTime(11); Map> taskInfo = stormComponentToTaskInfo(cluster, "mystorm"); - Map>> executorInfo = stormComponentToExecutorInfo(cluster, "mystorm"); + Map>> executorInfo = stormComponentToExecutorInfo(cluster, + "mystorm"); checkConsistency(cluster, "mystorm", true); @@ -1380,7 +1458,8 @@ public void testTopoHistory() throws Exception { DaemonConfig.NIMBUS_MONITOR_FREQ_SECS, 10, Config.TOPOLOGY_ACKER_EXECUTORS, 0)) .build()) { - Mockito.when(groupMapper.getGroups(ArgumentMatchers.any())).thenReturn(Set.of("alice-group")); + Mockito.when(groupMapper.getGroups(ArgumentMatchers.any())).thenReturn(Set + .of("alice-group")); IStormClusterState state = cluster.getClusterState(); Nimbus nimbus = cluster.getNimbus(); @@ -1443,7 +1522,8 @@ public void testTopoHistory() throws Exception { String stormId4 = state.getTopoId("testreadgroup").get(); // Current user should see 4 topologies - List histIds = new ArrayList<>(nimbus.getTopologyHistory(currentUser).get_topo_ids()); + List histIds = new ArrayList<>(nimbus.getTopologyHistory(currentUser) + .get_topo_ids()); java.util.Collections.sort(histIds); assertEquals(4, histIds.size()); assertEquals(stormId2, histIds.get(0)); @@ -1452,17 +1532,20 @@ public void testTopoHistory() throws Exception { assertEquals(stormId4, histIds.get(3)); // Alice should see 5 topologies - List aliceIds = new ArrayList<>(nimbus.getTopologyHistory("alice").get_topo_ids()); + List aliceIds = new ArrayList<>(nimbus.getTopologyHistory("alice") + .get_topo_ids()); java.util.Collections.sort(aliceIds); assertEquals(5, aliceIds.size()); // Admin should see all 6 - List adminIds = new ArrayList<>(nimbus.getTopologyHistory("admin-user").get_topo_ids()); + List adminIds = new ArrayList<>(nimbus.getTopologyHistory("admin-user") + .get_topo_ids()); java.util.Collections.sort(adminIds); assertEquals(6, adminIds.size()); // Group-only user should see 2 - List groupOnlyIds = new ArrayList<>(nimbus.getTopologyHistory("group-only-user").get_topo_ids()); + List groupOnlyIds = new ArrayList<>(nimbus.getTopologyHistory("group-only-user") + .get_topo_ids()); java.util.Collections.sort(groupOnlyIds); assertEquals(2, groupOnlyIds.size()); } @@ -1482,8 +1565,10 @@ public void testNimbusCheckAuthorizationParams() throws Exception { .withTopoCache(tc) .withNimbusWrapper(nimbus -> Mockito.spy(nimbus)) .withDaemonConf(Map.of( - DaemonConfig.NIMBUS_AUTHORIZER, "org.apache.storm.security.auth.authorizer.NoopAuthorizer", - DaemonConfig.SUPERVISOR_AUTHORIZER, "org.apache.storm.security.auth.authorizer.NoopAuthorizer")) + DaemonConfig.NIMBUS_AUTHORIZER, "org.apache.storm.security.auth.authorizer.Noo" + + "pAuthorizer", + DaemonConfig.SUPERVISOR_AUTHORIZER, "org.apache.storm.security.auth.authorizer" + + ".NoopAuthorizer")) .build()) { Nimbus nimbus = cluster.getNimbus(); String topologyName = "test-nimbus-check-autho-params"; @@ -1494,8 +1579,10 @@ public void testNimbusCheckAuthorizationParams() throws Exception { expectedConf.put("foo", "bar"); Mockito.when(clusterState.getTopoId(topologyName)).thenReturn(Optional.of(topologyId)); - Mockito.when(tc.readTopoConf(Mockito.any(String.class), ArgumentMatchers.any())).thenReturn(expectedConf); - Mockito.when(tc.readTopology(Mockito.any(String.class), ArgumentMatchers.any())).thenReturn(null); + Mockito.when(tc.readTopoConf(Mockito.any(String.class), ArgumentMatchers.any())) + .thenReturn(expectedConf); + Mockito.when(tc.readTopology(Mockito.any(String.class), ArgumentMatchers.any())) + .thenReturn(null); // getTopologyConf calls checkAuthorization with the correct parameters try { @@ -1506,23 +1593,28 @@ public void testNimbusCheckAuthorizationParams() throws Exception { } catch (NotAliveException e) { // acceptable } - Mockito.verify(nimbus).checkAuthorization(Mockito.isNull(), Mockito.isNull(), Mockito.eq("getClusterInfo")); - Mockito.verify(nimbus).checkAuthorization(Mockito.eq(topologyName), Mockito.any(Map.class), Mockito.eq("getTopologyConf")); + Mockito.verify(nimbus).checkAuthorization(Mockito.isNull(), Mockito.isNull(), Mockito + .eq("getClusterInfo")); + Mockito.verify(nimbus).checkAuthorization(Mockito.eq(topologyName), Mockito + .any(Map.class), Mockito.eq("getTopologyConf")); // getTopology calls checkAuthorization with the correct parameters StormCommon commonOverride = new StormCommon() { @Override - protected StormTopology systemTopologyImpl(Map topoConf, StormTopology topology) { + protected StormTopology systemTopologyImpl(Map topoConf, + StormTopology topology) { return null; } }; - try (org.apache.storm.utils.StormCommonInstaller ignored = new org.apache.storm.utils.StormCommonInstaller(commonOverride)) { + try (org.apache.storm.utils.StormCommonInstaller ignored = new org.apache.storm.utils + .StormCommonInstaller(commonOverride)) { try { nimbus.getTopology(topologyId); } catch (NotAliveException e) { // acceptable } - Mockito.verify(nimbus).checkAuthorization(Mockito.eq(topologyName), Mockito.any(Map.class), Mockito.eq("getTopology")); + Mockito.verify(nimbus).checkAuthorization(Mockito.eq(topologyName), Mockito + .any(Map.class), Mockito.eq("getTopology")); } // getUserTopology calls checkAuthorization with the correct parameters @@ -1531,8 +1623,10 @@ protected StormTopology systemTopologyImpl(Map topoConf, StormTo } catch (NotAliveException e) { // acceptable } - Mockito.verify(nimbus).checkAuthorization(Mockito.eq(topologyName), Mockito.any(Map.class), Mockito.eq("getUserTopology")); - Mockito.verify(tc, Mockito.times(2)).readTopology(Mockito.eq(topologyId), ArgumentMatchers.any()); + Mockito.verify(nimbus).checkAuthorization(Mockito.eq(topologyName), Mockito + .any(Map.class), Mockito.eq("getUserTopology")); + Mockito.verify(tc, Mockito.times(2)).readTopology(Mockito.eq(topologyId), + ArgumentMatchers.any()); } } @@ -1548,8 +1642,10 @@ public void testCheckAuthorizationGetSupervisorPageInfo() throws Exception { .withTopoCache(tc) .withNimbusWrapper(nimbus -> Mockito.spy(nimbus)) .withDaemonConf(Map.of( - DaemonConfig.NIMBUS_AUTHORIZER, "org.apache.storm.security.auth.authorizer.NoopAuthorizer", - DaemonConfig.SUPERVISOR_AUTHORIZER, "org.apache.storm.security.auth.authorizer.NoopAuthorizer")) + DaemonConfig.NIMBUS_AUTHORIZER, "org.apache.storm.security.auth.authorizer.Noo" + + "pAuthorizer", + DaemonConfig.SUPERVISOR_AUTHORIZER, "org.apache.storm.security.auth.authorizer" + + ".NoopAuthorizer")) .build()) { Nimbus nimbus = cluster.getNimbus(); String expectedName = "test-nimbus-check-autho-params"; @@ -1573,8 +1669,10 @@ public void testCheckAuthorizationGetSupervisorPageInfo() throws Exception { Map topoAssignment = new HashMap<>(); topoAssignment.put(expectedName, assignment); - HashMap allSupervisors = new HashMap<>(); - org.apache.storm.generated.SupervisorInfo si1 = new org.apache.storm.generated.SupervisorInfo(); + HashMap allSupervisors = + new HashMap<>(); + org.apache.storm.generated.SupervisorInfo si1 = new org.apache.storm.generated + .SupervisorInfo(); si1.set_hostname("host1"); si1.set_meta(List.of(1234L)); si1.set_uptime_secs(123); @@ -1582,7 +1680,8 @@ public void testCheckAuthorizationGetSupervisorPageInfo() throws Exception { si1.set_resources_map(Map.of()); allSupervisors.put("super1", si1); - org.apache.storm.generated.SupervisorInfo si2 = new org.apache.storm.generated.SupervisorInfo(); + org.apache.storm.generated.SupervisorInfo si2 = new org.apache.storm.generated + .SupervisorInfo(); si2.set_hostname("host2"); si2.set_meta(List.of(1234L)); si2.set_uptime_secs(123); @@ -1591,15 +1690,20 @@ public void testCheckAuthorizationGetSupervisorPageInfo() throws Exception { allSupervisors.put("super2", si2); Mockito.when(clusterState.allSupervisorInfo()).thenReturn(allSupervisors); - Mockito.when(tc.readTopoConf(Mockito.any(String.class), Mockito.any(Subject.class))).thenReturn(expectedConf); - Mockito.when(tc.readTopology(Mockito.any(String.class), Mockito.any(Subject.class))).thenReturn(topology); + Mockito.when(tc.readTopoConf(Mockito.any(String.class), Mockito.any(Subject.class))) + .thenReturn(expectedConf); + Mockito.when(tc.readTopology(Mockito.any(String.class), Mockito.any(Subject.class))) + .thenReturn(topology); Mockito.when(clusterState.assignmentsInfo()).thenReturn(topoAssignment); nimbus.getSupervisorPageInfo("super1", null, true); - Mockito.verify(nimbus).checkAuthorization(Mockito.eq(expectedName), Mockito.any(Map.class), Mockito.eq("getSupervisorPageInfo")); - Mockito.verify(nimbus).checkAuthorization(Mockito.isNull(), Mockito.isNull(), Mockito.eq("getClusterInfo")); - Mockito.verify(nimbus).checkAuthorization(Mockito.eq(expectedName), Mockito.any(Map.class), Mockito.eq("getTopology")); + Mockito.verify(nimbus).checkAuthorization(Mockito.eq(expectedName), Mockito + .any(Map.class), Mockito.eq("getSupervisorPageInfo")); + Mockito.verify(nimbus).checkAuthorization(Mockito.isNull(), Mockito.isNull(), Mockito + .eq("getClusterInfo")); + Mockito.verify(nimbus).checkAuthorization(Mockito.eq(expectedName), Mockito + .any(Map.class), Mockito.eq("getTopology")); } } @@ -1624,7 +1728,8 @@ public void testKillStorm() throws Exception { Map.of("1", Thrift.prepareSpoutDetails(new TestPlannerSpout(true), 14)), Map.of()); - cluster.submitTopology("test", Map.of(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, 20), topology); + cluster.submitTopology("test", Map.of(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, 20), + topology); String stormId = state.getTopoId("test").get(); cluster.advanceClusterTime(15); assertNotNull(state.stormBase(stormId, null)); @@ -1646,7 +1751,8 @@ public void testKillStorm() throws Exception { assertThrows(NotAliveException.class, () -> nimbus.killTopology("lalala")); - cluster.submitTopology("2test", Map.of(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, 10), topology); + cluster.submitTopology("2test", Map.of(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, 10), + topology); cluster.advanceClusterTime(11); assertThrows(org.apache.storm.generated.AlreadyAliveException.class, () -> cluster.submitTopology("2test", Map.of(), topology)); @@ -1667,7 +1773,8 @@ public void testKillStorm() throws Exception { cluster.advanceClusterTime(11); assertEquals(0, state.heartbeatStorms().size()); - cluster.submitTopology("test3", Map.of(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, 5), topology); + cluster.submitTopology("test3", Map.of(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, 5), + topology); String stormId3 = state.getTopoId("test3").get(); cluster.advanceClusterTime(11); nimbus.killTopology("test3"); @@ -1680,7 +1787,8 @@ public void testKillStorm() throws Exception { Time.advanceTimeSecs(11); cluster.waitForIdle(); - cluster.submitTopology("test3", Map.of(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, 5), topology); + cluster.submitTopology("test3", Map.of(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, 5), + topology); stormId3 = state.getTopoId("test3").get(); cluster.advanceClusterTime(11); @@ -1693,9 +1801,11 @@ public void testKillStorm() throws Exception { assertEquals(0, state.heartbeatStorms().size()); // test kill with opts - cluster.submitTopology("test4", Map.of(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, 100), topology); + cluster.submitTopology("test4", Map.of(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, 100), + topology); cluster.advanceClusterTime(11); - org.apache.storm.generated.KillOptions killOpts = new org.apache.storm.generated.KillOptions(); + org.apache.storm.generated.KillOptions killOpts = new org.apache.storm.generated + .KillOptions(); killOpts.set_wait_secs(10); nimbus.killTopologyWithOpts("test4", killOpts); String stormId4 = state.getTopoId("test4").get(); @@ -1768,7 +1878,8 @@ public void testReassignment() throws Exception { cluster.advanceClusterTime(31); // executor1 also timed out (launch timeout 60s) assertFalse(ass1.equals(executorAssignment(cluster, stormId, executorId1))); - assertEquals(ass2, executorAssignment(cluster, stormId, executorId2)); // tests launch timeout + assertEquals(ass2, executorAssignment(cluster, stormId, + executorId2)); // tests launch timeout checkConsistency(cluster, "test", true); ass1 = executorAssignment(cluster, stormId, executorId1); @@ -1790,8 +1901,10 @@ public void testReassignment() throws Exception { ass2 = executorAssignment(cluster, stormId, executorId2); assertNotNull(ass1); assertNotNull(ass2); - assertFalse(activeSupervisor.equals(executorAssignment(cluster, stormId, executorId2).get_node())); - assertFalse(activeSupervisor.equals(executorAssignment(cluster, stormId, executorId1).get_node())); + assertFalse(activeSupervisor.equals(executorAssignment(cluster, stormId, executorId2) + .get_node())); + assertFalse(activeSupervisor.equals(executorAssignment(cluster, stormId, executorId1) + .get_node())); checkConsistency(cluster, "test", true); // Kill all supervisors @@ -2044,7 +2157,8 @@ public void testRebalanceChangeParallelism() throws Exception { checkExecutorDistribution(slotAssignments(cluster, stormId), List.of(2, 2, 2, 2)); checkConsistency(cluster, "test", true); - Map>> executorInfo = stormComponentToExecutorInfo(cluster, "test"); + Map>> executorInfo = stormComponentToExecutorInfo(cluster, + "test"); List> exec1Tasks = new ArrayList<>(); for (List e : executorInfo.get("1")) { exec1Tasks.add(executorToTasks(e)); @@ -2099,7 +2213,8 @@ public void testRebalanceConstrainedCluster() throws Exception { } } - // --- Tests ported using real InProcessZookeeper (avoids MockedZookeeper static mocking crashes) --- + // --- Tests ported using real InProcessZookeeper (avoids MockedZookeeper static mocking + // crashes) --- /** * Port of test-leadership from nimbus_test.clj. @@ -2263,10 +2378,12 @@ public void testTopologyActionNotifier() throws Exception { nimbus.shutdown(); - // InMemoryTopologyActionNotifier uses a static map, so a new instance sees the same data + // InMemoryTopologyActionNotifier uses a static map, so a new instance sees the same + // data InMemoryTopologyActionNotifier notifier = new InMemoryTopologyActionNotifier(); assertEquals( - Arrays.asList("submitTopology", "activate", "deactivate", "activate", "rebalance", "killTopology"), + Arrays.asList("submitTopology", "activate", "deactivate", "activate", "rebalance", + "killTopology"), notifier.getTopologyActions("test-notification")); } } diff --git a/storm-server/src/test/java/org/apache/storm/daemon/nimbus/NimbusGetNimbusConfTest.java b/storm-server/src/test/java/org/apache/storm/daemon/nimbus/NimbusGetNimbusConfTest.java index c348697c8de..535ca6dbb1e 100644 --- a/storm-server/src/test/java/org/apache/storm/daemon/nimbus/NimbusGetNimbusConfTest.java +++ b/storm-server/src/test/java/org/apache/storm/daemon/nimbus/NimbusGetNimbusConfTest.java @@ -18,17 +18,16 @@ package org.apache.storm.daemon.nimbus; +import static org.junit.jupiter.api.Assertions.assertEquals; + import java.util.HashMap; import java.util.Map; - import net.minidev.json.JSONValue; import org.apache.storm.Config; import org.apache.storm.DaemonConfig; import org.apache.storm.LocalCluster; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; - public class NimbusGetNimbusConfTest { private static final String MASKED = "*****"; @@ -41,13 +40,16 @@ private static Map parse(String json) { @Test public void getNimbusConfMasksCredentialsAndKeepsOtherValues() throws Exception { Map daemonConf = new HashMap<>(); - daemonConf.put(DaemonConfig.NIMBUS_AUTHORIZER, "org.apache.storm.security.auth.authorizer.NoopAuthorizer"); - daemonConf.put(DaemonConfig.SUPERVISOR_AUTHORIZER, "org.apache.storm.security.auth.authorizer.NoopAuthorizer"); + daemonConf.put(DaemonConfig.NIMBUS_AUTHORIZER, + "org.apache.storm.security.auth.authorizer.NoopAuthorizer"); + daemonConf.put(DaemonConfig.SUPERVISOR_AUTHORIZER, + "org.apache.storm.security.auth.authorizer.NoopAuthorizer"); daemonConf.put(Config.STORM_ZOOKEEPER_AUTH_PAYLOAD, "zk-user:zk-secret"); daemonConf.put(Config.NIMBUS_THRIFT_TLS_SERVER_KEYSTORE_PASSWORD, "nimbus-keystore-secret"); daemonConf.put(DaemonConfig.UI_HTTPS_KEYSTORE_PASSWORD, "ui-keystore-secret"); daemonConf.put(Config.STORM_ZOOKEEPER_SSL_KEYSTORE_PASSWORD, "zk-ssl-keystore-secret"); - daemonConf.put("storm.daemon.metrics.reporter.plugin.prometheus.basic_auth_password", "plugin-secret"); + daemonConf.put("storm.daemon.metrics.reporter.plugin.prometheus.basic_auth_password", + "plugin-secret"); daemonConf.put(Config.NIMBUS_THRIFT_TLS_SERVER_KEYSTORE_PATH, "/etc/storm/nimbus.jks"); try (LocalCluster cluster = new LocalCluster.Builder().withDaemonConf(daemonConf).build()) { @@ -62,10 +64,12 @@ public void getNimbusConfMasksCredentialsAndKeepsOtherValues() throws Exception assertEquals(MASKED, served.get(Config.STORM_ZOOKEEPER_SSL_KEYSTORE_PASSWORD), "ZooKeeper TLS store passwords should be masked"); - assertEquals(MASKED, served.get("storm.daemon.metrics.reporter.plugin.prometheus.basic_auth_password"), + assertEquals(MASKED, served + .get("storm.daemon.metrics.reporter.plugin.prometheus.basic_auth_password"), "credential keys that only a plugin declares should be masked too"); - assertEquals("/etc/storm/nimbus.jks", served.get(Config.NIMBUS_THRIFT_TLS_SERVER_KEYSTORE_PATH), + assertEquals("/etc/storm/nimbus.jks", served + .get(Config.NIMBUS_THRIFT_TLS_SERVER_KEYSTORE_PATH), "non-credential values should be served unchanged"); } } diff --git a/storm-server/src/test/java/org/apache/storm/daemon/nimbus/NimbusGetTopologyConfTest.java b/storm-server/src/test/java/org/apache/storm/daemon/nimbus/NimbusGetTopologyConfTest.java index fae4ad566b3..fe2c5e02588 100644 --- a/storm-server/src/test/java/org/apache/storm/daemon/nimbus/NimbusGetTopologyConfTest.java +++ b/storm-server/src/test/java/org/apache/storm/daemon/nimbus/NimbusGetTopologyConfTest.java @@ -18,24 +18,22 @@ package org.apache.storm.daemon.nimbus; +import static org.junit.jupiter.api.Assertions.assertEquals; + import java.util.HashMap; import java.util.Map; import java.util.Optional; - +import net.minidev.json.JSONValue; import org.apache.storm.Config; import org.apache.storm.DaemonConfig; import org.apache.storm.LocalCluster; import org.apache.storm.blobstore.BlobStore; import org.apache.storm.cluster.IStormClusterState; -import net.minidev.json.JSONValue; - import org.apache.storm.security.serialization.BlowfishTupleSerializer; import org.junit.jupiter.api.Test; import org.mockito.ArgumentMatchers; import org.mockito.Mockito; -import static org.junit.jupiter.api.Assertions.assertEquals; - public class NimbusGetTopologyConfTest { private static final String MASKED = "*****"; @@ -64,8 +62,10 @@ public void getTopologyConfMasksCredentialsAndKeepsOtherValues() throws Exceptio .withBlobStore(blobStore) .withTopoCache(topoCache) .withDaemonConf(Map.of( - DaemonConfig.NIMBUS_AUTHORIZER, "org.apache.storm.security.auth.authorizer.NoopAuthorizer", - DaemonConfig.SUPERVISOR_AUTHORIZER, "org.apache.storm.security.auth.authorizer.NoopAuthorizer")) + DaemonConfig.NIMBUS_AUTHORIZER, "org.apache.storm.security.auth.authorizer.Noo" + + "pAuthorizer", + DaemonConfig.SUPERVISOR_AUTHORIZER, "org.apache.storm.security.auth.authorizer" + + ".NoopAuthorizer")) .build()) { Nimbus nimbus = cluster.getNimbus(); @@ -83,7 +83,8 @@ public void getTopologyConfMasksCredentialsAndKeepsOtherValues() throws Exceptio assertEquals(TOPO_NAME, served.get(Config.TOPOLOGY_NAME)); assertEquals(3, ((Number) served.get(Config.TOPOLOGY_WORKERS)).intValue()); - assertEquals("topology-zk-secret", storedConf.get(Config.STORM_ZOOKEEPER_TOPOLOGY_AUTH_PAYLOAD), + assertEquals("topology-zk-secret", storedConf + .get(Config.STORM_ZOOKEEPER_TOPOLOGY_AUTH_PAYLOAD), "the stored conf should keep its own values"); assertEquals("0123456789abcdef", storedConf.get(BlowfishTupleSerializer.SECRET_KEY), "the stored conf should keep its own values"); @@ -105,8 +106,10 @@ public void getTopologyConfLeavesCredentialFreeConfAlone() throws Exception { .withBlobStore(blobStore) .withTopoCache(topoCache) .withDaemonConf(Map.of( - DaemonConfig.NIMBUS_AUTHORIZER, "org.apache.storm.security.auth.authorizer.NoopAuthorizer", - DaemonConfig.SUPERVISOR_AUTHORIZER, "org.apache.storm.security.auth.authorizer.NoopAuthorizer")) + DaemonConfig.NIMBUS_AUTHORIZER, "org.apache.storm.security.auth.authorizer.Noo" + + "pAuthorizer", + DaemonConfig.SUPERVISOR_AUTHORIZER, "org.apache.storm.security.auth.authorizer" + + ".NoopAuthorizer")) .build()) { Nimbus nimbus = cluster.getNimbus(); diff --git a/storm-server/src/test/java/org/apache/storm/daemon/nimbus/NimbusGetTopologyPageInfoTest.java b/storm-server/src/test/java/org/apache/storm/daemon/nimbus/NimbusGetTopologyPageInfoTest.java index c9bf4d2c5e4..1157de36df7 100644 --- a/storm-server/src/test/java/org/apache/storm/daemon/nimbus/NimbusGetTopologyPageInfoTest.java +++ b/storm-server/src/test/java/org/apache/storm/daemon/nimbus/NimbusGetTopologyPageInfoTest.java @@ -18,11 +18,11 @@ package org.apache.storm.daemon.nimbus; +import static org.junit.jupiter.api.Assertions.assertEquals; + import java.util.HashMap; import java.util.Map; - import net.minidev.json.JSONValue; - import org.apache.storm.Config; import org.apache.storm.DaemonConfig; import org.apache.storm.LocalCluster; @@ -39,11 +39,11 @@ import org.mockito.ArgumentMatchers; import org.mockito.Mockito; -import static org.junit.jupiter.api.Assertions.assertEquals; - /** - * getTopologyPageInfo serves the daemon configuration merged with the topology configuration, and it is a - * topology read-only operation, so a principal that is only allowed to look at a topology reaches it. The + * getTopologyPageInfo serves the daemon configuration merged with the topology configuration, and + * it is a + * topology read-only operation, so a principal that is only allowed to look at a topology reaches + * it. The * merged map must therefore not carry credential values off the daemon. */ public class NimbusGetTopologyPageInfoTest { @@ -53,7 +53,8 @@ public class NimbusGetTopologyPageInfoTest { private static final String TOPO_ID = "fake-id"; private static final String PLUGIN_SECRET_KEY = "some.plugin.password"; - private static final String NIMBUS_KEYSTORE_PASSWORD_KEY = "nimbus.thrift.tls.keystore.password"; + private static final String NIMBUS_KEYSTORE_PASSWORD_KEY = + "nimbus.thrift.tls.keystore.password"; @SuppressWarnings("unchecked") private static Map parse(String json) { @@ -99,8 +100,10 @@ public void getTopologyPageInfoMasksDaemonAndTopologyCredentials() throws Except Map storedConf = storedTopoConf(); Map daemonConf = new HashMap<>(); - daemonConf.put(DaemonConfig.NIMBUS_AUTHORIZER, "org.apache.storm.security.auth.authorizer.NoopAuthorizer"); - daemonConf.put(DaemonConfig.SUPERVISOR_AUTHORIZER, "org.apache.storm.security.auth.authorizer.NoopAuthorizer"); + daemonConf.put(DaemonConfig.NIMBUS_AUTHORIZER, + "org.apache.storm.security.auth.authorizer.NoopAuthorizer"); + daemonConf.put(DaemonConfig.SUPERVISOR_AUTHORIZER, + "org.apache.storm.security.auth.authorizer.NoopAuthorizer"); // the daemon-side values that the merge pulls in on top of the topology's own conf daemonConf.put(Config.STORM_ZOOKEEPER_AUTH_PAYLOAD, "cluster-zk-digest-secret"); daemonConf.put(NIMBUS_KEYSTORE_PASSWORD_KEY, "keystore-secret"); @@ -143,12 +146,15 @@ public void getTopologyPageInfoMasksDaemonAndTopologyCredentials() throws Except // values that carry no credential are served untouched assertEquals(TOPO_NAME, served.get(Config.TOPOLOGY_NAME)); assertEquals(1, ((Number) served.get(Config.TOPOLOGY_WORKERS)).intValue()); - assertEquals(30, ((Number) served.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS)).intValue()); + assertEquals(30, ((Number) served.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS)) + .intValue()); // masking is applied to the served copy, never to the daemon's own configuration - assertEquals("cluster-zk-digest-secret", nimbus.getConf().get(Config.STORM_ZOOKEEPER_AUTH_PAYLOAD), + assertEquals("cluster-zk-digest-secret", nimbus.getConf() + .get(Config.STORM_ZOOKEEPER_AUTH_PAYLOAD), "the daemon conf should keep its own values"); - assertEquals("topology-zk-secret", storedConf.get(Config.STORM_ZOOKEEPER_TOPOLOGY_AUTH_PAYLOAD), + assertEquals("topology-zk-secret", storedConf + .get(Config.STORM_ZOOKEEPER_TOPOLOGY_AUTH_PAYLOAD), "the stored topology conf should keep its own values"); } } diff --git a/storm-server/src/test/java/org/apache/storm/daemon/nimbus/NimbusTest.java b/storm-server/src/test/java/org/apache/storm/daemon/nimbus/NimbusTest.java index 0b5215c6f14..b889a4a9871 100644 --- a/storm-server/src/test/java/org/apache/storm/daemon/nimbus/NimbusTest.java +++ b/storm-server/src/test/java/org/apache/storm/daemon/nimbus/NimbusTest.java @@ -18,6 +18,28 @@ package org.apache.storm.daemon.nimbus; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockConstruction; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.codahale.metrics.Meter; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -31,8 +53,6 @@ import java.util.Optional; import java.util.Set; import javax.security.auth.Subject; - -import com.codahale.metrics.Meter; import net.minidev.json.JSONValue; import org.apache.commons.io.FileUtils; import org.apache.storm.Config; @@ -49,8 +69,8 @@ import org.apache.storm.generated.InvalidTopologyException; import org.apache.storm.generated.KeyNotFoundException; import org.apache.storm.generated.ListBlobsResult; -import org.apache.storm.generated.RebalanceOptions; import org.apache.storm.generated.ReadableBlobMeta; +import org.apache.storm.generated.RebalanceOptions; import org.apache.storm.generated.SettableBlobMeta; import org.apache.storm.generated.StormTopology; import org.apache.storm.generated.SubmitOptions; @@ -85,34 +105,14 @@ import org.mockito.MockedConstruction; import org.mockito.MockitoAnnotations; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; -import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertSame; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyBoolean; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.mockConstruction; -import static org.mockito.Mockito.atLeastOnce; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - class NimbusTest { private static final String BLOB_FILE_KEY = "file-key"; private static final String TOPO_NAME = "topo"; private static final String TOPO_ID = "topology1-1-1"; // an artifact blob key from before the key carried a uuid, which several topologies can share private static final String LEGACY_ARTIFACT_KEY = "dep-group-artifact-1.0.0.jar"; - // a dependency key written by a current client, which splices a generated uuid into the file name + // a dependency key written by a current client, which splices a generated uuid into the file + // name private static final String UNIQUE_JAR_KEY = "dep-lib-11111111-1111-1111-1111-111111111111.jar"; @Mock @@ -139,7 +139,8 @@ public void setUp() throws Exception { MockitoAnnotations.openMocks(this).close(); Map conf = Map.of(DaemonConfig.NIMBUS_MONITOR_FREQ_SECS, 10); - nimbus = new Nimbus(conf, iNimbus, stormClusterState, nimbusInfo, localBlobStore, leaderElector, groupMapper, metricRegistry); + nimbus = new Nimbus(conf, iNimbus, stormClusterState, nimbusInfo, localBlobStore, + leaderElector, groupMapper, metricRegistry); } @AfterEach @@ -149,14 +150,17 @@ public void tearDown() { @Test public void testMemoryLoadLargerThanMaxHeapSize() { - // Topology will not be able to be successfully scheduled: Config TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB=128.0 < 129.0, + // Topology will not be able to be successfully scheduled: Config + // TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB=128.0 < 129.0, // Largest memory requirement of a component in the topology). TopologyBuilder builder1 = new TopologyBuilder(); builder1.setSpout("wordSpout1", new TestWordSpout(), 4); StormTopology stormTopology1 = builder1.createTopology(); Config config1 = new Config(); - config1.put(Config.STORM_NETWORK_TOPOGRAPHY_PLUGIN, "org.apache.storm.networktopography.DefaultRackDNSToSwitchMapping"); - config1.put(DaemonConfig.RESOURCE_AWARE_SCHEDULER_PRIORITY_STRATEGY, DefaultSchedulingPriorityStrategy.class.getName()); + config1.put(Config.STORM_NETWORK_TOPOGRAPHY_PLUGIN, + "org.apache.storm.networktopography.DefaultRackDNSToSwitchMapping"); + config1.put(DaemonConfig.RESOURCE_AWARE_SCHEDULER_PRIORITY_STRATEGY, + DefaultSchedulingPriorityStrategy.class.getName()); config1.put(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT, 10.0); config1.put(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB, 0.0); @@ -165,17 +169,18 @@ public void testMemoryLoadLargerThanMaxHeapSize() { config1.put(Config.TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB, 128.0); config1.put(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB, 129.0); Class[] strategyClasses = { - DefaultResourceAwareStrategy.class, - RoundRobinResourceAwareStrategy.class, - GenericResourceAwareStrategyOld.class}; - for (Class strategyClass: strategyClasses) { + DefaultResourceAwareStrategy.class, + RoundRobinResourceAwareStrategy.class, + GenericResourceAwareStrategyOld.class}; + for (Class strategyClass : strategyClasses) { String strategyClassName = strategyClass.getName(); config1.put(Config.TOPOLOGY_SCHEDULER_STRATEGY, strategyClassName); try { - ServerUtils.validateTopologyWorkerMaxHeapSizeConfigs(config1, stormTopology1, 768.0); + ServerUtils.validateTopologyWorkerMaxHeapSizeConfigs(config1, stormTopology1, + 768.0); fail("Expected exception not thrown when using Strategy " + strategyClassName); } catch (InvalidTopologyException e) { - //Expected... + // Expected... } } } @@ -230,7 +235,8 @@ void testCreateStateInZookeeperIsNotAllowedWhenTheAuthorizerDeniesIt() throws Ex when(authorizer.permit(any(), eq("createStateInZookeeper"), any())).thenReturn(false); nimbus.setAuthorizationHandler(authorizer); - assertThrows(AuthorizationException.class, () -> nimbus.createStateInZookeeper(BLOB_FILE_KEY)); + assertThrows(AuthorizationException.class, () -> nimbus + .createStateInZookeeper(BLOB_FILE_KEY)); verify(stormClusterState, never()).setupBlob(eq(BLOB_FILE_KEY), eq(nimbusInfo), any()); } @@ -249,7 +255,8 @@ void testCreateStateInZookeeperIsAllowedWhenTheAuthorizerPermitsIt() throws Exce void testCreateStateInZookeeperWithoutLocalFsBlobStoreInstanceShouldNotCreate() throws Exception { BlobStore blobStore = mock(BlobStore.class); Map conf = Map.of(DaemonConfig.NIMBUS_MONITOR_FREQ_SECS, 10); - nimbus = new Nimbus(conf, iNimbus, stormClusterState, nimbusInfo, blobStore, leaderElector, groupMapper, metricRegistry); + nimbus = new Nimbus(conf, iNimbus, stormClusterState, nimbusInfo, blobStore, leaderElector, + groupMapper, metricRegistry); nimbus.createStateInZookeeper(BLOB_FILE_KEY); @@ -258,7 +265,8 @@ void testCreateStateInZookeeperWithoutLocalFsBlobStoreInstanceShouldNotCreate() @Test void testCreateStateInZookeeperWhenFailToSetupBlobWithRuntimeExceptionThrowsRuntimeException() { - doThrow(new RuntimeException("Failed to setup blob")).when(stormClusterState).setupBlob(eq(BLOB_FILE_KEY), eq(nimbusInfo), any()); + doThrow(new RuntimeException("Failed to setup blob")).when(stormClusterState) + .setupBlob(eq(BLOB_FILE_KEY), eq(nimbusInfo), any()); assertThrows(RuntimeException.class, () -> nimbus.createStateInZookeeper(BLOB_FILE_KEY)); verify(stormClusterState).setupBlob(eq(BLOB_FILE_KEY), eq(nimbusInfo), any()); @@ -266,24 +274,29 @@ void testCreateStateInZookeeperWhenFailToSetupBlobWithRuntimeExceptionThrowsRunt @Test void testCreateStateInZookeeperWhenKeyNotFoundHandlesException() throws Exception { - try (MockedConstruction keySequenceNumber = mockConstruction(KeySequenceNumber.class, (mock, context) -> - when(mock.getKeySequenceNumber(any(), anyBoolean())).thenThrow(new KeyNotFoundException("Failed to setup blob")))) { + try (MockedConstruction keySequenceNumber = + mockConstruction(KeySequenceNumber.class, (mock, context) -> + when(mock.getKeySequenceNumber(any(), anyBoolean())) + .thenThrow(new KeyNotFoundException("Failed to setup blob")))) { nimbus.createStateInZookeeper(BLOB_FILE_KEY); - verify(keySequenceNumber.constructed().get(0)).getKeySequenceNumber(any(), anyBoolean()); + verify(keySequenceNumber.constructed().get(0)).getKeySequenceNumber(any(), + anyBoolean()); verify(stormClusterState, never()).setupBlob(eq(BLOB_FILE_KEY), eq(nimbusInfo), any()); } } @Test void testCreateStateInZookeeperOnlyLetsTheLeaderRegisterAKeyZookeeperDoesNotKnow() throws Exception { - try (MockedConstruction keySequenceNumber = mockConstruction(KeySequenceNumber.class)) { + try (MockedConstruction keySequenceNumber = + mockConstruction(KeySequenceNumber.class)) { when(leaderElector.isLeader()).thenReturn(false); nimbus.createStateInZookeeper(BLOB_FILE_KEY); when(leaderElector.isLeader()).thenReturn(true); nimbus.createStateInZookeeper(BLOB_FILE_KEY); - // a non-leader registering a key zookeeper does not know would bring back a key that was deleted + // a non-leader registering a key zookeeper does not know would bring back a key that + // was deleted verify(keySequenceNumber.constructed().get(0)).getKeySequenceNumber(any(), eq(false)); verify(keySequenceNumber.constructed().get(1)).getKeySequenceNumber(any(), eq(true)); } @@ -291,7 +304,8 @@ void testCreateStateInZookeeperOnlyLetsTheLeaderRegisterAKeyZookeeperDoesNotKnow @Test void testListBlobsOnlyReturnsKeysTheCallerMayReadTheMetadataOf() throws Exception { - when(localBlobStore.listKeys()).thenReturn(List.of("readable-key", "other-users-key").iterator()); + when(localBlobStore.listKeys()).thenReturn(List.of("readable-key", "other-users-key") + .iterator()); when(localBlobStore.getBlobMeta(eq("other-users-key"), any())) .thenThrow(new WrappedAuthorizationException("not allowed")); @@ -303,8 +317,10 @@ void testListBlobsOnlyReturnsKeysTheCallerMayReadTheMetadataOf() throws Exceptio @Test void testListBlobsIsAuthorized() throws Exception { Map conf = Map.of(DaemonConfig.NIMBUS_MONITOR_FREQ_SECS, 10, - DaemonConfig.NIMBUS_AUTHORIZER, DenyAuthorizer.class.getName()); - nimbus = new Nimbus(conf, iNimbus, stormClusterState, nimbusInfo, localBlobStore, leaderElector, groupMapper, metricRegistry); + DaemonConfig.NIMBUS_AUTHORIZER, DenyAuthorizer.class + .getName()); + nimbus = new Nimbus(conf, iNimbus, stormClusterState, nimbusInfo, localBlobStore, + leaderElector, groupMapper, metricRegistry); when(localBlobStore.listKeys()).thenReturn(List.of("readable-key").iterator()); assertThrows(AuthorizationException.class, () -> nimbus.listBlobs("")); @@ -320,7 +336,8 @@ void testUploadNewCredentialsRejectsACallerWhoIsNotTheOwner() throws Exception { Credentials creds = new Credentials(Map.of("key", "value")); creds.set_topoOwner("alice"); - assertThrows(AuthorizationException.class, () -> nimbus.uploadNewCredentials(TOPO_NAME, creds)); + assertThrows(AuthorizationException.class, () -> nimbus.uploadNewCredentials(TOPO_NAME, + creds)); verify(stormClusterState, never()).setCredentials(eq(TOPO_ID), any(), any()); } @@ -345,11 +362,13 @@ void testUploadNewCredentialsRejectsAnOwnerMismatchClaimedByTheOwner() throws Ex Credentials creds = new Credentials(Map.of("key", "value")); creds.set_topoOwner("bob"); - assertThrows(AuthorizationException.class, () -> nimbus.uploadNewCredentials(TOPO_NAME, creds)); + assertThrows(AuthorizationException.class, () -> nimbus.uploadNewCredentials(TOPO_NAME, + creds)); verify(stormClusterState, never()).setCredentials(eq(TOPO_ID), any(), any()); } - private Nimbus makeNimbusOwningTopology(String topoName, String topoId, String owner) throws Exception { + private Nimbus makeNimbusOwningTopology(String topoName, String topoId, + String owner) throws Exception { Map topoConf = new HashMap<>(); topoConf.put(Config.TOPOLOGY_SUBMITTER_PRINCIPAL, owner); topoConf.put(Config.TOPOLOGY_SUBMITTER_USER, owner); @@ -359,7 +378,8 @@ private Nimbus makeNimbusOwningTopology(String topoName, String topoId, String o Map conf = Map.of(DaemonConfig.NIMBUS_MONITOR_FREQ_SECS, 10, Config.STORM_PRINCIPAL_TO_LOCAL_PLUGIN, DefaultPrincipalToLocal.class.getName()); - return new Nimbus(conf, iNimbus, stormClusterState, nimbusInfo, localBlobStore, topoCache, leaderElector, groupMapper, + return new Nimbus(conf, iNimbus, stormClusterState, nimbusInfo, localBlobStore, topoCache, + leaderElector, groupMapper, metricRegistry); } @@ -369,19 +389,24 @@ void testValidateUploadedJarLocationRejectsLocationsOutsideTheInbox() throws Exc Path sibling = Paths.get(inbox + "evil"); try { Path jar = Files.write(inbox.resolve("stormjar-cafebabe.jar"), new byte[]{ 1 }); - Path outside = Files.write(Files.createDirectory(sibling).resolve("stormjar-cafebabe.jar"), new byte[]{ 1 }); + Path outside = Files.write(Files.createDirectory(sibling) + .resolve("stormjar-cafebabe.jar"), new byte[]{ 1 }); - // a location handed out by beginFileUpload is accepted, and so is one that only walks inside the inbox + // a location handed out by beginFileUpload is accepted, and so is one that only walks + // inside the inbox Nimbus.validateUploadedJarLocation(inbox.toString(), jar.toString()); Files.createDirectory(inbox.resolve("nested")); - Nimbus.validateUploadedJarLocation(inbox.toString(), inbox + "/nested/../stormjar-cafebabe.jar"); + Nimbus.validateUploadedJarLocation(inbox.toString(), inbox + + "/nested/../stormjar-cafebabe.jar"); - // an absolute path elsewhere, a ".." walk out of the inbox, the inbox itself and a sibling directory + // an absolute path elsewhere, a ".." walk out of the inbox, the inbox itself and a + // sibling directory // whose name merely starts with the inbox path are all rejected assertThrows(AuthorizationException.class, () -> Nimbus.validateUploadedJarLocation(inbox.toString(), "/etc/passwd")); assertThrows(AuthorizationException.class, - () -> Nimbus.validateUploadedJarLocation(inbox.toString(), inbox + "/../../etc/passwd")); + () -> Nimbus.validateUploadedJarLocation(inbox.toString(), inbox + + "/../../etc/passwd")); assertThrows(AuthorizationException.class, () -> Nimbus.validateUploadedJarLocation(inbox.toString(), inbox.toString())); assertThrows(AuthorizationException.class, @@ -405,7 +430,8 @@ void testRebalanceRejectsConfOverridesWithBlobsTheCallerCannotRead() throws Exce TopoCache topoCache = mock(TopoCache.class); Map conf = Map.of(DaemonConfig.NIMBUS_MONITOR_FREQ_SECS, 10); - nimbus = new Nimbus(conf, iNimbus, stormClusterState, nimbusInfo, localBlobStore, topoCache, leaderElector, groupMapper, + nimbus = new Nimbus(conf, iNimbus, stormClusterState, nimbusInfo, localBlobStore, topoCache, + leaderElector, groupMapper, new StormMetricsRegistry()); StormTopology topology = new StormTopology(); @@ -413,22 +439,25 @@ void testRebalanceRejectsConfOverridesWithBlobsTheCallerCannotRead() throws Exce topology.set_bolts(new HashMap<>()); topology.set_state_spouts(new HashMap<>()); when(stormClusterState.getTopoId(topoName)).thenReturn(Optional.of(topoId)); - when(topoCache.readTopoConf(eq(topoId), any())).thenReturn(new HashMap<>(Map.of(Config.TOPOLOGY_NAME, topoName))); + when(topoCache.readTopoConf(eq(topoId), any())).thenReturn(new HashMap<>(Map + .of(Config.TOPOLOGY_NAME, topoName))); when(topoCache.readTopology(eq(topoId), any())).thenReturn(topology); doThrow(new AuthorizationException("does not have READ access to " + blobKey)) .when(localBlobStore).getBlobMeta(eq(blobKey), any()); RebalanceOptions options = new RebalanceOptions(); options.set_topology_conf_overrides( - JSONValue.toJSONString(Map.of(Config.TOPOLOGY_BLOBSTORE_MAP, Map.of(blobKey, new HashMap<>())))); + JSONValue.toJSONString(Map.of(Config.TOPOLOGY_BLOBSTORE_MAP, Map.of(blobKey, + new HashMap<>())))); - Subject caller = new Subject(false, Set.of(new SingleUserPrincipal("alice")), Set.of(), Set.of()); + Subject caller = new Subject(false, Set.of(new SingleUserPrincipal("alice")), Set.of(), Set + .of()); ReqContext.context().setSubject(caller); try { ArgumentCaptor subjectCaptor = ArgumentCaptor.forClass(Subject.class); assertThrows(AuthorizationException.class, () -> nimbus.rebalance(topoName, options)); verify(localBlobStore).getBlobMeta(eq(blobKey), subjectCaptor.capture()); - //the blobs are looked up as the one asking for the rebalance, not as nimbus + // the blobs are looked up as the one asking for the rebalance, not as nimbus assertSame(caller, subjectCaptor.getValue()); } finally { ReqContext.reset(); @@ -441,7 +470,8 @@ void testGetTopologyHistoryFiltersByTheAuthenticatedCaller() throws Exception { conf.put(DaemonConfig.NIMBUS_MONITOR_FREQ_SECS, 10); conf.put(Config.STORM_PRINCIPAL_TO_LOCAL_PLUGIN, DefaultPrincipalToLocal.class.getName()); conf.put(Config.NIMBUS_ADMINS, Collections.singletonList("admin")); - nimbus = new Nimbus(conf, iNimbus, stormClusterState, nimbusInfo, localBlobStore, leaderElector, groupMapper, metricRegistry); + nimbus = new Nimbus(conf, iNimbus, stormClusterState, nimbusInfo, localBlobStore, + leaderElector, groupMapper, metricRegistry); Map topoConf = new HashMap<>(); topoConf.put(Config.TOPOLOGY_NAME, "topology1"); @@ -454,18 +484,22 @@ void testGetTopologyHistoryFiltersByTheAuthenticatedCaller() throws Exception { try { setCaller("bob"); - // asking for somebody else's history is only for admins, the ui daemon is expected to be one. - // a caller that is not an admin gets its own history back rather than an error, so a ui that + // asking for somebody else's history is only for admins, the ui daemon is expected to + // be one. + // a caller that is not an admin gets its own history back rather than an error, so a ui + // that // was left out of nimbus.admins keeps serving the page instead of failing it assertTrue(nimbus.getTopologyHistory("alice").get_topo_ids().isEmpty()); // and no user argument at all is the caller's own history, not everybody's assertTrue(nimbus.getTopologyHistory(null).get_topo_ids().isEmpty()); setCaller("alice"); - assertEquals(Collections.singletonList(TOPO_ID), nimbus.getTopologyHistory(null).get_topo_ids()); + assertEquals(Collections.singletonList(TOPO_ID), nimbus.getTopologyHistory(null) + .get_topo_ids()); setCaller("admin"); - assertEquals(Collections.singletonList(TOPO_ID), nimbus.getTopologyHistory("alice").get_topo_ids()); + assertEquals(Collections.singletonList(TOPO_ID), nimbus.getTopologyHistory("alice") + .get_topo_ids()); assertTrue(nimbus.getTopologyHistory("bob").get_topo_ids().isEmpty()); } finally { ReqContext.reset(); @@ -475,7 +509,8 @@ void testGetTopologyHistoryFiltersByTheAuthenticatedCaller() throws Exception { /** * Register a topology in the blob store mock so that Nimbus can read its dependency lists back. */ - private static void storeTopology(BlobStore store, String topoId, List jars, List artifacts) + private static void storeTopology(BlobStore store, String topoId, List jars, + List artifacts) throws Exception { StormTopology topo = new StormTopology(); topo.set_spouts(new HashMap<>()); @@ -495,7 +530,8 @@ private Nimbus cleanupNimbus(BlobStore store, IStormClusterState state) throws E conf.put(DaemonConfig.NIMBUS_TOPOLOGY_BLOBSTORE_DELETION_DELAY_MS, 0); conf.put(Config.NIMBUS_THRIFT_TLS_PORT, 0); when(leaderElector.isLeader()).thenReturn(true); - return new Nimbus(conf, iNimbus, state, nimbusInfo, store, leaderElector, groupMapper, new StormMetricsRegistry()); + return new Nimbus(conf, iNimbus, state, nimbusInfo, store, leaderElector, groupMapper, + new StormMetricsRegistry()); } @Test @@ -504,13 +540,16 @@ void doCleanupRemovesBothDependencyJarsAndArtifactsOfADeadTopology() throws Exce IStormClusterState state = mock(IStormClusterState.class); when(store.storedTopoIds()).thenReturn(Set.of("dead-topo")); when(state.activeStorms()).thenReturn(List.of()); - storeTopology(store, "dead-topo", List.of("dep-lib-11111111-1111-1111-1111-111111111111.jar"), + storeTopology(store, "dead-topo", List + .of("dep-lib-11111111-1111-1111-1111-111111111111.jar"), List.of("dep-group-artifact-1.0.0-22222222-2222-2222-2222-222222222222.jar")); cleanupNimbus(store, state).doCleanup(); verify(store).deleteBlob(eq("dep-lib-11111111-1111-1111-1111-111111111111.jar"), any()); - verify(store).deleteBlob(eq("dep-group-artifact-1.0.0-22222222-2222-2222-2222-222222222222.jar"), any()); + verify(store) + .deleteBlob(eq("dep-group-artifact-1.0.0-22222222-2222-2222-2222-222222222222.jar"), + any()); } @Test @@ -520,9 +559,11 @@ void doCleanupKeepsADependencyBlobThatAnotherLiveTopologyStillReferences() throw when(store.storedTopoIds()).thenReturn(Set.of("dead-topo", "live-topo")); when(state.activeStorms()).thenReturn(List.of("live-topo")); // both were submitted before artifact keys carried a uuid, so they share one artifact blob - storeTopology(store, "dead-topo", List.of("dep-lib-11111111-1111-1111-1111-111111111111.jar"), + storeTopology(store, "dead-topo", List + .of("dep-lib-11111111-1111-1111-1111-111111111111.jar"), List.of(LEGACY_ARTIFACT_KEY)); - storeTopology(store, "live-topo", List.of("dep-lib-33333333-3333-3333-3333-333333333333.jar"), + storeTopology(store, "live-topo", List + .of("dep-lib-33333333-3333-3333-3333-333333333333.jar"), List.of(LEGACY_ARTIFACT_KEY)); cleanupNimbus(store, state).doCleanup(); @@ -559,8 +600,10 @@ void doCleanupReclaimsOnlyProvablyUniqueDependencyBlobsWhenTheReferencesCannotBe cleanupNimbus(store, state).doCleanup(); - // the dead topology's code blob goes away in this pass and a dependency key carries no topology id, so a - // blob that is not reclaimed now can never be found again; the key that carries a uuid cannot be shared + // the dead topology's code blob goes away in this pass and a dependency key carries no + // topology id, so a + // blob that is not reclaimed now can never be found again; the key that carries a uuid + // cannot be shared verify(store).deleteBlob(eq(UNIQUE_JAR_KEY), any()); // the legacy key could still be listed by the topology whose references could not be read verify(store, never()).deleteBlob(eq(LEGACY_ARTIFACT_KEY), any()); @@ -577,14 +620,17 @@ void doCleanupContinuesTheReferenceScanWhenACandidateTopologyHasNoCodeBlob() thr storeTopology(store, "dead-topo", List.of(UNIQUE_JAR_KEY), List.of(LEGACY_ARTIFACT_KEY, "dep-other-artifact-2.0.0.jar")); storeTopology(store, "live-topo", List.of(), List.of(LEGACY_ARTIFACT_KEY)); - // one candidate topology has no code blob at all, so it references no dependencies; unlike a read - // failure this does not abort the scan, the remaining topologies' references are still collected + // one candidate topology has no code blob at all, so it references no dependencies; unlike + // a read + // failure this does not abort the scan, the remaining topologies' references are still + // collected when(store.readBlob(eq(ConfigUtils.masterStormCodeKey("gone-topo")), any())) .thenThrow(new KeyNotFoundException(ConfigUtils.masterStormCodeKey("gone-topo"))); cleanupNimbus(store, state).doCleanup(); - // the scan succeeded, so even a shareable-shaped key is reclaimed once nothing references it + // the scan succeeded, so even a shareable-shaped key is reclaimed once nothing references + // it verify(store).deleteBlob(eq("dep-other-artifact-2.0.0.jar"), any()); verify(store).deleteBlob(eq(UNIQUE_JAR_KEY), any()); // while the reference of the topology that could be read is honoured @@ -615,12 +661,14 @@ void doCleanupSweepsADependencyBlobNoTopologyRefersToOnceTheInboxJarExpirationHa IStormClusterState state = mock(IStormClusterState.class); when(store.storedTopoIds()).thenReturn(Set.of()); when(state.activeStorms()).thenReturn(List.of()); - // outlived the pass that cleaned up its topology, e.g. because it was downloaded back from another nimbus + // outlived the pass that cleaned up its topology, e.g. because it was downloaded back + // from another nimbus storeKeys(store, UNIQUE_JAR_KEY); Nimbus nimbus = cleanupNimbus(store, state); nimbus.doCleanup(); - // one second short of the default nimbus.inbox.jar.expiration.secs, a submission may still be under way + // one second short of the default nimbus.inbox.jar.expiration.secs, a submission may + // still be under way Time.advanceTimeSecs(3599); nimbus.doCleanup(); verify(store, never()).deleteBlob(eq(UNIQUE_JAR_KEY), any()); @@ -639,7 +687,8 @@ void doCleanupNeverSweepsADependencyBlobThatIsReferencedOrCouldBeShared() throws when(store.storedTopoIds()).thenReturn(Set.of("live-topo")); when(state.activeStorms()).thenReturn(List.of("live-topo")); storeTopology(store, "live-topo", List.of(UNIQUE_JAR_KEY), List.of()); - // nothing refers to the legacy key, but an older client that finds it in the store refers to it without + // nothing refers to the legacy key, but an older client that finds it in the store + // refers to it without // uploading it again, so it may be about to be used storeKeys(store, UNIQUE_JAR_KEY, LEGACY_ARTIFACT_KEY); Nimbus nimbus = cleanupNimbus(store, state); @@ -660,7 +709,8 @@ void doCleanupDoesNotSweepDependencyBlobsWhenTheReferencesCannotBeRead() throws IStormClusterState state = mock(IStormClusterState.class); when(store.storedTopoIds()).thenReturn(Set.of("live-topo")); when(state.activeStorms()).thenReturn(List.of("live-topo")); - // the live topology's code blob cannot be read, so it may be the one referring to the key + // the live topology's code blob cannot be read, so it may be the one referring to the + // key when(store.readBlob(eq(ConfigUtils.masterStormCodeKey("live-topo")), any())) .thenThrow(new IOException("blob store is unhappy")); storeKeys(store, UNIQUE_JAR_KEY); @@ -687,7 +737,8 @@ void aDependencyBlobThatComesBackAfterItsTopologyWasCleanedUpIsRemovedAgain() th nimbus.doCleanup(); verify(store).deleteBlob(eq(UNIQUE_JAR_KEY), any()); - // the topology is gone for good, but another nimbus still had a copy of the blob and it was downloaded + // the topology is gone for good, but another nimbus still had a copy of the blob and it + // was downloaded // back, so the pass that knew which topology it belonged to is over when(store.storedTopoIds()).thenReturn(Set.of()); storeKeys(store, UNIQUE_JAR_KEY); @@ -701,10 +752,12 @@ void aDependencyBlobThatComesBackAfterItsTopologyWasCleanedUpIsRemovedAgain() th @Test void everyDependencyKeyACurrentClientGeneratesIsRecognisedAsUniqueToOneTopology() { - for (String fileName : List.of("commons-lang3-3.12.0.jar", "some.lib.tar.gz", "noextension")) { + for (String fileName : List.of("commons-lang3-3.12.0.jar", "some.lib.tar.gz", + "noextension")) { String key = DependencyBlobStoreUtils.generateDependencyBlobKey( DependencyBlobStoreUtils.applyUUIDToFileName(fileName)); - assertTrue(Nimbus.isProvablyUniqueDependencyKey(key), key + " should be recognised as unique"); + assertTrue(Nimbus.isProvablyUniqueDependencyKey(key), key + + " should be recognised as unique"); } } @@ -714,7 +767,8 @@ void testGetTopologyHistoryIsAuthorized() throws Exception { conf.put(DaemonConfig.NIMBUS_MONITOR_FREQ_SECS, 10); conf.put(Config.STORM_PRINCIPAL_TO_LOCAL_PLUGIN, DefaultPrincipalToLocal.class.getName()); conf.put(DaemonConfig.NIMBUS_AUTHORIZER, DenyAuthorizer.class.getName()); - nimbus = new Nimbus(conf, iNimbus, stormClusterState, nimbusInfo, localBlobStore, leaderElector, groupMapper, metricRegistry); + nimbus = new Nimbus(conf, iNimbus, stormClusterState, nimbusInfo, localBlobStore, + leaderElector, groupMapper, metricRegistry); try { setCaller("bob"); @@ -726,19 +780,24 @@ void testGetTopologyHistoryIsAuthorized() throws Exception { @Test void aDependencyKeyThatDoesNotCarryAUuidIsNotTreatedAsUniqueToOneTopology() { - // the shapes an older client wrote for an artifact, dep- plus the maven coordinate with : replaced by - + // the shapes an older client wrote for an artifact, dep- plus the maven coordinate with : + // replaced by - for (String key : List.of("dep-group-artifact-1.0.0.jar", "dep-org.apache.commons-commons-lang3-3.12.0.jar", "dep-a-b-1.2.3-SNAPSHOT.jar", - // hexadecimal looking coordinates of the wrong lengths are not a uuid either + // hexadecimal looking coordinates of the wrong lengths are not a + // uuid either "dep-com.deadbeef-cafebabe-1.0.jar", "dep-abcdefab-abcd-abcd-abcd-abcdefabcdef-1.0.jar", - // the uuid a client splices in is the last thing before the extension, so a - // coordinate that merely contains a uuid shaped run is still shareable + // the uuid a client splices in is the last thing before the + // extension, so a + // coordinate that merely contains a uuid shaped run is still + // shareable "dep-com.acme-abcdefab-abcd-abcd-abcd-abcdefabcdef-1.0.jar", // a uuid with a dot instead of a dash is not canonical "dep-lib-11111111.1111-1111-1111-111111111111.jar")) { - assertFalse(Nimbus.isProvablyUniqueDependencyKey(key), key + " should not be recognised as unique"); + assertFalse(Nimbus.isProvablyUniqueDependencyKey(key), key + + " should not be recognised as unique"); } assertFalse(Nimbus.isProvablyUniqueDependencyKey(null)); } @@ -751,7 +810,8 @@ private static void setCaller(String user) { @Test void testValidateDependencyBlobKeysRejectsKeysThatAreNotDependencies() throws Exception { - // a topology fills its own dependency lists in on the client side, so nimbus has to check that they only + // a topology fills its own dependency lists in on the client side, so nimbus has to check + // that they only // name dependency blobs before it takes ownership of them and deletes them during cleanup String victimJarKey = ConfigUtils.masterStormJarKey("victim-1-1234567890"); String victimConfKey = ConfigUtils.masterStormConfKey("victim-1-1234567890"); @@ -768,8 +828,10 @@ void testValidateDependencyBlobKeysRejectsKeysThatAreNotDependencies() throws Ex artifactField.set_dependency_artifacts(List.of(victimConfKey)); InvalidTopologyException artifactException = assertThrows(InvalidTopologyException.class, () -> Nimbus.validateDependencyBlobKeys(artifactField, localBlobStore, submitter)); - assertTrue(artifactException.get_msg().contains(victimConfKey), artifactException.get_msg()); - assertTrue(artifactException.get_msg().contains("dependency_artifacts"), artifactException.get_msg()); + assertTrue(artifactException.get_msg().contains(victimConfKey), artifactException + .get_msg()); + assertTrue(artifactException.get_msg().contains("dependency_artifacts"), artifactException + .get_msg()); // a good key followed by a bad one is caught too, and the message names the bad one StormTopology mixed = new StormTopology(); @@ -778,20 +840,24 @@ void testValidateDependencyBlobKeysRejectsKeysThatAreNotDependencies() throws Ex () -> Nimbus.validateDependencyBlobKeys(mixed, localBlobStore, submitter)); assertTrue(mixedException.get_msg().contains(victimJarKey), mixedException.get_msg()); - // a key that is not a dependency key at all is rejected on its name, without asking the blobstore about it + // a key that is not a dependency key at all is rejected on its name, without asking the + // blobstore about it verify(localBlobStore, never()).getBlobMeta(eq(victimJarKey), any()); verify(localBlobStore, never()).getBlobMeta(eq(victimConfKey), any()); } @Test void testValidateDependencyBlobKeysRejectsKeyThatIsNotInTheBlobStore() throws Exception { - // a key that merely looks like a dependency key is just as damaging: on gaining leadership a nimbus gives up - // leadership again when an active topology names a dependency it cannot find, so an unresolvable key leaves + // a key that merely looks like a dependency key is just as damaging: on gaining leadership + // a nimbus gives up + // leadership again when an active topology names a dependency it cannot find, so an + // unresolvable key leaves // the cluster without a leader String presentKey = dependencyKey("present.jar"); String missingKey = dependencyKey("missing.jar"); Subject submitter = new Subject(); - when(localBlobStore.getBlobMeta(eq(missingKey), any())).thenThrow(new KeyNotFoundException(missingKey)); + when(localBlobStore.getBlobMeta(eq(missingKey), any())) + .thenThrow(new KeyNotFoundException(missingKey)); StormTopology jarField = new StormTopology(); jarField.set_dependency_jars(List.of(presentKey, missingKey)); @@ -806,22 +872,27 @@ void testValidateDependencyBlobKeysRejectsKeyThatIsNotInTheBlobStore() throws Ex InvalidTopologyException artifactException = assertThrows(InvalidTopologyException.class, () -> Nimbus.validateDependencyBlobKeys(artifactField, localBlobStore, submitter)); assertTrue(artifactException.get_msg().contains(missingKey), artifactException.get_msg()); - assertTrue(artifactException.get_msg().contains("dependency_artifacts"), artifactException.get_msg()); + assertTrue(artifactException.get_msg().contains("dependency_artifacts"), artifactException + .get_msg()); } @Test void testValidateDependencyBlobKeysLooksBlobsUpAsTheSubmitter() throws Exception { - // looking the blob up as the submitter, the way the TOPOLOGY_BLOBSTORE_MAP entries are looked up, also - // answers whether the submitter is allowed to read the dependency it claims; a dependency blob is uploaded + // looking the blob up as the submitter, the way the TOPOLOGY_BLOBSTORE_MAP entries are + // looked up, also + // answers whether the submitter is allowed to read the dependency it claims; a dependency + // blob is uploaded // with OTHER READ, so a legitimate submission passes String key = dependencyKey("some-jar.jar"); Subject submitter = new Subject(); StormTopology topology = new StormTopology(); - // listed under both fields to show that a key is looked up once no matter how often it is named + // listed under both fields to show that a key is looked up once no matter how often it is + // named topology.set_dependency_jars(List.of(key, key)); topology.set_dependency_artifacts(List.of(key)); - assertDoesNotThrow(() -> Nimbus.validateDependencyBlobKeys(topology, localBlobStore, submitter)); + assertDoesNotThrow(() -> Nimbus.validateDependencyBlobKeys(topology, localBlobStore, + submitter)); verify(localBlobStore, times(1)).getBlobMeta(key, submitter); } @@ -830,19 +901,23 @@ void testValidateDependencyBlobKeysLooksBlobsUpAsTheSubmitter() throws Exception void testValidateDependencyBlobKeysAcceptsGeneratedKeysThatExist() throws Exception { Subject submitter = new Subject(); StormTopology topology = new StormTopology(); - topology.set_dependency_jars(List.of(dependencyKey("some-jar.jar"), dependencyKey("no-extension"))); + topology.set_dependency_jars(List.of(dependencyKey("some-jar.jar"), + dependencyKey("no-extension"))); topology.set_dependency_artifacts(List.of(dependencyKey("group-artifact-1.0.jar"))); - assertDoesNotThrow(() -> Nimbus.validateDependencyBlobKeys(topology, localBlobStore, submitter)); + assertDoesNotThrow(() -> Nimbus.validateDependencyBlobKeys(topology, localBlobStore, + submitter)); // unset lists are how a topology submitted without dependencies looks - assertDoesNotThrow(() -> Nimbus.validateDependencyBlobKeys(new StormTopology(), localBlobStore, submitter)); + assertDoesNotThrow(() -> Nimbus.validateDependencyBlobKeys(new StormTopology(), + localBlobStore, submitter)); verify(localBlobStore, never()).getBlobMeta(eq(null), any()); } @Test void testSubmitTopologyRejectsDependencyBlobKeyOfAnotherTopology() throws Exception { Map conf = Map.of(DaemonConfig.NIMBUS_MONITOR_FREQ_SECS, 10); - Nimbus submitNimbus = new Nimbus(conf, iNimbus, stormClusterState, nimbusInfo, localBlobStore, leaderElector, + Nimbus submitNimbus = new Nimbus(conf, iNimbus, stormClusterState, nimbusInfo, + localBlobStore, leaderElector, groupMapper, new StormMetricsRegistry()); when(leaderElector.isLeader()).thenReturn(true); when(stormClusterState.getTopoId(any())).thenReturn(Optional.empty()); @@ -865,12 +940,14 @@ void testSubmitTopologyRejectsDependencyBlobKeyOfAnotherTopology() throws Except @Test void testSubmitTopologyRejectsDependencyBlobKeyThatDoesNotExist() throws Exception { Map conf = Map.of(DaemonConfig.NIMBUS_MONITOR_FREQ_SECS, 10); - Nimbus submitNimbus = new Nimbus(conf, iNimbus, stormClusterState, nimbusInfo, localBlobStore, leaderElector, + Nimbus submitNimbus = new Nimbus(conf, iNimbus, stormClusterState, nimbusInfo, + localBlobStore, leaderElector, groupMapper, new StormMetricsRegistry()); when(leaderElector.isLeader()).thenReturn(true); when(stormClusterState.getTopoId(any())).thenReturn(Optional.empty()); String missingKey = dependencyKey("missing.jar"); - when(localBlobStore.getBlobMeta(eq(missingKey), any())).thenThrow(new KeyNotFoundException(missingKey)); + when(localBlobStore.getBlobMeta(eq(missingKey), any())) + .thenThrow(new KeyNotFoundException(missingKey)); TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("wordSpout", new TestWordSpout(), 1); @@ -888,6 +965,7 @@ void testSubmitTopologyRejectsDependencyBlobKeyThatDoesNotExist() throws Excepti } private static String dependencyKey(String fileName) { - return DependencyBlobStoreUtils.generateDependencyBlobKey(DependencyBlobStoreUtils.applyUUIDToFileName(fileName)); + return DependencyBlobStoreUtils.generateDependencyBlobKey(DependencyBlobStoreUtils + .applyUUIDToFileName(fileName)); } } diff --git a/storm-server/src/test/java/org/apache/storm/daemon/nimbus/Y2038HeartbeatTest.java b/storm-server/src/test/java/org/apache/storm/daemon/nimbus/Y2038HeartbeatTest.java index e4cc6d573d8..9b909f73a58 100644 --- a/storm-server/src/test/java/org/apache/storm/daemon/nimbus/Y2038HeartbeatTest.java +++ b/storm-server/src/test/java/org/apache/storm/daemon/nimbus/Y2038HeartbeatTest.java @@ -18,13 +18,17 @@ package org.apache.storm.daemon.nimbus; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; - import org.apache.storm.generated.Assignment; import org.apache.storm.generated.ClusterWorkerHeartbeat; import org.apache.storm.generated.LSWorkerHeartbeat; @@ -43,11 +47,6 @@ import org.apache.storm.utils.Time; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; - /** * Regression tests for the Y2038 heartbeat overflow (STORM issue #7897). * @@ -83,7 +82,8 @@ void testCwhTimeSecsSurvivesPost2038RoundTrip() throws Exception { TDeserializer des = new TDeserializer(new TBinaryProtocol.Factory()); des.deserialize(read, bytes); - long timeSecs = ((Number) read.getFieldValue(ClusterWorkerHeartbeat._Fields.TIME_SECS)).longValue(); + long timeSecs = ((Number) read.getFieldValue(ClusterWorkerHeartbeat._Fields.TIME_SECS)) + .longValue(); assertTrue(timeSecs > Integer.MAX_VALUE, "time_secs must not be narrowed to i32"); assertEquals(POST_2038_EPOCH_SECS, timeSecs, "time_secs must round-trip unchanged"); } @@ -102,13 +102,16 @@ void testHeartbeatCacheNoFalseTimeoutPost2038() { Set> allExecutors = Collections.singleton(Arrays.asList(1, 1)); Assignment assignment = mkAssignment(POST_2038_EPOCH_SECS, 1, 1); - cache.updateFromZkHeartbeat(TOPO_ID, mkZkExecutorBeats(1, 1, POST_2038_EPOCH_SECS), allExecutors, + cache.updateFromZkHeartbeat(TOPO_ID, mkZkExecutorBeats(1, 1, POST_2038_EPOCH_SECS), + allExecutors, TIMEOUT_SECS); Time.advanceTimeSecs(1); cache.timeoutOldHeartbeats(TOPO_ID, TIMEOUT_SECS); - Set> alive = cache.getAliveExecutors(TOPO_ID, allExecutors, assignment, TIMEOUT_SECS); - assertFalse(alive.isEmpty(), "A fresh post-2038 heartbeat must not be flagged as timed out"); + Set> alive = cache.getAliveExecutors(TOPO_ID, allExecutors, assignment, + TIMEOUT_SECS); + assertFalse(alive.isEmpty(), + "A fresh post-2038 heartbeat must not be flagged as timed out"); } } @@ -155,7 +158,8 @@ void testExecutorStartedPost2038NotMisclassifiedDead() { // No heartbeat reported yet; the executor is within the task launch window. Time.advanceTimeSecs(1); - Set> alive = cache.getAliveExecutors(TOPO_ID, allExecutors, assignment, TIMEOUT_SECS); + Set> alive = cache.getAliveExecutors(TOPO_ID, allExecutors, assignment, + TIMEOUT_SECS); assertFalse(alive.isEmpty(), "An executor launched post-2038 must stay alive during its launch window"); } @@ -188,7 +192,8 @@ private byte[] writeLegacyI32Lswh(int timeSecs, String topologyId, int port) thr return Arrays.copyOf(buffer.getArray(), buffer.length()); } - private Map, Map> mkZkExecutorBeats(int taskStart, int taskEnd, long timeSecs) { + private Map, Map> mkZkExecutorBeats(int taskStart, int taskEnd, + long timeSecs) { Map beat = new HashMap<>(); beat.put(ClientStatsUtil.TIME_SECS, timeSecs); return Collections.singletonMap(Arrays.asList(taskStart, taskEnd), beat); diff --git a/storm-server/src/test/java/org/apache/storm/daemon/supervisor/BasicContainerTest.java b/storm-server/src/test/java/org/apache/storm/daemon/supervisor/BasicContainerTest.java index 4f941d42ef4..202e83eaecc 100644 --- a/storm-server/src/test/java/org/apache/storm/daemon/supervisor/BasicContainerTest.java +++ b/storm-server/src/test/java/org/apache/storm/daemon/supervisor/BasicContainerTest.java @@ -1,17 +1,31 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.daemon.supervisor; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + import java.io.File; import java.io.IOException; import java.util.Arrays; @@ -28,24 +42,14 @@ import org.apache.storm.generated.ProfileAction; import org.apache.storm.generated.ProfileRequest; import org.apache.storm.generated.StormTopology; +import org.apache.storm.metric.StormMetricsRegistry; +import org.apache.storm.utils.ConfigUtils; import org.apache.storm.utils.LocalState; import org.apache.storm.utils.SimpleVersion; import org.apache.storm.utils.Utils; import org.apache.storm.utils.VersionInfo; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -import org.apache.storm.metric.StormMetricsRegistry; -import org.apache.storm.utils.ConfigUtils; - public class BasicContainerTest { private static void setSystemProp(String key, String value) { if (value == null) { @@ -57,7 +61,8 @@ private static void setSystemProp(String key, String value) { private static void checkpoint(Run r, String... newValues) throws Exception { if (newValues.length % 2 != 0) { - throw new IllegalArgumentException("Parameters are of the form system property name, new value"); + throw new IllegalArgumentException("Parameters are of the form system property name, " + + "new value"); } Map orig = new HashMap<>(); try { @@ -111,7 +116,7 @@ public void testCreateNewWorkerId() throws Exception { MockBasicContainer mc = new MockBasicContainer(ContainerType.LAUNCH, superConf, "SUPERVISOR", supervisorPort, port, la, iso, ls, null, new StormMetricsRegistry(), new HashMap<>(), ops, "profile"); - //null worker id means generate one... + // null worker id means generate one... assertNotNull(mc.workerId); verify(ls).getApprovedWorkers(); @@ -168,7 +173,7 @@ public void testRecoveryMiss() throws Exception { new HashMap<>(), null, "profile"); fail("Container recovered worker incorrectly"); } catch (ContainerRecoveryException e) { - //Expected + // Expected } } @@ -235,7 +240,7 @@ public void testRunProfiling() throws Exception { "SUPERVISOR", supervisorPort, port, la, iso, ls, workerId, new StormMetricsRegistry(), new HashMap<>(), ops, "profile"); - //HEAP DUMP + // HEAP DUMP ProfileRequest req = new ProfileRequest(); req.set_action(ProfileAction.JMAP_DUMP); @@ -258,7 +263,7 @@ public void testRunProfiling() throws Exception { assertEquals(Arrays.asList("profile", String.valueOf(pid), "jstack", topoRoot), cmd.cmd); assertEquals(new File(topoRoot), cmd.pwd); - //RESTART + // RESTART req.set_action(ProfileAction.JVM_RESTART); mc.runProfiling(req, false); @@ -269,7 +274,7 @@ public void testRunProfiling() throws Exception { assertEquals(Arrays.asList("profile", String.valueOf(pid), "kill"), cmd.cmd); assertEquals(new File(topoRoot), cmd.pwd); - //JPROFILE DUMP + // JPROFILE DUMP req.set_action(ProfileAction.JPROFILE_DUMP); mc.runProfiling(req, false); @@ -280,7 +285,7 @@ public void testRunProfiling() throws Exception { assertEquals(Arrays.asList("profile", String.valueOf(pid), "dump", topoRoot), cmd.cmd); assertEquals(new File(topoRoot), cmd.pwd); - //JPROFILE START + // JPROFILE START req.set_action(ProfileAction.JPROFILE_STOP); mc.runProfiling(req, false); @@ -291,7 +296,7 @@ public void testRunProfiling() throws Exception { assertEquals(Arrays.asList("profile", String.valueOf(pid), "start"), cmd.cmd); assertEquals(new File(topoRoot), cmd.pwd); - //JPROFILE STOP + // JPROFILE STOP req.set_action(ProfileAction.JPROFILE_STOP); mc.runProfiling(req, true); @@ -312,7 +317,8 @@ public void testLaunch() throws Exception { final String stormLogDir = ContainerTest.asFile(".", "target").getCanonicalPath(); final String workerId = "worker-id"; final String stormLocal = ContainerTest.asAbsPath("tmp", "storm-local"); - final String distRoot = ContainerTest.asAbsPath(stormLocal, "supervisor", "stormdist", topoId); + final String distRoot = ContainerTest.asAbsPath(stormLocal, "supervisor", "stormdist", + topoId); final File stormcode = new File(distRoot, "stormcode.ser"); final File stormjar = new File(distRoot, "stormjar.jar"); final String log4jdir = ContainerTest.asAbsPath(stormHome, "conf"); @@ -344,16 +350,16 @@ public void testLaunch() throws Exception { MockResourceIsolationManager iso = new MockResourceIsolationManager(); checkpoint(() -> { - MockBasicContainer mc = new MockBasicContainer(ContainerType.LAUNCH, superConf, - "SUPERVISOR", supervisorPort, port, la, iso, ls, workerId, new StormMetricsRegistry(), - new HashMap<>(), ops, "profile"); + MockBasicContainer mc = new MockBasicContainer(ContainerType.LAUNCH, superConf, + "SUPERVISOR", supervisorPort, port, la, iso, ls, workerId, new StormMetricsRegistry(), + new HashMap<>(), ops, "profile"); - mc.launch(); + mc.launch(); - assertEquals(1, iso.workerCmds.size()); - CommandRun cmd = iso.workerCmds.get(0); - iso.workerCmds.clear(); - assertListEquals(Arrays.asList( + assertEquals(1, iso.workerCmds.size()); + CommandRun cmd = iso.workerCmds.get(0); + iso.workerCmds.clear(); + assertListEquals(Arrays.asList( "java", "-cp", "FRAMEWORK_CP:" + stormjar.getAbsolutePath(), @@ -365,7 +371,8 @@ public void testLaunch() throws Exception { "-Dworker.id=" + workerId, "-Dworker.port=" + port, "-Dstorm.log.dir=" + stormLogDir, - "-DLog4jContextSelector=org.apache.logging.log4j.core.selector.BasicContextSelector", + "-DLog4jContextSelector=org.apache.logging.log4j.core.selector.BasicCon" + + "textSelector", "-Dstorm.local.dir=" + stormLocal, "-Dworker.memory_limit_mb=768", "-Dlog4j.configurationFile=" + workerConf, @@ -380,7 +387,8 @@ public void testLaunch() throws Exception { "-Dworker.id=" + workerId, "-Dworker.port=" + port, "-Dstorm.log.dir=" + stormLogDir, - "-DLog4jContextSelector=org.apache.logging.log4j.core.selector.BasicContextSelector", + "-DLog4jContextSelector=org.apache.logging.log4j.core.selector.BasicCon" + + "textSelector", "-Dstorm.local.dir=" + stormLocal, "-Dworker.memory_limit_mb=768", "-Dlog4j.configurationFile=" + workerConf, @@ -398,8 +406,8 @@ public void testLaunch() throws Exception { String.valueOf(port), workerId ), cmd.cmd); - assertEquals(new File(workerRoot), cmd.pwd); - }, + assertEquals(new File(workerRoot), cmd.pwd); + }, ConfigUtils.STORM_HOME, stormHome, "storm.log.dir", stormLogDir); } @@ -427,20 +435,22 @@ public void testFrameworkClasspathIncludesSharedAndWorkerLibs() throws Exception MockResourceIsolationManager iso = new MockResourceIsolationManager(); checkpoint(() -> { - MockBasicContainer mc = new MockBasicContainer(ContainerType.LAUNCH, superConf, + MockBasicContainer mc = new MockBasicContainer(ContainerType.LAUNCH, superConf, "SUPERVISOR", supervisorPort, port, la, iso, ls, "worker-id", new StormMetricsRegistry(), new HashMap<>(), ops, "profile"); - List cp = mc.realFrameworkClasspath(VersionInfo.OUR_VERSION); - - // The distribution de-duplicates the jars shared by the daemon and worker - // classpaths into lib-common; storm-client (LogWriter, Worker) ships there, - // so a worker launched without lib-common on the classpath cannot start. - String libCommon = stormHome + File.separator + "lib-common" + File.separator + "*"; - String libWorker = stormHome + File.separator + "lib-worker" + File.separator + "*"; - assertTrue(cp.contains(libCommon), "worker classpath must include lib-common/*, got: " + cp); - assertTrue(cp.contains(libWorker), "worker classpath must include lib-worker/*, got: " + cp); - }, + List cp = mc.realFrameworkClasspath(VersionInfo.OUR_VERSION); + + // The distribution de-duplicates the jars shared by the daemon and worker + // classpaths into lib-common; storm-client (LogWriter, Worker) ships there, + // so a worker launched without lib-common on the classpath cannot start. + String libCommon = stormHome + File.separator + "lib-common" + File.separator + "*"; + String libWorker = stormHome + File.separator + "lib-worker" + File.separator + "*"; + assertTrue(cp.contains(libCommon), "worker classpath must include lib-common/*, got: " + + cp); + assertTrue(cp.contains(libWorker), "worker classpath must include lib-worker/*, got: " + + cp); + }, ConfigUtils.STORM_HOME, stormHome, "storm.log.dir", stormLogDir); } @@ -454,7 +464,8 @@ public void testLaunchStorm1version() throws Exception { final String stormLogDir = ContainerTest.asFile(".", "target").getCanonicalPath(); final String workerId = "worker-id"; final String stormLocal = ContainerTest.asAbsPath("tmp", "storm-local"); - final String distRoot = ContainerTest.asAbsPath(stormLocal, "supervisor", "stormdist", topoId); + final String distRoot = ContainerTest.asAbsPath(stormLocal, "supervisor", "stormdist", + topoId); final File stormcode = new File(distRoot, "stormcode.ser"); final File stormjar = new File(distRoot, "stormjar.jar"); final String log4jdir = ContainerTest.asAbsPath(stormHome, "conf"); @@ -489,16 +500,16 @@ public void testLaunchStorm1version() throws Exception { MockResourceIsolationManager iso = new MockResourceIsolationManager(); checkpoint(() -> { - MockBasicContainer mc = new MockBasicContainer(ContainerType.LAUNCH, superConf, - "SUPERVISOR", supervisorPort, port, la, iso, ls, workerId, new StormMetricsRegistry(), - new HashMap<>(), ops, "profile"); + MockBasicContainer mc = new MockBasicContainer(ContainerType.LAUNCH, superConf, + "SUPERVISOR", supervisorPort, port, la, iso, ls, workerId, new StormMetricsRegistry(), + new HashMap<>(), ops, "profile"); - mc.launch(); + mc.launch(); - assertEquals(1, iso.workerCmds.size()); - CommandRun cmd = iso.workerCmds.get(0); - iso.workerCmds.clear(); - assertListEquals(Arrays.asList( + assertEquals(1, iso.workerCmds.size()); + CommandRun cmd = iso.workerCmds.get(0); + iso.workerCmds.clear(); + assertListEquals(Arrays.asList( "java", "-cp", "FRAMEWORK_CP:" + stormjar.getAbsolutePath(), @@ -510,7 +521,8 @@ public void testLaunchStorm1version() throws Exception { "-Dworker.id=" + workerId, "-Dworker.port=" + port, "-Dstorm.log.dir=" + stormLogDir, - "-DLog4jContextSelector=org.apache.logging.log4j.core.selector.BasicContextSelector", + "-DLog4jContextSelector=org.apache.logging.log4j.core.selector.BasicCon" + + "textSelector", "-Dstorm.local.dir=" + stormLocal, "-Dworker.memory_limit_mb=768", "-Dlog4j.configurationFile=" + workerConf, @@ -525,7 +537,8 @@ public void testLaunchStorm1version() throws Exception { "-Dworker.id=" + workerId, "-Dworker.port=" + port, "-Dstorm.log.dir=" + stormLogDir, - "-DLog4jContextSelector=org.apache.logging.log4j.core.selector.BasicContextSelector", + "-DLog4jContextSelector=org.apache.logging.log4j.core.selector.BasicCon" + + "textSelector", "-Dstorm.local.dir=" + stormLocal, "-Dworker.memory_limit_mb=768", "-Dlog4j.configurationFile=" + workerConf, @@ -542,8 +555,8 @@ public void testLaunchStorm1version() throws Exception { String.valueOf(port), workerId ), cmd.cmd); - assertEquals(new File(workerRoot), cmd.pwd); - }, + assertEquals(new File(workerRoot), cmd.pwd); + }, ConfigUtils.STORM_HOME, stormHome, "storm.log.dir", stormLogDir); } @@ -557,7 +570,8 @@ public void testLaunchStorm0version() throws Exception { final String stormLogDir = ContainerTest.asFile(".", "target").getCanonicalPath(); final String workerId = "worker-id"; final String stormLocal = ContainerTest.asAbsPath("tmp", "storm-local"); - final String distRoot = ContainerTest.asAbsPath(stormLocal, "supervisor", "stormdist", topoId); + final String distRoot = ContainerTest.asAbsPath(stormLocal, "supervisor", "stormdist", + topoId); final File stormcode = new File(distRoot, "stormcode.ser"); final File stormjar = new File(distRoot, "stormjar.jar"); final String log4jdir = ContainerTest.asAbsPath(stormHome, "conf"); @@ -592,16 +606,16 @@ public void testLaunchStorm0version() throws Exception { MockResourceIsolationManager iso = new MockResourceIsolationManager(); checkpoint(() -> { - MockBasicContainer mc = new MockBasicContainer(ContainerType.LAUNCH, superConf, - "SUPERVISOR", supervisorPort, port, la, iso, ls, workerId, new StormMetricsRegistry(), - new HashMap<>(), ops, "profile"); + MockBasicContainer mc = new MockBasicContainer(ContainerType.LAUNCH, superConf, + "SUPERVISOR", supervisorPort, port, la, iso, ls, workerId, new StormMetricsRegistry(), + new HashMap<>(), ops, "profile"); - mc.launch(); + mc.launch(); - assertEquals(1, iso.workerCmds.size()); - CommandRun cmd = iso.workerCmds.get(0); - iso.workerCmds.clear(); - assertListEquals(Arrays.asList( + assertEquals(1, iso.workerCmds.size()); + CommandRun cmd = iso.workerCmds.get(0); + iso.workerCmds.clear(); + assertListEquals(Arrays.asList( "java", "-cp", "FRAMEWORK_CP:" + stormjar.getAbsolutePath(), @@ -613,7 +627,8 @@ public void testLaunchStorm0version() throws Exception { "-Dworker.id=" + workerId, "-Dworker.port=" + port, "-Dstorm.log.dir=" + stormLogDir, - "-DLog4jContextSelector=org.apache.logging.log4j.core.selector.BasicContextSelector", + "-DLog4jContextSelector=org.apache.logging.log4j.core.selector.BasicCon" + + "textSelector", "-Dstorm.local.dir=" + stormLocal, "-Dworker.memory_limit_mb=768", "-Dlog4j.configurationFile=" + workerConf, @@ -628,7 +643,8 @@ public void testLaunchStorm0version() throws Exception { "-Dworker.id=" + workerId, "-Dworker.port=" + port, "-Dstorm.log.dir=" + stormLogDir, - "-DLog4jContextSelector=org.apache.logging.log4j.core.selector.BasicContextSelector", + "-DLog4jContextSelector=org.apache.logging.log4j.core.selector.BasicCon" + + "textSelector", "-Dstorm.local.dir=" + stormLocal, "-Dworker.memory_limit_mb=768", "-Dlog4j.configurationFile=" + workerConf, @@ -645,8 +661,8 @@ public void testLaunchStorm0version() throws Exception { String.valueOf(port), workerId ), cmd.cmd); - assertEquals(new File(workerRoot), cmd.pwd); - }, + assertEquals(new File(workerRoot), cmd.pwd); + }, ConfigUtils.STORM_HOME, stormHome, "storm.log.dir", stormLogDir); } @@ -681,7 +697,9 @@ public void testSubstChildOpts() throws Exception { "-Xms256m", "-Xmx512m", "-XX:MaxDirectMemorySize=256m"), mc.substituteChildopts( - "-Xloggc:/tmp/storm/logs/gc.worker-%ID%-%TOPOLOGY-ID%-%WORKER-ID%-%WORKER-PORT%.log -Xms256m -Xmx%HEAP-MEM%m -XX:MaxDirectMemorySize=%OFF-HEAP-MEM%m", + "-Xloggc:/tmp/storm/logs/gc.worker-%ID%-%TOPOLOGY-ID%-%WORKER-ID%-%WORKER-" + + "PORT%.log -Xms256m -Xmx%HEAP-MEM%m " + + "-XX:MaxDirectMemorySize=%OFF-HEAP-MEM%m", memOnheap, memOffheap)); assertListEquals( @@ -692,7 +710,8 @@ public void testSubstChildOpts() throws Exception { ), mc.substituteChildopts( Arrays.asList( - "-Xloggc:/tmp/storm/logs/gc.worker-%ID%-%TOPOLOGY-ID%-%WORKER-ID%-%WORKER-PORT%.log", "-Xms256m", + "-Xloggc:/tmp/storm/logs/gc.worker-%ID%-%TOPOLOGY-ID%-%WORKER-ID%-%WORKER-" + + "PORT%.log", "-Xms256m", "-Xmx%HEAP-MEM%m" ), memOnheap, memOffheap ) @@ -719,12 +738,14 @@ public CommandRun(List cmd, Map env, File pwd) { } public static class MockBasicContainer extends BasicContainer { - public MockBasicContainer(ContainerType type, Map conf, String supervisorId, int supervisorPort, + public MockBasicContainer(ContainerType type, Map conf, String supervisorId, + int supervisorPort, int port, LocalAssignment assignment, ResourceIsolationInterface resourceIsolationManager, LocalState localState, String workerId, StormMetricsRegistry metricsRegistry, Map topoConf, AdvancedFSOps ops, String profileCmd) throws IOException { - super(type, conf, supervisorId, supervisorPort, port, assignment, resourceIsolationManager, localState, - workerId, metricsRegistry,new ContainerMemoryTracker(metricsRegistry), topoConf, ops, profileCmd); + super(type, conf, supervisorId, supervisorPort, port, assignment, + resourceIsolationManager, localState, + workerId, metricsRegistry, new ContainerMemoryTracker(metricsRegistry), topoConf, ops, profileCmd); } @Override @@ -744,13 +765,13 @@ public List substituteChildopts(Object value, int memOnheap, int memOffH @Override protected String javaCmd(String cmd) { - //avoid system dependent things + // avoid system dependent things return cmd; } @Override protected List frameworkClasspath(SimpleVersion version) { - //We are not really running anything so make this + // We are not really running anything so make this // simple to check for return Collections.singletonList("FRAMEWORK_CP"); } diff --git a/storm-server/src/test/java/org/apache/storm/daemon/supervisor/ContainerTest.java b/storm-server/src/test/java/org/apache/storm/daemon/supervisor/ContainerTest.java index bf17ea2a185..e6e130b2c81 100644 --- a/storm-server/src/test/java/org/apache/storm/daemon/supervisor/ContainerTest.java +++ b/storm-server/src/test/java/org/apache/storm/daemon/supervisor/ContainerTest.java @@ -1,19 +1,35 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.daemon.supervisor; -import com.google.common.base.Joiner; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import com.google.common.base.Joiner; import java.io.File; import java.io.IOException; import java.io.StringWriter; @@ -24,7 +40,6 @@ import java.util.HashSet; import java.util.List; import java.util.Map; - import org.apache.storm.Config; import org.apache.storm.DaemonConfig; import org.apache.storm.container.ResourceIsolationInterface; @@ -32,23 +47,11 @@ import org.apache.storm.daemon.supervisor.Container.ContainerType; import org.apache.storm.generated.LocalAssignment; import org.apache.storm.generated.ProfileRequest; +import org.apache.storm.metric.StormMetricsRegistry; import org.apache.storm.utils.ObjectReader; import org.junit.jupiter.api.Test; import org.yaml.snakeyaml.Yaml; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.fail; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -import org.apache.storm.metric.StormMetricsRegistry; - public class ContainerTest { private static final Joiner PATH_JOIN = Joiner.on(File.separator).skipNulls(); private static final String DOUBLE_SEP = File.separator + File.separator; @@ -142,13 +145,13 @@ public void testSetup() throws Exception { mc.setup(); - //Initial Setup + // Initial Setup verify(ops).forceMkdir(new File(workerRoot, "pids")); verify(ops).forceMkdir(new File(workerRoot, "tmp")); verify(ops).forceMkdir(new File(workerRoot, "heartbeats")); verify(ops).fileExists(workerArtifacts); - //Log file permissions + // Log file permissions verify(ops).getWriter(logMetadataFile); String yamlResult = yamlDump.toString(); @@ -158,20 +161,23 @@ public void testSetup() throws Exception { assertEquals(user, result.get(Config.TOPOLOGY_SUBMITTER_USER)); HashSet allowedUsers = new HashSet<>(topoUsers); allowedUsers.addAll(logUsers); - assertEquals(allowedUsers, new HashSet<>(ObjectReader.getStrings(result.get(DaemonConfig.LOGS_USERS)))); + assertEquals(allowedUsers, new HashSet<>(ObjectReader.getStrings(result + .get(DaemonConfig.LOGS_USERS)))); HashSet allowedGroups = new HashSet<>(topoGroups); allowedGroups.addAll(logGroups); - assertEquals(allowedGroups, new HashSet<>(ObjectReader.getStrings(result.get(DaemonConfig.LOGS_GROUPS)))); + assertEquals(allowedGroups, new HashSet<>(ObjectReader.getStrings(result + .get(DaemonConfig.LOGS_GROUPS)))); - //Save the current user to help with recovery + // Save the current user to help with recovery verify(ops).dump(workerUserFile, user); - //Create links to artifacts dir + // Create links to artifacts dir verify(ops).createSymlink(new File(workerRoot, "artifacts"), workerArtifacts); - //Create links to blobs - verify(ops, never()).createSymlink(new File(workerRoot, "resources"), new File(distRoot, "resources")); + // Create links to blobs + verify(ops, never()).createSymlink(new File(workerRoot, "resources"), new File(distRoot, + "resources")); } @Test @@ -202,9 +208,10 @@ public void testCreateBlobstoreLinks() throws Exception { "SUPERVISOR", 6628, port, la, iso, workerId, topoConf, ops, new StormMetricsRegistry()); mc.createBlobstoreLinks(); - verify(ops).createSymlink(new File(workerRoot, "simple.txt"), new File(distRoot, "simple.txt")); + verify(ops).createSymlink(new File(workerRoot, "simple.txt"), new File(distRoot, + "simple.txt")); - //a localname that points outside of the worker root must not result in any link + // a localname that points outside of the worker root must not result in any link AdvancedFSOps badOps = mock(AdvancedFSOps.class); when(badOps.doRequiredTopoFilesExist(superConf, topoId)).thenReturn(true); blobInfo.put("localname", asPath("..", "..", "escaped.txt")); @@ -263,10 +270,12 @@ public void testCleanup() throws Exception { public static class MockContainer extends Container { - protected MockContainer(ContainerType type, Map conf, String supervisorId, int supervisorPort, + protected MockContainer(ContainerType type, Map conf, String supervisorId, + int supervisorPort, int port, LocalAssignment assignment, ResourceIsolationInterface resourceIsolationManager, String workerId, Map topoConf, AdvancedFSOps ops, StormMetricsRegistry metricsRegistry) throws IOException { - super(type, conf, supervisorId, supervisorPort, port, assignment, resourceIsolationManager, workerId, + super(type, conf, supervisorId, supervisorPort, port, assignment, + resourceIsolationManager, workerId, topoConf, ops, metricsRegistry, new ContainerMemoryTracker(new StormMetricsRegistry())); } @@ -307,12 +316,14 @@ public void prepare(Map conf) throws IOException { } @Override - public void reserveResourcesForWorker(String workerId, Integer workerMemory, Integer workerCpu, String numaId) { + public void reserveResourcesForWorker(String workerId, Integer workerMemory, + Integer workerCpu, String numaId) { fail("THIS IS NOT UNDER TEST"); } @Override - public void launchWorkerProcess(String user, String topologyId, Map topoConf, + public void launchWorkerProcess(String user, String topologyId, Map topoConf, int port, String workerId, List command, Map env, String logPrefix, ExitCodeCallback processExitCallback, File targetDir) { @@ -348,14 +359,15 @@ public boolean areAllProcessesDead(String user, String workerId) { } @Override - public boolean runProfilingCommand(String user, String workerId, List command, Map env, String logPrefix, File targetDir) { + public boolean runProfilingCommand(String user, String workerId, List command, + Map env, String logPrefix, File targetDir) { profileCmds.add(new CommandRun(command, env, targetDir)); return true; } @Override public void cleanup(String user, String workerId, int port) { - //NO OP + // NO OP } @Override diff --git a/storm-server/src/test/java/org/apache/storm/daemon/supervisor/SlotTest.java b/storm-server/src/test/java/org/apache/storm/daemon/supervisor/SlotTest.java index 5dacb3fa2e3..0b909a8850f 100644 --- a/storm-server/src/test/java/org/apache/storm/daemon/supervisor/SlotTest.java +++ b/storm-server/src/test/java/org/apache/storm/daemon/supervisor/SlotTest.java @@ -1,12 +1,17 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -14,6 +19,24 @@ import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.nullValue; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.lessThan; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.anyBoolean; +import static org.mockito.Mockito.anyInt; +import static org.mockito.Mockito.anyLong; +import static org.mockito.Mockito.clearInvocations; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import java.util.ArrayList; import java.util.Collections; @@ -21,6 +44,7 @@ import java.util.List; import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import org.apache.storm.daemon.supervisor.Slot.DynamicState; import org.apache.storm.daemon.supervisor.Slot.MachineState; @@ -37,25 +61,13 @@ import org.apache.storm.localizer.BlobChangingCallback; import org.apache.storm.localizer.GoodToGo; import org.apache.storm.localizer.LocallyCachedBlob; +import org.apache.storm.metric.StormMetricsRegistry; import org.apache.storm.scheduler.ISupervisor; import org.apache.storm.utils.LocalState; -import org.apache.storm.utils.Time; import org.apache.storm.utils.Time.SimulatedTime; +import org.apache.storm.utils.Time; import org.junit.jupiter.api.Test; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.lessThan; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertSame; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.*; - -import java.util.concurrent.ExecutionException; -import org.apache.storm.metric.StormMetricsRegistry; - public class SlotTest { static WorkerResources mkWorkerResources(Double cpu, Double mem_on_heap, Double mem_off_heap) { @@ -74,7 +86,8 @@ static WorkerResources mkWorkerResources(Double cpu, Double mem_on_heap, Double return resources; } - static LSWorkerHeartbeat mkWorkerHB(String id, int port, List exec, Integer timeSecs) { + static LSWorkerHeartbeat mkWorkerHB(String id, int port, List exec, + Integer timeSecs) { LSWorkerHeartbeat ret = new LSWorkerHeartbeat(); ret.set_topology_id(id); ret.set_port(port); @@ -94,7 +107,8 @@ static List mkExecutorInfoList(int... executors) { return ret; } - static LocalAssignment mkLocalAssignment(String id, List exec, WorkerResources resources) { + static LocalAssignment mkLocalAssignment(String id, List exec, + WorkerResources resources) { LocalAssignment ret = new LocalAssignment(); ret.set_topology_id(id); ret.set_executors(exec); @@ -106,10 +120,14 @@ static LocalAssignment mkLocalAssignment(String id, List exec, Wor @Test public void testForSameTopology() { - LocalAssignment a = mkLocalAssignment("A", mkExecutorInfoList(1, 2, 3, 4, 5), mkWorkerResources(100.0, 100.0, 100.0)); - LocalAssignment aResized = mkLocalAssignment("A", mkExecutorInfoList(1, 2, 3, 4, 5), mkWorkerResources(100.0, 200.0, 100.0)); - LocalAssignment b = mkLocalAssignment("B", mkExecutorInfoList(1, 2, 3, 4, 5, 6), mkWorkerResources(100.0, 100.0, 100.0)); - LocalAssignment bReordered = mkLocalAssignment("B", mkExecutorInfoList(6, 5, 4, 3, 2, 1), mkWorkerResources(100.0, 100.0, 100.0)); + LocalAssignment a = mkLocalAssignment("A", mkExecutorInfoList(1, 2, 3, 4, 5), + mkWorkerResources(100.0, 100.0, 100.0)); + LocalAssignment aResized = mkLocalAssignment("A", mkExecutorInfoList(1, 2, 3, 4, 5), + mkWorkerResources(100.0, 200.0, 100.0)); + LocalAssignment b = mkLocalAssignment("B", mkExecutorInfoList(1, 2, 3, 4, 5, 6), + mkWorkerResources(100.0, 100.0, 100.0)); + LocalAssignment bReordered = mkLocalAssignment("B", mkExecutorInfoList(6, 5, 4, 3, 2, 1), + mkWorkerResources(100.0, 100.0, 100.0)); assertTrue(Slot.forSameTopology(null, null)); assertTrue(Slot.forSameTopology(a, a)); @@ -156,13 +174,15 @@ public void testLaunchContainerFromEmpty() throws Exception { Container container = mock(Container.class); LocalState state = mock(LocalState.class); ContainerLauncher containerLauncher = mock(ContainerLauncher.class); - when(containerLauncher.launchContainer(port, newAssignment, state)).thenReturn(container); + when(containerLauncher.launchContainer(port, newAssignment, state)) + .thenReturn(container); LSWorkerHeartbeat hb = mkWorkerHB(topoId, port, execList, Time.currentTimeSecs()); when(container.readHeartbeat()).thenReturn(hb, hb); @SuppressWarnings("unchecked") CompletableFuture blobFuture = mock(CompletableFuture.class); - when(localizer.requestDownloadTopologyBlobs(newAssignment, port, cb)).thenReturn(blobFuture); + when(localizer.requestDownloadTopologyBlobs(newAssignment, port, cb)) + .thenReturn(blobFuture); ISupervisor iSuper = mock(ISupervisor.class); SlotMetrics slotMetrics = new SlotMetrics(new StormMetricsRegistry()); @@ -228,14 +248,16 @@ public void testErrorHandlingWhenLocalizationFails() throws Exception { Container container = mock(Container.class); LocalState state = mock(LocalState.class); ContainerLauncher containerLauncher = mock(ContainerLauncher.class); - when(containerLauncher.launchContainer(port, newAssignment, state)).thenReturn(container); + when(containerLauncher.launchContainer(port, newAssignment, state)) + .thenReturn(container); LSWorkerHeartbeat hb = mkWorkerHB(topoId, port, execList, Time.currentTimeSecs()); when(container.readHeartbeat()).thenReturn(hb, hb); @SuppressWarnings("unchecked") CompletableFuture blobFuture = mock(CompletableFuture.class); CompletableFuture secondBlobFuture = mock(CompletableFuture.class); - when(secondBlobFuture.get(anyLong(), any())).thenThrow(new ExecutionException(new RuntimeException("Localization failure"))); + when(secondBlobFuture.get(anyLong(), any())) + .thenThrow(new ExecutionException(new RuntimeException("Localization failure"))); CompletableFuture thirdBlobFuture = mock(CompletableFuture.class); when(localizer.requestDownloadTopologyBlobs(newAssignment, port, cb)) .thenReturn(blobFuture) @@ -256,7 +278,7 @@ public void testErrorHandlingWhenLocalizationFails() throws Exception { assertEquals(newAssignment, nextState.pendingLocalization); assertEquals(0, Time.currentTimeMillis()); - //Assignment has changed + // Assignment has changed nextState = Slot.stateMachineStep(nextState.withNewAssignment(null), staticState); assertThat(nextState.state, is(MachineState.EMPTY)); assertThat(nextState.pendingChangingBlobs, is(Collections.emptySet())); @@ -265,18 +287,21 @@ public void testErrorHandlingWhenLocalizationFails() throws Exception { assertThat(nextState.pendingDownload, nullValue()); clearInvocations(localizer); - nextState = Slot.stateMachineStep(dynamicState.withNewAssignment(newAssignment), staticState); + nextState = Slot.stateMachineStep(dynamicState.withNewAssignment(newAssignment), + staticState); verify(localizer).requestDownloadTopologyBlobs(newAssignment, port, cb); assertEquals(MachineState.WAITING_FOR_BLOB_LOCALIZATION, nextState.state); - assertSame(secondBlobFuture, nextState.pendingDownload, "pendingDownload not set properly"); + assertSame(secondBlobFuture, nextState.pendingDownload, + "pendingDownload not set properly"); assertEquals(newAssignment, nextState.pendingLocalization); - //Error occurs, but assignment has not changed + // Error occurs, but assignment has not changed clearInvocations(localizer); nextState = Slot.stateMachineStep(nextState, staticState); verify(localizer).requestDownloadTopologyBlobs(newAssignment, port, cb); assertEquals(MachineState.WAITING_FOR_BLOB_LOCALIZATION, nextState.state); - assertSame(thirdBlobFuture, nextState.pendingDownload, "pendingDownload not set properly"); + assertSame(thirdBlobFuture, nextState.pendingDownload, + "pendingDownload not set properly"); assertEquals(newAssignment, nextState.pendingLocalization); assertThat(Time.currentTimeMillis(), greaterThan(3L)); @@ -304,7 +329,8 @@ public void testRelaunch() throws Exception { BlobChangingCallback cb = mock(BlobChangingCallback.class); Container container = mock(Container.class); ContainerLauncher containerLauncher = mock(ContainerLauncher.class); - LSWorkerHeartbeat oldhb = mkWorkerHB(topoId, port, execList, Time.currentTimeSecs() - 10); + LSWorkerHeartbeat oldhb = mkWorkerHB(topoId, port, execList, Time + .currentTimeSecs() - 10); LSWorkerHeartbeat goodhb = mkWorkerHB(topoId, port, execList, Time.currentTimeSecs()); when(container.readHeartbeat()).thenReturn(oldhb, oldhb, goodhb, goodhb); when(container.areAllProcessesDead()).thenReturn(false, false, true); @@ -314,7 +340,8 @@ public void testRelaunch() throws Exception { SlotMetrics slotMetrics = new SlotMetrics(new StormMetricsRegistry()); StaticState staticState = new StaticState(localizer, 5000, 120000, 1000, 1000, containerLauncher, "localhost", port, iSuper, state, cb, null, null, slotMetrics); - DynamicState dynamicState = new DynamicState(assignment, container, assignment, slotMetrics); + DynamicState dynamicState = new DynamicState(assignment, container, assignment, + slotMetrics); DynamicState nextState = Slot.stateMachineStep(dynamicState, staticState); assertEquals(MachineState.KILL_AND_RELAUNCH, nextState.state); @@ -364,19 +391,22 @@ public void testReschedule() throws Exception { Container nContainer = mock(Container.class); LocalState state = mock(LocalState.class); ContainerLauncher containerLauncher = mock(ContainerLauncher.class); - when(containerLauncher.launchContainer(port, nAssignment, state)).thenReturn(nContainer); + when(containerLauncher.launchContainer(port, nAssignment, state)) + .thenReturn(nContainer); LSWorkerHeartbeat nhb = mkWorkerHB(nTopoId, 100, nExecList, Time.currentTimeSecs()); when(nContainer.readHeartbeat()).thenReturn(nhb, nhb); @SuppressWarnings("unchecked") CompletableFuture blobFuture = mock(CompletableFuture.class); - when(localizer.requestDownloadTopologyBlobs(nAssignment, port, cb)).thenReturn(blobFuture); + when(localizer.requestDownloadTopologyBlobs(nAssignment, port, cb)) + .thenReturn(blobFuture); ISupervisor iSuper = mock(ISupervisor.class); SlotMetrics slotMetrics = new SlotMetrics(new StormMetricsRegistry()); StaticState staticState = new StaticState(localizer, 5000, 120000, 1000, 1000, containerLauncher, "localhost", port, iSuper, state, cb, null, null, slotMetrics); - DynamicState dynamicState = new DynamicState(cAssignment, cContainer, nAssignment, slotMetrics); + DynamicState dynamicState = new DynamicState(cAssignment, cContainer, nAssignment, + slotMetrics); DynamicState nextState = Slot.stateMachineStep(dynamicState, staticState); assertEquals(MachineState.KILL, nextState.state); @@ -458,7 +488,8 @@ public void testRunningToEmpty() throws Exception { SlotMetrics slotMetrics = new SlotMetrics(new StormMetricsRegistry()); StaticState staticState = new StaticState(localizer, 5000, 120000, 1000, 1000, containerLauncher, "localhost", port, iSuper, state, cb, null, null, slotMetrics); - DynamicState dynamicState = new DynamicState(cAssignment, cContainer, null, slotMetrics); + DynamicState dynamicState = new DynamicState(cAssignment, cContainer, null, + slotMetrics); DynamicState nextState = Slot.stateMachineStep(dynamicState, staticState); assertEquals(MachineState.KILL, nextState.state); @@ -507,7 +538,8 @@ public void testRunWithProfileActions() throws Exception { mkLocalAssignment(cTopoId, cExecList, mkWorkerResources(100.0, 100.0, 100.0)); Container cContainer = mock(Container.class); - LSWorkerHeartbeat chb = mkWorkerHB(cTopoId, port, cExecList, Time.currentTimeSecs() + 100); //NOT going to timeout for a while + LSWorkerHeartbeat chb = mkWorkerHB(cTopoId, port, cExecList, Time.currentTimeSecs() + + 100); // NOT going to timeout for a while when(cContainer.readHeartbeat()).thenReturn(chb, chb, chb, chb, chb, chb); when(cContainer.runProfiling(any(ProfileRequest.class), anyBoolean())).thenReturn(true); @@ -526,7 +558,7 @@ public void testRunWithProfileActions() throws Exception { info.set_node("localhost"); info.add_to_port(port); request.set_nodeInfo(info); - request.set_time_stamp(Time.currentTimeMillis() + 3000);//3 seconds from now + request.set_time_stamp(Time.currentTimeMillis() + 3000); // 3 seconds from now TopoProfileAction profile = new TopoProfileAction(cTopoId, request); profileActions.add(profile); @@ -534,7 +566,8 @@ public void testRunWithProfileActions() throws Exception { expectedPending.add(profile); SlotMetrics slotMetrics = new SlotMetrics(new StormMetricsRegistry()); - DynamicState dynamicState = new DynamicState(cAssignment, cContainer, cAssignment, slotMetrics) + DynamicState dynamicState = new DynamicState(cAssignment, cContainer, cAssignment, + slotMetrics) .withProfileActions(profileActions, Collections.emptySet()); DynamicState nextState = Slot.stateMachineStep(dynamicState, staticState); @@ -560,13 +593,15 @@ public void testRunWithProfileActions() throws Exception { nextState = Slot.stateMachineStep(nextState, staticState); assertEquals(MachineState.RUNNING, nextState.state); verify(cContainer).runProfiling(request, true); - assertEquals(Collections.emptySet(), nextState.pendingStopProfileActions); + assertEquals(Collections.emptySet(), + nextState.pendingStopProfileActions); assertEquals(Collections.emptySet(), nextState.profileActions); assertTrue(Time.currentTimeMillis() > 4000); nextState = Slot.stateMachineStep(nextState, staticState); assertEquals(MachineState.RUNNING, nextState.state); - assertEquals(Collections.emptySet(), nextState.pendingStopProfileActions); + assertEquals(Collections.emptySet(), + nextState.pendingStopProfileActions); assertEquals(Collections.emptySet(), nextState.profileActions); assertTrue(Time.currentTimeMillis() > 5000); } @@ -582,7 +617,8 @@ public void testResourcesChangedFiltered() throws Exception { mkLocalAssignment(cTopoId, cExecList, mkWorkerResources(100.0, 100.0, 100.0)); String otherTopoId = "OTHER"; - LocalAssignment otherAssignment = mkLocalAssignment(otherTopoId, cExecList, mkWorkerResources(100.0, 100.0, 100.0)); + LocalAssignment otherAssignment = mkLocalAssignment(otherTopoId, cExecList, + mkWorkerResources(100.0, 100.0, 100.0)); BlobChangingCallback cb = mock(BlobChangingCallback.class); @@ -594,12 +630,14 @@ public void testResourcesChangedFiltered() throws Exception { Container nContainer = mock(Container.class); LocalState state = mock(LocalState.class); ContainerLauncher containerLauncher = mock(ContainerLauncher.class); - when(containerLauncher.launchContainer(port, cAssignment, state)).thenReturn(nContainer); + when(containerLauncher.launchContainer(port, cAssignment, state)) + .thenReturn(nContainer); when(nContainer.readHeartbeat()).thenReturn(chb, chb); ISupervisor iSuper = mock(ISupervisor.class); long heartbeatTimeoutMs = 5000; - StaticState staticState = new StaticState(localizer, heartbeatTimeoutMs, 120_000, 1000, 1000, + StaticState staticState = new StaticState(localizer, heartbeatTimeoutMs, 120_000, 1000, + 1000, containerLauncher, "localhost", port, iSuper, state, cb, null, null, new SlotMetrics(new StormMetricsRegistry())); Set changing = new HashSet<>(); @@ -616,7 +654,8 @@ public void testResourcesChangedFiltered() throws Exception { changing.add(new Slot.BlobChanging(otherAssignment, otherJar, otherJarLatch)); SlotMetrics slotMetrics = new SlotMetrics(new StormMetricsRegistry()); - DynamicState dynamicState = new DynamicState(cAssignment, cContainer, cAssignment, slotMetrics).withChangingBlobs(changing); + DynamicState dynamicState = new DynamicState(cAssignment, cContainer, cAssignment, + slotMetrics).withChangingBlobs(changing); DynamicState nextState = Slot.stateMachineStep(dynamicState, staticState); assertEquals(MachineState.KILL_BLOB_UPDATE, nextState.state); diff --git a/storm-server/src/test/java/org/apache/storm/daemon/supervisor/timer/ReportWorkerHeartbeatsTest.java b/storm-server/src/test/java/org/apache/storm/daemon/supervisor/timer/ReportWorkerHeartbeatsTest.java index c9c85cde121..11209a8aeee 100644 --- a/storm-server/src/test/java/org/apache/storm/daemon/supervisor/timer/ReportWorkerHeartbeatsTest.java +++ b/storm-server/src/test/java/org/apache/storm/daemon/supervisor/timer/ReportWorkerHeartbeatsTest.java @@ -1,12 +1,17 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -96,14 +101,16 @@ public void freshHeartbeatsAreReportedAndStaleOnesAreFilteredOut() { Map local = new LinkedHashMap<>(); // live worker: heartbeat just refreshed local.put("w-fresh", mkHeartbeat("topo-fresh", now)); - // exactly at the timeout boundary is still considered alive (age == timeout, not > timeout) + // exactly at the timeout boundary is still considered alive (age == timeout, not > + // timeout) local.put("w-boundary", mkHeartbeat("topo-boundary", now - WORKER_TIMEOUT_SECS)); // just past the timeout: the worker is already considered dead local.put("w-stale", mkHeartbeat("topo-stale", now - WORKER_TIMEOUT_SECS - 1)); // orphaned worker directory left behind long ago for a topology that no longer exists local.put("w-orphan", mkHeartbeat("topo-orphan", now - 86_400)); - SupervisorWorkerHeartbeats result = mkReporter().getSupervisorWorkerHeartbeatsFromLocal(local); + SupervisorWorkerHeartbeats result = mkReporter() + .getSupervisorWorkerHeartbeatsFromLocal(local); assertEquals(SUPERVISOR_ID, result.get_supervisor_id()); assertEquals(Set.of("topo-fresh", "topo-boundary"), reportedTopologies(result)); @@ -120,7 +127,8 @@ public void nullLocalHeartbeatsAreSkipped() { local.put("w-null", null); local.put("w-fresh", mkHeartbeat("topo-fresh", now)); - SupervisorWorkerHeartbeats result = mkReporter().getSupervisorWorkerHeartbeatsFromLocal(local); + SupervisorWorkerHeartbeats result = mkReporter() + .getSupervisorWorkerHeartbeatsFromLocal(local); assertEquals(Set.of("topo-fresh"), reportedTopologies(result)); } @@ -128,7 +136,8 @@ public void nullLocalHeartbeatsAreSkipped() { @Test public void perTopologyTimeoutOverrideExtendsTheStaleThreshold() throws Exception { - // topo-long raises its own worker timeout well above the global one, mirroring Slot.getHbTimeoutMs. + // topo-long raises its own worker timeout well above the global one, mirroring + // Slot.getHbTimeoutMs. when(mockConfigUtils.readSupervisorStormConfImpl(any(), eq("topo-long"))) .thenReturn(topoConf(Config.TOPOLOGY_WORKER_TIMEOUT_SECS, 300)); @@ -142,7 +151,8 @@ public void perTopologyTimeoutOverrideExtendsTheStaleThreshold() throws Exceptio // same age, no override: already dead under the global timeout -> filtered out local.put("w-default", mkHeartbeat("topo-default", now - 100)); - SupervisorWorkerHeartbeats result = mkReporter().getSupervisorWorkerHeartbeatsFromLocal(local); + SupervisorWorkerHeartbeats result = mkReporter() + .getSupervisorWorkerHeartbeatsFromLocal(local); assertEquals(Set.of("topo-long"), reportedTopologies(result)); } @@ -150,8 +160,10 @@ public void perTopologyTimeoutOverrideExtendsTheStaleThreshold() throws Exceptio @Test public void perTopologyTimeoutBelowGlobalKeepsTheGlobalThreshold() throws Exception { - // An override smaller than the global timeout must not shrink the effective timeout below the - // global one: effectiveWorkerTimeoutSecs takes max(global, override), matching Slot.getHbTimeoutMs. + // An override smaller than the global timeout must not shrink the effective timeout below + // the + // global one: effectiveWorkerTimeoutSecs takes max(global, override), matching + // Slot.getHbTimeoutMs. when(mockConfigUtils.readSupervisorStormConfImpl(any(), eq("topo-small"))) .thenReturn(topoConf(Config.TOPOLOGY_WORKER_TIMEOUT_SECS, 10)); @@ -163,7 +175,8 @@ public void perTopologyTimeoutBelowGlobalKeepsTheGlobalThreshold() throws Except // age 20s: stale under the 10s override, but still alive under the global 30s floor local.put("w-small", mkHeartbeat("topo-small", now - 20)); - SupervisorWorkerHeartbeats result = mkReporter().getSupervisorWorkerHeartbeatsFromLocal(local); + SupervisorWorkerHeartbeats result = mkReporter() + .getSupervisorWorkerHeartbeatsFromLocal(local); assertEquals(Set.of("topo-small"), reportedTopologies(result)); } @@ -171,7 +184,8 @@ public void perTopologyTimeoutBelowGlobalKeepsTheGlobalThreshold() throws Except @Test public void topologyConfIsReadOncePerRoundForMultipleWorkers() throws Exception { - // Multiple workers of the same topology on this supervisor must share a single conf read per round. + // Multiple workers of the same topology on this supervisor must share a single conf read + // per round. when(mockConfigUtils.readSupervisorStormConfImpl(any(), eq("topo-shared"))) .thenReturn(topoConf(Config.TOPOLOGY_WORKER_TIMEOUT_SECS, 300)); @@ -192,11 +206,14 @@ public void topologyConfIsReadOncePerRoundForMultipleWorkers() throws Exception @Test public void perTopologyTimeoutIsCappedByWorkerMaxTimeout() throws Exception { - // An override beyond the cap must clamp to WORKER_MAX_TIMEOUT_SECS, just like Nimbus does at submission. + // An override beyond the cap must clamp to WORKER_MAX_TIMEOUT_SECS, just like Nimbus does + // at submission. when(mockConfigUtils.readSupervisorStormConfImpl(any(), eq("topo-huge-stale"))) - .thenReturn(topoConf(Config.TOPOLOGY_WORKER_TIMEOUT_SECS, WORKER_MAX_TIMEOUT_SECS * 100)); + .thenReturn(topoConf(Config.TOPOLOGY_WORKER_TIMEOUT_SECS, + WORKER_MAX_TIMEOUT_SECS * 100)); when(mockConfigUtils.readSupervisorStormConfImpl(any(), eq("topo-huge-fresh"))) - .thenReturn(topoConf(Config.TOPOLOGY_WORKER_TIMEOUT_SECS, WORKER_MAX_TIMEOUT_SECS * 100)); + .thenReturn(topoConf(Config.TOPOLOGY_WORKER_TIMEOUT_SECS, + WORKER_MAX_TIMEOUT_SECS * 100)); try (Time.SimulatedTime ignored = new Time.SimulatedTime()) { Time.advanceTimeSecs(1_000_000); @@ -208,7 +225,8 @@ public void perTopologyTimeoutIsCappedByWorkerMaxTimeout() throws Exception { // just within the cap: still alive local.put("w-fresh", mkHeartbeat("topo-huge-fresh", now - WORKER_MAX_TIMEOUT_SECS + 1)); - SupervisorWorkerHeartbeats result = mkReporter().getSupervisorWorkerHeartbeatsFromLocal(local); + SupervisorWorkerHeartbeats result = mkReporter() + .getSupervisorWorkerHeartbeatsFromLocal(local); assertEquals(Set.of("topo-huge-fresh"), reportedTopologies(result)); } @@ -216,7 +234,8 @@ public void perTopologyTimeoutIsCappedByWorkerMaxTimeout() throws Exception { @Test public void unreadableTopologyConfFallsBackToGlobalTimeout() throws Exception { - // Orphaned worker dirs often outlive their topology conf; a read failure must fall back to the + // Orphaned worker dirs often outlive their topology conf; a read failure must fall back to + // the // global timeout rather than reporting a dead worker indefinitely. when(mockConfigUtils.readSupervisorStormConfImpl(any(), eq("topo-orphan-fresh"))) .thenThrow(new IOException("topology conf gone")); @@ -233,7 +252,8 @@ public void unreadableTopologyConfFallsBackToGlobalTimeout() throws Exception { // stale under the fallback global timeout -> filtered out local.put("w-stale", mkHeartbeat("topo-orphan-stale", now - WORKER_TIMEOUT_SECS - 1)); - SupervisorWorkerHeartbeats result = mkReporter().getSupervisorWorkerHeartbeatsFromLocal(local); + SupervisorWorkerHeartbeats result = mkReporter() + .getSupervisorWorkerHeartbeatsFromLocal(local); assertEquals(Set.of("topo-orphan-fresh"), reportedTopologies(result)); } diff --git a/storm-server/src/test/java/org/apache/storm/drpc/DRPCIntegrationTest.java b/storm-server/src/test/java/org/apache/storm/drpc/DRPCIntegrationTest.java index 885282eabc9..7f66c6ec409 100644 --- a/storm-server/src/test/java/org/apache/storm/drpc/DRPCIntegrationTest.java +++ b/storm-server/src/test/java/org/apache/storm/drpc/DRPCIntegrationTest.java @@ -18,9 +18,11 @@ package org.apache.storm.drpc; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + import java.util.Map; import java.util.concurrent.ConcurrentHashMap; - import org.apache.storm.ILocalDRPC; import org.apache.storm.LocalCluster; import org.apache.storm.LocalDRPC; @@ -47,12 +49,10 @@ import org.mockito.ArgumentMatchers; import org.mockito.Mockito; -import static org.junit.jupiter.api.Assertions.*; - /** * Integration and unit tests for DRPC topology patterns. * - * Ported from storm-core/test/clj/org/apache/storm/drpc_test.clj + *

      Ported from storm-core/test/clj/org/apache/storm/drpc_test.clj */ public class DRPCIntegrationTest { @@ -117,7 +117,8 @@ static class PartialCount extends BaseRichBolt implements CoordinatedBolt.Finish private final Map counts = new ConcurrentHashMap<>(); @Override - public void prepare(Map conf, TopologyContext context, OutputCollector collector) { + public void prepare(Map conf, TopologyContext context, + OutputCollector collector) { this.collector = collector; } @@ -147,7 +148,8 @@ static class CountAggregator extends BaseRichBolt implements CoordinatedBolt.Fin private final Map counts = new ConcurrentHashMap<>(); @Override - public void prepare(Map conf, TopologyContext context, OutputCollector collector) { + public void prepare(Map conf, TopologyContext context, + OutputCollector collector) { this.collector = collector; } @@ -192,7 +194,8 @@ static class EmitFinish extends BaseRichBolt implements CoordinatedBolt.Finished private OutputCollector collector; @Override - public void prepare(Map conf, TopologyContext context, OutputCollector collector) { + public void prepare(Map conf, TopologyContext context, + OutputCollector collector) { this.collector = collector; } @@ -219,7 +222,8 @@ static class FailFinishBolt extends BaseRichBolt implements CoordinatedBolt.Fini private OutputCollector collector; @Override - public void prepare(Map conf, TopologyContext context, OutputCollector collector) { + public void prepare(Map conf, TopologyContext context, + OutputCollector collector) { this.collector = collector; } diff --git a/storm-server/src/test/java/org/apache/storm/grouping/GroupingIntegrationTest.java b/storm-server/src/test/java/org/apache/storm/grouping/GroupingIntegrationTest.java index c61bb492209..07199a8e8bc 100644 --- a/storm-server/src/test/java/org/apache/storm/grouping/GroupingIntegrationTest.java +++ b/storm-server/src/test/java/org/apache/storm/grouping/GroupingIntegrationTest.java @@ -18,12 +18,13 @@ package org.apache.storm.grouping; +import static org.junit.jupiter.api.Assertions.assertTrue; + import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; - import org.apache.storm.Config; import org.apache.storm.LocalCluster; import org.apache.storm.Testing; @@ -39,21 +40,19 @@ import org.apache.storm.testing.NGrouping; import org.apache.storm.testing.TestWordBytesCounter; import org.apache.storm.testing.TestWordSpout; -import org.apache.storm.tuple.Values; import org.apache.storm.topology.BasicOutputCollector; import org.apache.storm.topology.OutputFieldsDeclarer; import org.apache.storm.topology.base.BaseBasicBolt; import org.apache.storm.tuple.Fields; import org.apache.storm.tuple.Tuple; +import org.apache.storm.tuple.Values; import org.apache.storm.utils.Utils; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.*; - /** * Tests for shuffle, field, and custom grouping behaviors. * - * Ported from storm-core/test/clj/org/apache/storm/grouping_test.clj + *

      Ported from storm-core/test/clj/org/apache/storm/grouping_test.clj */ public class GroupingIntegrationTest { @@ -111,7 +110,8 @@ public void testFieldGrouping() throws Exception { var topology = Thrift.buildTopology( Map.of("1", Thrift.prepareSpoutDetails(new TestWordSpout(true), spoutPhint)), Map.of("2", Thrift.prepareBoltDetails( - Map.of(Utils.getGlobalStreamId("1", null), Thrift.prepareFieldsGrouping(List.of("word"))), + Map.of(Utils.getGlobalStreamId("1", null), Thrift.prepareFieldsGrouping(List + .of("word"))), new TestWordBytesCounter(), boltPhint))); // Build mocked source data: repeat [bytes("a"), bytes("b")] spoutPhint*boltPhint times @@ -126,7 +126,8 @@ public void testFieldGrouping() throws Exception { CompleteTopologyParam param = new CompleteTopologyParam(); param.setMockedSources(mockedSources); - Map> results = Testing.completeTopology(cluster, topology, param); + Map> results = Testing.completeTopology(cluster, topology, + param); // Expected: for each word, counts from 1 to spoutPhint*boltPhint List> expected = new ArrayList<>(); @@ -167,7 +168,8 @@ public void testCustomGroupings() throws Exception { CompleteTopologyParam param = new CompleteTopologyParam(); param.setMockedSources(mockedSources); - Map> results = Testing.completeTopology(cluster, topology, param); + Map> results = Testing.completeTopology(cluster, topology, + param); // NGrouping(2) sends each tuple to 2 tasks assertTrue(Testing.multiseteq( diff --git a/storm-server/src/test/java/org/apache/storm/localizer/AsyncLocalizerTest.java b/storm-server/src/test/java/org/apache/storm/localizer/AsyncLocalizerTest.java index 30cfdc4f94b..3b5a6f1f594 100644 --- a/storm-server/src/test/java/org/apache/storm/localizer/AsyncLocalizerTest.java +++ b/storm-server/src/test/java/org/apache/storm/localizer/AsyncLocalizerTest.java @@ -1,17 +1,40 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.localizer; +import static org.apache.storm.blobstore.BlobStoreAclHandler.WORLD_EVERYTHING; +import static org.apache.storm.localizer.LocalizedResource.USERCACHE; +import static org.apache.storm.localizer.LocallyCachedTopologyBlob.LOCAL_MODE_JAR_VERSION; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + import com.codahale.metrics.Timer; import com.google.common.base.Joiner; import java.io.File; @@ -34,7 +57,6 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; - import org.apache.commons.io.IOUtils; import org.apache.storm.Config; import org.apache.storm.DaemonConfig; @@ -52,6 +74,7 @@ import org.apache.storm.generated.ReadableBlobMeta; import org.apache.storm.generated.SettableBlobMeta; import org.apache.storm.generated.StormTopology; +import org.apache.storm.metric.StormMetricsRegistry; import org.apache.storm.security.auth.DefaultPrincipalToLocal; import org.apache.storm.testing.InProcessZookeeper; import org.apache.storm.testing.TmpPath; @@ -66,26 +89,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.apache.storm.blobstore.BlobStoreAclHandler.WORLD_EVERYTHING; -import static org.apache.storm.localizer.LocalizedResource.USERCACHE; -import static org.apache.storm.localizer.LocallyCachedTopologyBlob.LOCAL_MODE_JAR_VERSION; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -import org.apache.storm.metric.StormMetricsRegistry; - public class AsyncLocalizerTest { private static final Logger LOG = LoggerFactory.getLogger(AsyncLocalizerTest.class); @@ -100,24 +103,29 @@ public void testRequestDownloadBaseTopologyBlobs() throws Exception { ReflectionUtils mockedReflectionUtils = mock(ReflectionUtils.class); ServerUtils mockedServerUtils = mock(ServerUtils.class); - ReflectionUtils previousReflectionUtils = ReflectionUtils.setInstance(mockedReflectionUtils); + ReflectionUtils previousReflectionUtils = ReflectionUtils + .setInstance(mockedReflectionUtils); ServerUtils previousServerUtils = ServerUtils.setInstance(mockedServerUtils); - // cannot use automatic resource management here in this try because the AsyncLocalizer depends on a config map, - // which should take the storm local dir, and that storm local dir is declared in the try-with-resources. + // cannot use automatic resource management here in this try because the AsyncLocalizer + // depends on a config map, + // which should take the storm local dir, and that storm local dir is declared in the + // try-with-resources. AsyncLocalizer victim = null; try (TmpPath stormRoot = new TmpPath(); TmpPath localizerRoot = new TmpPath()) { Map conf = new HashMap<>(); conf.put(DaemonConfig.SUPERVISOR_BLOBSTORE, ClientBlobStore.class.getName()); - conf.put(Config.STORM_PRINCIPAL_TO_LOCAL_PLUGIN, DefaultPrincipalToLocal.class.getName()); + conf.put(Config.STORM_PRINCIPAL_TO_LOCAL_PLUGIN, DefaultPrincipalToLocal.class + .getName()); conf.put(Config.STORM_CLUSTER_MODE, "distributed"); conf.put(Config.STORM_LOCAL_DIR, stormRoot.getPath()); AdvancedFSOps ops = AdvancedFSOps.make(conf); - victim = spy(new AsyncLocalizer(conf, ops, localizerRoot.getPath(), new StormMetricsRegistry())); + victim = spy(new AsyncLocalizer(conf, ops, localizerRoot.getPath(), + new StormMetricsRegistry())); final String topoId = "TOPO"; @@ -148,7 +156,8 @@ public void testRequestDownloadBaseTopologyBlobs() throws Exception { when(confBlob.fetchUnzipToTemp(any())).thenReturn(300L); when(confBlob.isUsed()).thenReturn(true); - when(mockedReflectionUtils.newInstanceImpl(ClientBlobStore.class)).thenReturn(blobStore); + when(mockedReflectionUtils.newInstanceImpl(ClientBlobStore.class)) + .thenReturn(blobStore); PortAndAssignment pna = new PortAndAssignmentImpl(port, localAssignment); Future f = victim.requestDownloadBaseTopologyBlobs(pna, null); @@ -205,14 +214,17 @@ public void testRequestDownloadTopologyBlobs() throws Exception { topoConf.put(Config.TOPOLOGY_NAME, "TOPO"); List localizedList = new ArrayList<>(); - LocalizedResource simpleLocal = new LocalizedResource(simpleKey, localizerRoot.getFile().toPath(), false, ops, conf, user, metricsRegistry); + LocalizedResource simpleLocal = new LocalizedResource(simpleKey, localizerRoot.getFile() + .toPath(), false, ops, conf, user, metricsRegistry); localizedList.add(simpleLocal); - when(mockedConfigUtils.supervisorStormDistRootImpl(conf, topoId)).thenReturn(topologyDirRoot.toString()); + when(mockedConfigUtils.supervisorStormDistRootImpl(conf, topoId)) + .thenReturn(topologyDirRoot.toString()); when(mockedConfigUtils.readSupervisorStormConfImpl(conf, topoId)).thenReturn(topoConf); - when(mockedConfigUtils.readSupervisorTopologyImpl(conf, topoId, ops)).thenReturn(constructEmptyStormTopology()); + when(mockedConfigUtils.readSupervisorTopologyImpl(conf, topoId, ops)) + .thenReturn(constructEmptyStormTopology()); - //Write the mocking backwards so the actual method is not called on the spy object + // Write the mocking backwards so the actual method is not called on the spy object doReturn(CompletableFuture.supplyAsync(() -> null)).when(victim) .requestDownloadBaseTopologyBlobs(any(), eq(null)); @@ -221,7 +233,8 @@ public void testRequestDownloadTopologyBlobs() throws Exception { doReturn(userDir.toFile()).when(victim).getLocalUserFileCacheDir(user); doReturn(localizedList).when(victim).getBlobs(any(List.class), any(), any()); - Future f = victim.requestDownloadTopologyBlobs(constructLocalAssignment(topoId, user), port, null); + Future f = victim.requestDownloadTopologyBlobs(constructLocalAssignment(topoId, + user), port, null); f.get(20, TimeUnit.SECONDS); // We should be done now... @@ -242,7 +255,6 @@ public void testRequestDownloadTopologyBlobs() throws Exception { } } - @Test public void testRequestDownloadTopologyBlobsWithLocalNameOutsideOfStormRoot() throws Exception { ConfigUtils mockedConfigUtils = mock(ConfigUtils.class); @@ -266,7 +278,8 @@ public void testRequestDownloadTopologyBlobsWithLocalNameOutsideOfStormRoot() th final Path userDir = Paths.get(stormLocal.getPath(), user); final Path topologyDirRoot = Paths.get(stormLocal.getPath(), topoId); - // the localname comes from the topology conf and tries to point outside of the topology's dist dir + // the localname comes from the topology conf and tries to point outside of the + // topology's dist dir final String escapingLocalName = Joiner.on(File.separator).join("..", "escaped.txt"); final String simpleKey = "simple"; Map> topoBlobMap = new HashMap<>(); @@ -282,15 +295,18 @@ public void testRequestDownloadTopologyBlobsWithLocalNameOutsideOfStormRoot() th topoConf.put(Config.TOPOLOGY_NAME, "TOPO"); List localizedList = new ArrayList<>(); - LocalizedResource simpleLocal = new LocalizedResource(simpleKey, localizerRoot.getFile().toPath(), false, ops, conf, user, + LocalizedResource simpleLocal = new LocalizedResource(simpleKey, localizerRoot.getFile() + .toPath(), false, ops, conf, user, metricsRegistry); localizedList.add(simpleLocal); - when(mockedConfigUtils.supervisorStormDistRootImpl(conf, topoId)).thenReturn(topologyDirRoot.toString()); + when(mockedConfigUtils.supervisorStormDistRootImpl(conf, topoId)) + .thenReturn(topologyDirRoot.toString()); when(mockedConfigUtils.readSupervisorStormConfImpl(conf, topoId)).thenReturn(topoConf); - when(mockedConfigUtils.readSupervisorTopologyImpl(conf, topoId, ops)).thenReturn(constructEmptyStormTopology()); + when(mockedConfigUtils.readSupervisorTopologyImpl(conf, topoId, ops)) + .thenReturn(constructEmptyStormTopology()); - //Write the mocking backwards so the actual method is not called on the spy object + // Write the mocking backwards so the actual method is not called on the spy object doReturn(CompletableFuture.supplyAsync(() -> null)).when(victim) .requestDownloadBaseTopologyBlobs(any(), eq(null)); @@ -299,11 +315,13 @@ public void testRequestDownloadTopologyBlobsWithLocalNameOutsideOfStormRoot() th doReturn(userDir.toFile()).when(victim).getLocalUserFileCacheDir(user); doReturn(localizedList).when(victim).getBlobs(any(List.class), any(), any()); - Future f = victim.requestDownloadTopologyBlobs(constructLocalAssignment(topoId, user), port, null); + Future f = victim.requestDownloadTopologyBlobs(constructLocalAssignment(topoId, + user), port, null); assertThrows(ExecutionException.class, () -> f.get(20, TimeUnit.SECONDS)); // nothing was created outside of the topology's dist dir - assertFalse(Files.exists(topologyDirRoot.getParent().resolve("escaped.txt"), LinkOption.NOFOLLOW_LINKS)); + assertFalse(Files.exists(topologyDirRoot.getParent().resolve("escaped.txt"), + LinkOption.NOFOLLOW_LINKS)); } finally { ConfigUtils.setInstance(previousConfigUtils); @@ -359,12 +377,15 @@ public void testRequestDownloadTopologyBlobsLocalMode() throws Exception { topoConf.put(Config.TOPOLOGY_NAME, "TOPO"); List localizedList = new ArrayList<>(); - LocalizedResource simpleLocal = new LocalizedResource(simpleKey, localizerRoot.getFile().toPath(), false, ops, conf, user, metricsRegistry); + LocalizedResource simpleLocal = new LocalizedResource(simpleKey, localizerRoot.getFile() + .toPath(), false, ops, conf, user, metricsRegistry); localizedList.add(simpleLocal); - when(mockedConfigUtils.supervisorStormDistRootImpl(conf, topoId)).thenReturn(stormRoot.toString()); + when(mockedConfigUtils.supervisorStormDistRootImpl(conf, topoId)).thenReturn(stormRoot + .toString()); when(mockedConfigUtils.readSupervisorStormConfImpl(conf, topoId)).thenReturn(topoConf); - when(mockedConfigUtils.readSupervisorTopologyImpl(conf, topoId, ops)).thenReturn(constructEmptyStormTopology()); + when(mockedConfigUtils.readSupervisorTopologyImpl(conf, topoId, ops)) + .thenReturn(constructEmptyStormTopology()); doReturn(mockBlobStore).when(victim).getClientBlobStore(); doReturn(userDir.toFile()).when(victim).getLocalUserFileCacheDir(user); @@ -373,9 +394,11 @@ public void testRequestDownloadTopologyBlobsLocalMode() throws Exception { ReadableBlobMeta blobMeta = new ReadableBlobMeta(); blobMeta.set_version(1); doReturn(blobMeta).when(mockBlobStore).getBlobMeta(any()); - when(mockBlobStore.getBlob(any())).thenAnswer(invocation -> new TestInputStreamWithMeta(LOCAL_MODE_JAR_VERSION)); + when(mockBlobStore.getBlob(any())) + .thenAnswer(invocation -> new TestInputStreamWithMeta(LOCAL_MODE_JAR_VERSION)); - Future f = victim.requestDownloadTopologyBlobs(constructLocalAssignment(topoId, user), port, null); + Future f = victim.requestDownloadTopologyBlobs(constructLocalAssignment(topoId, + user), port, null); f.get(20, TimeUnit.SECONDS); verify(victim).getLocalUserFileCacheDir(user); @@ -385,7 +408,9 @@ public void testRequestDownloadTopologyBlobsLocalMode() throws Exception { verify(victim).getBlobs(any(List.class), any(), any()); // make sure resources directory after blob version commit is created. - Path extractionDir = stormRoot.resolve(LocallyCachedTopologyBlob.TopologyBlobType.TOPO_JAR.getExtractionDir()); + Path extractionDir = stormRoot + .resolve(LocallyCachedTopologyBlob.TopologyBlobType.TOPO_JAR + .getExtractionDir()); assertTrue(ops.fileExists(extractionDir)); } finally { @@ -404,7 +429,8 @@ private LocalAssignment constructLocalAssignment(String topoId, String owner) { ); } - private LocalAssignment constructLocalAssignment(String topoId, String owner, List executorInfos) { + private LocalAssignment constructLocalAssignment(String topoId, String owner, + List executorInfos) { LocalAssignment assignment = new LocalAssignment(topoId, executorInfos); assignment.set_owner(owner); return assignment; @@ -427,11 +453,13 @@ private String constructUserCacheDir(String base, String user) { } private String constructExpectedFilesDir(String base, String user) { - return joinPath(constructUserCacheDir(base, user), LocalizedResource.FILECACHE, LocalizedResource.FILESDIR); + return joinPath(constructUserCacheDir(base, user), LocalizedResource.FILECACHE, + LocalizedResource.FILESDIR); } private String constructExpectedArchivesDir(String base, String user) { - return joinPath(constructUserCacheDir(base, user), LocalizedResource.FILECACHE, LocalizedResource.ARCHIVESDIR); + return joinPath(constructUserCacheDir(base, user), LocalizedResource.FILECACHE, + LocalizedResource.ARCHIVESDIR); } @Test @@ -442,17 +470,19 @@ public void testDirPaths() throws Exception { String expectedDir = constructUserCacheDir(tmp.getPath(), user1); assertEquals(expectedDir, - localizer.getLocalUserDir(user1).toString(), "get local user dir doesn't return right value"); + localizer.getLocalUserDir(user1) + .toString(), "get local user dir doesn't return right value"); String expectedFileDir = joinPath(expectedDir, LocalizedResource.FILECACHE); assertEquals(expectedFileDir, - localizer.getLocalUserFileCacheDir(user1).toString(), "get local user file dir doesn't return right value"); + localizer.getLocalUserFileCacheDir(user1) + .toString(), "get local user file dir doesn't return right value"); } } @Test public void testReconstruct() throws Exception { - try (TmpPath tmp = new TmpPath()){ + try (TmpPath tmp = new TmpPath()) { Map conf = new HashMap<>(); String expectedFileDir1 = constructExpectedFilesDir(tmp.getPath(), user1); @@ -468,13 +498,19 @@ public void testReconstruct() throws Exception { String archive1 = "archive1"; String archive2 = "archive2"; - File user1file1 = new File(expectedFileDir1, key1 + LocalizedResource.CURRENT_BLOB_SUFFIX); - File user1file2 = new File(expectedFileDir1, key2 + LocalizedResource.CURRENT_BLOB_SUFFIX); - File user2file3 = new File(expectedFileDir2, key3 + LocalizedResource.CURRENT_BLOB_SUFFIX); - File user2file4 = new File(expectedFileDir2, key4 + LocalizedResource.CURRENT_BLOB_SUFFIX); - - File user1archive1 = new File(expectedArchiveDir1, archive1 + LocalizedResource.CURRENT_BLOB_SUFFIX); - File user2archive2 = new File(expectedArchiveDir2, archive2 + LocalizedResource.CURRENT_BLOB_SUFFIX); + File user1file1 = new File(expectedFileDir1, key1 + + LocalizedResource.CURRENT_BLOB_SUFFIX); + File user1file2 = new File(expectedFileDir1, key2 + + LocalizedResource.CURRENT_BLOB_SUFFIX); + File user2file3 = new File(expectedFileDir2, key3 + + LocalizedResource.CURRENT_BLOB_SUFFIX); + File user2file4 = new File(expectedFileDir2, key4 + + LocalizedResource.CURRENT_BLOB_SUFFIX); + + File user1archive1 = new File(expectedArchiveDir1, archive1 + + LocalizedResource.CURRENT_BLOB_SUFFIX); + File user2archive2 = new File(expectedArchiveDir2, archive2 + + LocalizedResource.CURRENT_BLOB_SUFFIX); File user1archive1file = new File(user1archive1, "file1"); File user2archive2file = new File(user2archive2, "file2"); @@ -495,28 +531,36 @@ public void testReconstruct() throws Exception { ArrayList arrUser1Keys = new ArrayList<>(); arrUser1Keys.add(new LocalResource(key1, false, false)); arrUser1Keys.add(new LocalResource(archive1, true, false)); - LocalAssignment topo1 = constructLocalAssignment("topo1", user1, Collections.emptyList()); + LocalAssignment topo1 = constructLocalAssignment("topo1", user1, Collections + .emptyList()); localizer.addReferences(arrUser1Keys, new PortAndAssignmentImpl(1, topo1), null); - ConcurrentMap lrsrcFiles = localizer.getUserFiles().get(user1); - ConcurrentMap lrsrcArchives = localizer.getUserArchives().get(user1); - assertEquals(3, lrsrcFiles.size() + lrsrcArchives.size(), "local resource set size wrong"); + ConcurrentMap lrsrcFiles = localizer.getUserFiles() + .get(user1); + ConcurrentMap lrsrcArchives = localizer.getUserArchives() + .get(user1); + assertEquals(3, lrsrcFiles.size() + lrsrcArchives.size(), + "local resource set size wrong"); LocalizedResource key1rsrc = lrsrcFiles.get(key1); assertNotNull(key1rsrc, "Local resource doesn't exist but should"); assertEquals(key1, key1rsrc.getKey(), "key doesn't match"); assertTrue(key1rsrc.isUsed(), "references doesn't match " + key1rsrc.getDependencies()); LocalizedResource key2rsrc = lrsrcFiles.get(key2); assertNotNull(key2rsrc, "Local resource doesn't exist but should"); - assertEquals(key2, key2rsrc.getKey(),"key doesn't match"); + assertEquals(key2, key2rsrc.getKey(), "key doesn't match"); assertFalse(key2rsrc.isUsed(), "refcount doesn't match " + key2rsrc.getDependencies()); LocalizedResource archive1rsrc = lrsrcArchives.get(archive1); assertNotNull(archive1rsrc, "Local resource doesn't exist but should"); assertEquals(archive1, archive1rsrc.getKey(), "key doesn't match"); - assertTrue(archive1rsrc.isUsed(), "refcount doesn't match " + archive1rsrc.getDependencies()); - - ConcurrentMap lrsrcFiles2 = localizer.getUserFiles().get(user2); - ConcurrentMap lrsrcArchives2 = localizer.getUserArchives().get(user2); - assertEquals(3, lrsrcFiles2.size() + lrsrcArchives2.size(), "local resource set size wrong"); + assertTrue(archive1rsrc.isUsed(), "refcount doesn't match " + archive1rsrc + .getDependencies()); + + ConcurrentMap lrsrcFiles2 = localizer.getUserFiles() + .get(user2); + ConcurrentMap lrsrcArchives2 = localizer.getUserArchives() + .get(user2); + assertEquals(3, lrsrcFiles2.size() + lrsrcArchives2.size(), + "local resource set size wrong"); LocalizedResource key3rsrc = lrsrcFiles2.get(key3); assertNotNull(key3rsrc, "Local resource doesn't exist but should"); assertEquals(key3, key3rsrc.getKey(), "key doesn't match"); @@ -528,13 +572,15 @@ public void testReconstruct() throws Exception { LocalizedResource archive2rsrc = lrsrcArchives2.get(archive2); assertNotNull(archive2rsrc, "Local resource doesn't exist but should"); assertEquals(archive2, archive2rsrc.getKey(), "key doesn't match"); - assertFalse(archive2rsrc.isUsed(), "refcount doesn't match " + archive2rsrc.getDependencies()); + assertFalse(archive2rsrc.isUsed(), "refcount doesn't match " + archive2rsrc + .getDependencies()); } } @Test public void testArchivesTgz() throws Exception { - testArchives(getFileFromResource(joinPath("localizer", "localtestwithsymlink.tgz")), true, 21344); + testArchives(getFileFromResource(joinPath("localizer", "localtestwithsymlink.tgz")), true, + 21344); } @Test @@ -544,17 +590,20 @@ public void testArchivesZip() throws Exception { @Test public void testArchivesTarGz() throws Exception { - testArchives(getFileFromResource(joinPath("localizer", "localtestwithsymlink.tar.gz")), true, 21344); + testArchives(getFileFromResource(joinPath("localizer", "localtestwithsymlink.tar.gz")), + true, 21344); } @Test public void testArchivesTar() throws Exception { - testArchives(getFileFromResource(joinPath("localizer", "localtestwithsymlink.tar")), true, 21344); + testArchives(getFileFromResource(joinPath("localizer", "localtestwithsymlink.tar")), true, + 21344); } @Test public void testArchivesJar() throws Exception { - testArchives(getFileFromResource(joinPath("localizer", "localtestwithsymlink.jar")), false, 21416); + testArchives(getFileFromResource(joinPath("localizer", "localtestwithsymlink.jar")), false, + 21416); } private File getFileFromResource(String archivePath) { @@ -565,7 +614,8 @@ private File getFileFromResource(String archivePath) { // archive passed in must contain symlink named tmptestsymlink if not a zip file public void testArchives(File archiveFile, boolean supportSymlinks, int size) throws Exception { if (Utils.isOnWindows()) { - // Windows should set this to false cause symlink in compressed file doesn't work properly. + // Windows should set this to false cause symlink in compressed file doesn't work + // properly. supportSymlinks = false; } try (Time.SimulatedTime ignored = new Time.SimulatedTime(); TmpPath tmp = new TmpPath()) { @@ -594,15 +644,18 @@ public void testArchives(File archiveFile, boolean supportSymlinks, int size) th Time.advanceTime(10); File user1Dir = localizer.getLocalUserFileCacheDir(user1); assertTrue(user1Dir.mkdirs(), "failed to create user dir"); - LocalAssignment topo1Assignment = constructLocalAssignment(topo1, user1, Collections.emptyList()); + LocalAssignment topo1Assignment = constructLocalAssignment(topo1, user1, Collections + .emptyList()); PortAndAssignment topo1Pna = new PortAndAssignmentImpl(1, topo1Assignment); - LocalizedResource lrsrc = localizer.getBlob(new LocalResource(key1, true, false), topo1Pna, null); + LocalizedResource lrsrc = localizer.getBlob(new LocalResource(key1, true, false), + topo1Pna, null); Time.advanceTime(10); long timeAfter = Time.currentTimeMillis(); Time.advanceTime(10); String expectedUserDir = joinPath(tmp.getPath(), USERCACHE, user1); - String expectedFileDir = joinPath(expectedUserDir, LocalizedResource.FILECACHE, LocalizedResource.ARCHIVESDIR); + String expectedFileDir = joinPath(expectedUserDir, LocalizedResource.FILECACHE, + LocalizedResource.ARCHIVESDIR); assertTrue(new File(expectedFileDir).exists(), "user filecache dir not created"); File keyFile = new File(expectedFileDir, key1 + ".0"); assertTrue(keyFile.exists(), "blob not created " + keyFile); @@ -611,18 +664,22 @@ public void testArchives(File archiveFile, boolean supportSymlinks, int size) th if (supportSymlinks) { assertTrue(Files.isSymbolicLink( - symlinkFile.toPath()), "blob uncompressed doesn't contain symlink"); + symlinkFile + .toPath()), "blob uncompressed doesn't contain symlink"); } else { assertTrue(symlinkFile.exists(), "blob symlink file doesn't exist"); } - ConcurrentMap lrsrcSet = localizer.getUserArchives().get(user1); + ConcurrentMap lrsrcSet = localizer.getUserArchives() + .get(user1); assertEquals(1, lrsrcSet.size(), "local resource set size wrong"); LocalizedResource key1rsrc = lrsrcSet.get(key1); assertNotNull(key1rsrc, "Local resource doesn't exist but should"); assertEquals(key1, key1rsrc.getKey(), "key doesn't match"); - assertEquals(true, key1rsrc.isUsed(), "refcount doesn't match " + key1rsrc.getDependencies()); - assertEquals(keyFile.toPath(), key1rsrc.getFilePathWithVersion(), "file path doesn't match"); + assertEquals(true, key1rsrc.isUsed(), "refcount doesn't match " + key1rsrc + .getDependencies()); + assertEquals(keyFile.toPath(), key1rsrc.getFilePathWithVersion(), + "file path doesn't match"); assertEquals(size, key1rsrc.getSizeOnDisk(), "size doesn't match"); assertTrue((key1rsrc.getLastUsed() >= timeBefore && key1rsrc .getLastUsed() <= timeAfter), "timestamp not within range"); @@ -638,7 +695,8 @@ public void testArchives(File archiveFile, boolean supportSymlinks, int size) th assertEquals(1, lrsrcSet.size(), "local resource set size wrong"); key1rsrc = lrsrcSet.get(key1); assertNotNull(key1rsrc, "Local resource doesn't exist but should"); - assertEquals(false, key1rsrc.isUsed(), "refcount doesn't match " + key1rsrc.getDependencies()); + assertEquals(false, key1rsrc.isUsed(), "refcount doesn't match " + key1rsrc + .getDependencies()); assertTrue((key1rsrc.getLastUsed() >= timeBefore && key1rsrc .getLastUsed() <= timeAfter), "timestamp not within range"); @@ -678,17 +736,21 @@ public void testBasic() throws Exception { File user1Dir = localizer.getLocalUserFileCacheDir(user1); assertTrue(user1Dir.mkdirs(), "failed to create user dir"); Time.advanceTime(10); - LocalAssignment topo1Assignment = constructLocalAssignment(topo1, user1, Collections.emptyList()); + LocalAssignment topo1Assignment = constructLocalAssignment(topo1, user1, Collections + .emptyList()); PortAndAssignment topo1Pna = new PortAndAssignmentImpl(1, topo1Assignment); - LocalizedResource lrsrc = localizer.getBlob(new LocalResource(key1, false, false), topo1Pna, null); + LocalizedResource lrsrc = localizer.getBlob(new LocalResource(key1, false, false), + topo1Pna, null); long timeAfter = Time.currentTimeMillis(); Time.advanceTime(10); String expectedUserDir = joinPath(tmp.getPath(), USERCACHE, user1); - String expectedFileDir = joinPath(expectedUserDir, LocalizedResource.FILECACHE, LocalizedResource.FILESDIR); + String expectedFileDir = joinPath(expectedUserDir, LocalizedResource.FILECACHE, + LocalizedResource.FILESDIR); assertTrue(new File(expectedFileDir).exists(), "user filecache dir not created"); File keyFile = new File(expectedFileDir, key1 + ".current"); - File keyFileCurrentSymlink = new File(expectedFileDir, key1 + LocalizedResource.CURRENT_BLOB_SUFFIX); + File keyFileCurrentSymlink = new File(expectedFileDir, key1 + + LocalizedResource.CURRENT_BLOB_SUFFIX); assertTrue(keyFileCurrentSymlink.exists(), "blob not created"); @@ -697,8 +759,10 @@ public void testBasic() throws Exception { LocalizedResource key1rsrc = lrsrcSet.get(key1); assertNotNull(key1rsrc, "Local resource doesn't exist but should"); assertEquals(key1, key1rsrc.getKey(), "key doesn't match"); - assertEquals(true, key1rsrc.isUsed(), "refcount doesn't match " + key1rsrc.getDependencies()); - assertEquals(keyFile.toPath(), key1rsrc.getCurrentSymlinkPath(), "file path doesn't match"); + assertEquals(true, key1rsrc.isUsed(), "refcount doesn't match " + key1rsrc + .getDependencies()); + assertEquals(keyFile.toPath(), key1rsrc.getCurrentSymlinkPath(), + "file path doesn't match"); assertEquals(34, key1rsrc.getSizeOnDisk(), "size doesn't match"); assertTrue((key1rsrc.getLastUsed() >= timeBefore && key1rsrc .getLastUsed() <= timeAfter), "timestamp not within range"); @@ -714,9 +778,12 @@ public void testBasic() throws Exception { assertEquals(1, lrsrcSet.size(), "local resource set size wrong"); key1rsrc = lrsrcSet.get(key1); assertNotNull(key1rsrc, "Local resource doesn't exist but should"); - assertEquals(false, key1rsrc.isUsed(), "refcount doesn't match " + key1rsrc.getDependencies()); - assertTrue((key1rsrc.getLastUsed() >= timeBefore && key1rsrc.getLastUsed() <= timeAfter), - "timestamp not within range " + timeBefore + " " + key1rsrc.getLastUsed() + " " + timeAfter); + assertEquals(false, key1rsrc.isUsed(), "refcount doesn't match " + key1rsrc + .getDependencies()); + assertTrue((key1rsrc.getLastUsed() >= timeBefore && key1rsrc + .getLastUsed() <= timeAfter), + "timestamp not within range " + timeBefore + " " + key1rsrc.getLastUsed() + " " + + timeAfter); // should remove the blob since cache size set really small localizer.cleanup(); @@ -757,7 +824,8 @@ public void testMultipleKeysOneUser() throws Exception { File user1Dir = localizer.getLocalUserFileCacheDir(user1); assertTrue(user1Dir.mkdirs(), "failed to create user dir"); - LocalAssignment topo1Assignment = constructLocalAssignment(topo1, user1, Collections.emptyList()); + LocalAssignment topo1Assignment = constructLocalAssignment(topo1, user1, Collections + .emptyList()); PortAndAssignment topo1Pna = new PortAndAssignmentImpl(1, topo1Assignment); List lrsrcs = localizer.getBlobs(keys, topo1Pna, null); LocalizedResource lrsrc = lrsrcs.get(0); @@ -801,8 +869,9 @@ public void testMultipleKeysOneUser() throws Exception { lrsrc = localizer.getBlob(new LocalResource(key1, false, false), topo1Pna, null); LOG.info("Got Blob..."); assertTrue((lrsrc.getLastUsed() >= timeBefore && lrsrc.getLastUsed() <= timeAfter), - "timestamp not within range " + timeBefore + " <= " + lrsrc.getLastUsed() + " <= " + timeAfter); - //Resets the last access time for key1 + "timestamp not within range " + timeBefore + " <= " + lrsrc.getLastUsed() + " <= " + + timeAfter); + // Resets the last access time for key1 localizer.removeBlobReference(lrsrc.getKey(), topo1Pna, false); // should remove the second blob first @@ -838,7 +907,8 @@ public void testFailAcls() { try (TmpPath tmp = new TmpPath()) { Map conf = new HashMap<>(); // set clean time really high so doesn't kick in - conf.put(DaemonConfig.SUPERVISOR_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS, 60 * 60 * 1000); + conf.put(DaemonConfig.SUPERVISOR_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS, + 60 * 60 * 1000); // enable blobstore acl validation conf.put(Config.STORM_BLOBSTORE_ACL_VALIDATION_ENABLED, true); @@ -848,7 +918,8 @@ public void testFailAcls() { ReadableBlobMeta rbm = new ReadableBlobMeta(); // set acl so user doesn't have read access - AccessControl acl = new AccessControl(AccessControlType.USER, BlobStoreAclHandler.ADMIN); + AccessControl acl = new AccessControl(AccessControlType.USER, + BlobStoreAclHandler.ADMIN); acl.set_name(user1); rbm.set_settable(new SettableBlobMeta(Collections.singletonList(acl))); when(mockBlobStore.getBlobMeta(anyString())).thenReturn(rbm); @@ -856,7 +927,8 @@ public void testFailAcls() { File user1Dir = localizer.getLocalUserFileCacheDir(user1); assertTrue(user1Dir.mkdirs(), "failed to create user dir"); - LocalAssignment topo1Assignment = constructLocalAssignment(topo1, user1, Collections.emptyList()); + LocalAssignment topo1Assignment = constructLocalAssignment(topo1, user1, Collections + .emptyList()); PortAndAssignment topo1Pna = new PortAndAssignmentImpl(1, topo1Assignment); // This should throw AuthorizationException because auth failed localizer.getBlob(new LocalResource(key1, false, false), topo1Pna, null); @@ -884,7 +956,7 @@ public void testKeyNotFoundException() throws Exception { @Test public void testMultipleUsers() throws Exception { - try (TmpPath tmp = new TmpPath()){ + try (TmpPath tmp = new TmpPath()) { Map conf = new HashMap<>(); // set clean time really high so doesn't kick in conf.put(DaemonConfig.SUPERVISOR_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS, 60 * 60 * 1000); @@ -902,7 +974,8 @@ public void testMultipleUsers() throws Exception { ReadableBlobMeta rbm = new ReadableBlobMeta(); rbm.set_settable(new SettableBlobMeta(WORLD_EVERYTHING)); when(mockBlobStore.getBlobMeta(anyString())).thenReturn(rbm); - //thenReturn always returns the same object, which is already consumed by the time User3 tries to getBlob! + // thenReturn always returns the same object, which is already consumed by the time + // User3 tries to getBlob! when(mockBlobStore.getBlob(key1)).thenAnswer((i) -> new TestInputStreamWithMeta(1)); when(mockBlobStore.getBlob(key2)).thenReturn(new TestInputStreamWithMeta(1)); when(mockBlobStore.getBlob(key3)).thenReturn(new TestInputStreamWithMeta(1)); @@ -914,53 +987,70 @@ public void testMultipleUsers() throws Exception { File user3Dir = localizer.getLocalUserFileCacheDir(user3); assertTrue(user3Dir.mkdirs(), "failed to create user dir"); - LocalAssignment topo1Assignment = constructLocalAssignment(topo1, user1, Collections.emptyList()); + LocalAssignment topo1Assignment = constructLocalAssignment(topo1, user1, Collections + .emptyList()); PortAndAssignment topo1Pna = new PortAndAssignmentImpl(1, topo1Assignment); - LocalizedResource lrsrc = localizer.getBlob(new LocalResource(key1, false, false), topo1Pna, null); + LocalizedResource lrsrc = localizer.getBlob(new LocalResource(key1, false, false), + topo1Pna, null); - LocalAssignment topo2Assignment = constructLocalAssignment(topo2, user2, Collections.emptyList()); + LocalAssignment topo2Assignment = constructLocalAssignment(topo2, user2, Collections + .emptyList()); PortAndAssignment topo2Pna = new PortAndAssignmentImpl(2, topo2Assignment); - LocalizedResource lrsrc2 = localizer.getBlob(new LocalResource(key2, false, false), topo2Pna, null); + LocalizedResource lrsrc2 = localizer.getBlob(new LocalResource(key2, false, false), + topo2Pna, null); - LocalAssignment topo3Assignment = constructLocalAssignment(topo3, user3, Collections.emptyList()); + LocalAssignment topo3Assignment = constructLocalAssignment(topo3, user3, Collections + .emptyList()); PortAndAssignment topo3Pna = new PortAndAssignmentImpl(3, topo3Assignment); - LocalizedResource lrsrc3 = localizer.getBlob(new LocalResource(key3, false, false), topo3Pna, null); + LocalizedResource lrsrc3 = localizer.getBlob(new LocalResource(key3, false, false), + topo3Pna, null); // make sure we support different user reading same blob - LocalizedResource lrsrc1_user3 = localizer.getBlob(new LocalResource(key1, false, false), topo3Pna, null); + LocalizedResource lrsrc1_user3 = localizer.getBlob(new LocalResource(key1, false, + false), topo3Pna, null); String expectedUserDir1 = joinPath(tmp.getPath(), USERCACHE, user1); - String expectedFileDirUser1 = joinPath(expectedUserDir1, LocalizedResource.FILECACHE, LocalizedResource.FILESDIR); + String expectedFileDirUser1 = joinPath(expectedUserDir1, LocalizedResource.FILECACHE, + LocalizedResource.FILESDIR); String expectedFileDirUser2 = joinPath(tmp.getPath(), USERCACHE, user2, LocalizedResource.FILECACHE, LocalizedResource.FILESDIR); String expectedFileDirUser3 = joinPath(tmp.getPath(), USERCACHE, user3, LocalizedResource.FILECACHE, LocalizedResource.FILESDIR); - assertTrue(new File(expectedFileDirUser1).exists(), "user filecache dir user1 not created"); - assertTrue(new File(expectedFileDirUser2).exists(), "user filecache dir user2 not created"); - assertTrue(new File(expectedFileDirUser3).exists(), "user filecache dir user3 not created"); - - File keyFile = new File(expectedFileDirUser1, key1 + LocalizedResource.CURRENT_BLOB_SUFFIX); - File keyFile2 = new File(expectedFileDirUser2, key2 + LocalizedResource.CURRENT_BLOB_SUFFIX); - File keyFile3 = new File(expectedFileDirUser3, key3 + LocalizedResource.CURRENT_BLOB_SUFFIX); - File keyFile1user3 = new File(expectedFileDirUser3, key1 + LocalizedResource.CURRENT_BLOB_SUFFIX); + assertTrue(new File(expectedFileDirUser1).exists(), + "user filecache dir user1 not created"); + assertTrue(new File(expectedFileDirUser2).exists(), + "user filecache dir user2 not created"); + assertTrue(new File(expectedFileDirUser3).exists(), + "user filecache dir user3 not created"); + + File keyFile = new File(expectedFileDirUser1, key1 + + LocalizedResource.CURRENT_BLOB_SUFFIX); + File keyFile2 = new File(expectedFileDirUser2, key2 + + LocalizedResource.CURRENT_BLOB_SUFFIX); + File keyFile3 = new File(expectedFileDirUser3, key3 + + LocalizedResource.CURRENT_BLOB_SUFFIX); + File keyFile1user3 = new File(expectedFileDirUser3, key1 + + LocalizedResource.CURRENT_BLOB_SUFFIX); assertTrue(keyFile.exists(), "blob not created"); assertTrue(keyFile2.exists(), "blob not created"); assertTrue(keyFile3.exists(), "blob not created"); assertTrue(keyFile1user3.exists(), "blob not created"); - //Should assert file size + // Should assert file size assertEquals(34, lrsrc.getSizeOnDisk(), "size doesn't match"); assertEquals(34, lrsrc2.getSizeOnDisk(), "size doesn't match"); assertEquals(34, lrsrc3.getSizeOnDisk(), "size doesn't match"); - //This was 0 byte in test + // This was 0 byte in test assertEquals(34, lrsrc1_user3.getSizeOnDisk(), "size doesn't match"); ConcurrentMap lrsrcSet = localizer.getUserFiles().get(user1); assertEquals(1, lrsrcSet.size(), "local resource set size wrong"); - ConcurrentMap lrsrcSet2 = localizer.getUserFiles().get(user2); + ConcurrentMap lrsrcSet2 = localizer.getUserFiles() + .get(user2); assertEquals(1, lrsrcSet2.size(), "local resource set size wrong"); - ConcurrentMap lrsrcSet3 = localizer.getUserFiles().get(user3); + ConcurrentMap lrsrcSet3 = localizer.getUserFiles() + .get(user3); assertEquals(2, lrsrcSet3.size(), "local resource set size wrong"); localizer.removeBlobReference(lrsrc.getKey(), topo1Pna, false); @@ -1001,33 +1091,43 @@ public void testUpdate() throws Exception { File user1Dir = localizer.getLocalUserFileCacheDir(user1); assertTrue(user1Dir.mkdirs(), "failed to create user dir"); - LocalAssignment topo1Assignment = constructLocalAssignment(topo1, user1, Collections.emptyList()); + LocalAssignment topo1Assignment = constructLocalAssignment(topo1, user1, Collections + .emptyList()); PortAndAssignment topo1Pna = new PortAndAssignmentImpl(1, topo1Assignment); - LocalizedResource lrsrc = localizer.getBlob(new LocalResource(key1, false, false), topo1Pna, null); + LocalizedResource lrsrc = localizer.getBlob(new LocalResource(key1, false, false), + topo1Pna, null); String expectedUserDir = joinPath(tmp.getPath(), USERCACHE, user1); - String expectedFileDir = joinPath(expectedUserDir, LocalizedResource.FILECACHE, LocalizedResource.FILESDIR); + String expectedFileDir = joinPath(expectedUserDir, LocalizedResource.FILECACHE, + LocalizedResource.FILESDIR); assertTrue(new File(expectedFileDir).exists(), "user filecache dir not created"); Path keyVersionFile = Paths.get(expectedFileDir, key1 + ".version"); - File keyFileCurrentSymlink = new File(expectedFileDir, key1 + LocalizedResource.CURRENT_BLOB_SUFFIX); + File keyFileCurrentSymlink = new File(expectedFileDir, key1 + + LocalizedResource.CURRENT_BLOB_SUFFIX); assertTrue(keyFileCurrentSymlink.exists(), "blob not created"); - File versionFile = new File(expectedFileDir, key1 + LocalizedResource.BLOB_VERSION_SUFFIX); + File versionFile = new File(expectedFileDir, key1 + + LocalizedResource.BLOB_VERSION_SUFFIX); assertTrue(versionFile.exists(), "blob version file not created"); - assertEquals(1, LocalizedResource.localVersionOfBlob(keyVersionFile), "blob version not correct"); + assertEquals(1, LocalizedResource.localVersionOfBlob(keyVersionFile), + "blob version not correct"); ConcurrentMap lrsrcSet = localizer.getUserFiles().get(user1); assertEquals(1, lrsrcSet.size(), "local resource set size wrong"); - // test another topology getting blob with updated version - it should update version now + // test another topology getting blob with updated version - it should update version + // now rbm.set_version(2); when(mockBlobStore.getBlob(key1)).thenReturn(new TestInputStreamWithMeta(2)); - LocalAssignment topo2Assignment = constructLocalAssignment(topo2, user1, Collections.emptyList()); + LocalAssignment topo2Assignment = constructLocalAssignment(topo2, user1, Collections + .emptyList()); PortAndAssignment topo2Pna = new PortAndAssignmentImpl(1, topo2Assignment); localizer.getBlob(new LocalResource(key1, false, false), topo2Pna, null); assertTrue(versionFile.exists(), "blob version file not created"); - assertEquals(2, LocalizedResource.localVersionOfBlob(keyVersionFile), "blob version not correct"); - assertTrue(new File(expectedFileDir, key1 + ".2").exists(), "blob file with version 2 not created"); + assertEquals(2, LocalizedResource.localVersionOfBlob(keyVersionFile), + "blob version not correct"); + assertTrue(new File(expectedFileDir, key1 + ".2").exists(), + "blob file with version 2 not created"); // now test regular updateBlob rbm.set_version(3); @@ -1037,8 +1137,10 @@ public void testUpdate() throws Exception { arr.add(new LocalResource(key1, false, false)); localizer.updateBlobs(); assertTrue(versionFile.exists(), "blob version file not created"); - assertEquals(3, LocalizedResource.localVersionOfBlob(keyVersionFile), "blob version not correct"); - assertTrue(new File(expectedFileDir, key1 + ".3").exists(), "blob file with version 3 not created"); + assertEquals(3, LocalizedResource.localVersionOfBlob(keyVersionFile), + "blob version not correct"); + assertTrue(new File(expectedFileDir, key1 + ".3").exists(), + "blob file with version 3 not created"); } } @@ -1064,20 +1166,24 @@ protected ClientBlobStore getClientBlobStore() { return mockBlobStore; } - synchronized void addReferences(List localresource, PortAndAssignment pna, BlobChangingCallback cb) { + synchronized void addReferences(List localresource, PortAndAssignment pna, + BlobChangingCallback cb) { String user = pna.getOwner(); for (LocalResource blob : localresource) { - ConcurrentMap lrsrcSet = blob.shouldUncompress() ? userArchives.get(user) : userFiles.get(user); + ConcurrentMap lrsrcSet = blob.shouldUncompress() + ? userArchives.get(user) : userFiles.get(user); if (lrsrcSet != null) { LocalizedResource lrsrc = lrsrcSet.get(blob.getBlobName()); if (lrsrc != null) { lrsrc.addReference(pna, blob.needsCallback() ? cb : null); LOG.debug("added reference for topo: {} key: {}", pna, blob); } else { - LOG.warn("trying to add reference to non-existent blob, key: {} topo: {}", blob, pna); + LOG.warn("trying to add reference to non-existent blob, key: {} topo: {}", + blob, pna); } } else { - LOG.warn("trying to add reference to non-existent local resource set, user: {} topo: {}", user, pna); + LOG.warn("trying to add reference to non-existent local resource set, user: " + + "{} topo: {}", user, pna); } } } @@ -1099,14 +1205,16 @@ ConcurrentHashMap> getUserA * This function either returns the blob in the existing cache or if it doesn't exist in the * cache, it will download the blob and will block until the download is complete. */ - LocalizedResource getBlob(LocalResource localResource, PortAndAssignment pna, BlobChangingCallback cb) + LocalizedResource getBlob(LocalResource localResource, PortAndAssignment pna, + BlobChangingCallback cb) throws AuthorizationException, KeyNotFoundException, IOException { ArrayList arr = new ArrayList<>(); arr.add(localResource); List results = getBlobs(arr, pna, cb); if (results.isEmpty() || results.size() != 1) { - throw new IOException("Unknown error getting blob: " + localResource + ", for user: " + pna.getOwner() + - ", topo: " + pna); + throw new IOException("Unknown error getting blob: " + localResource + + ", for user: " + pna.getOwner() + + ", topo: " + pna); } return results.get(0); } diff --git a/storm-server/src/test/java/org/apache/storm/localizer/LocalizedResourceRetentionSetTest.java b/storm-server/src/test/java/org/apache/storm/localizer/LocalizedResourceRetentionSetTest.java index 39fd6fc7f9a..d349d0ea80a 100644 --- a/storm-server/src/test/java/org/apache/storm/localizer/LocalizedResourceRetentionSetTest.java +++ b/storm-server/src/test/java/org/apache/storm/localizer/LocalizedResourceRetentionSetTest.java @@ -1,17 +1,30 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.localizer; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; import java.nio.file.Paths; import java.util.ArrayList; @@ -27,34 +40,29 @@ import org.apache.storm.generated.LocalAssignment; import org.apache.storm.generated.ReadableBlobMeta; import org.apache.storm.generated.SettableBlobMeta; +import org.apache.storm.metric.StormMetricsRegistry; import org.apache.storm.utils.EquivalenceUtils; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.when; - -import org.apache.storm.metric.StormMetricsRegistry; - public class LocalizedResourceRetentionSetTest { @Test public void testAddResources() { - PortAndAssignment pna1 = new PortAndAssignmentImpl(1, new LocalAssignment("topo1", Collections.emptyList())); - PortAndAssignment pna2 = new PortAndAssignmentImpl(1, new LocalAssignment("topo2", Collections.emptyList())); + PortAndAssignment pna1 = new PortAndAssignmentImpl(1, new LocalAssignment("topo1", + Collections.emptyList())); + PortAndAssignment pna2 = new PortAndAssignmentImpl(1, new LocalAssignment("topo2", + Collections.emptyList())); String user = "user"; Map conf = new HashMap<>(); IAdvancedFSOps ops = mock(IAdvancedFSOps.class); LocalizedResourceRetentionSet lrretset = new LocalizedResourceRetentionSet(10); ConcurrentMap lrset = new ConcurrentHashMap<>(); StormMetricsRegistry metricsRegistry = new StormMetricsRegistry(); - LocalizedResource localresource1 = new LocalizedResource("key1", Paths.get("testfile1"), false, ops, conf, user, metricsRegistry); + LocalizedResource localresource1 = new LocalizedResource("key1", Paths.get("testfile1"), + false, ops, conf, user, metricsRegistry); localresource1.addReference(pna1, null); - LocalizedResource localresource2 = new LocalizedResource("key2", Paths.get("testfile2"), false, ops, conf, user, metricsRegistry); + LocalizedResource localresource2 = new LocalizedResource("key2", Paths.get("testfile2"), + false, ops, conf, user, metricsRegistry); localresource2.addReference(pna1, null); // check adding reference to local resource with topology of same name localresource2.addReference(pna2, null); @@ -62,21 +70,25 @@ public void testAddResources() { lrset.put("key1", localresource1); lrset.put("key2", localresource2); lrretset.addResources(lrset); - assertEquals(0, lrretset.getSizeWithNoReferences(), "number to clean is not 0 " + lrretset.noReferences); + assertEquals(0, lrretset.getSizeWithNoReferences(), "number to clean is not 0 " + + lrretset.noReferences); assertTrue(localresource1.removeReference(pna1)); lrretset = new LocalizedResourceRetentionSet(10); lrretset.addResources(lrset); - assertEquals(1, lrretset.getSizeWithNoReferences(), "number to clean is not 1 " + lrretset.noReferences); + assertEquals(1, lrretset.getSizeWithNoReferences(), "number to clean is not 1 " + + lrretset.noReferences); assertTrue(localresource2.removeReference(pna1)); lrretset = new LocalizedResourceRetentionSet(10); lrretset.addResources(lrset); - assertEquals(1, lrretset.getSizeWithNoReferences(), "number to clean is not 1 " + lrretset.noReferences); + assertEquals(1, lrretset.getSizeWithNoReferences(), "number to clean is not 1 " + + lrretset.noReferences); assertTrue(localresource2.removeReference(pna2)); lrretset = new LocalizedResourceRetentionSet(10); lrretset.addResources(lrset); - assertEquals(2, lrretset.getSizeWithNoReferences(), "number to clean is not 2 " + lrretset.noReferences); + assertEquals(2, lrretset.getSizeWithNoReferences(), "number to clean is not 2 " + + lrretset.noReferences); } @Test @@ -122,8 +134,10 @@ public void testRemoveEquivalent() { @Test public void testCleanup() throws Exception { ClientBlobStore mockBlobstore = mock(ClientBlobStore.class); - when(mockBlobstore.getBlobMeta(any())).thenReturn(new ReadableBlobMeta(new SettableBlobMeta(), 1)); - PortAndAssignment pna1 = new PortAndAssignmentImpl(1, new LocalAssignment("topo1", Collections.emptyList())); + when(mockBlobstore.getBlobMeta(any())) + .thenReturn(new ReadableBlobMeta(new SettableBlobMeta(), 1)); + PortAndAssignment pna1 = new PortAndAssignmentImpl(1, new LocalAssignment("topo1", + Collections.emptyList())); String user = "user"; Map conf = new HashMap<>(); IAdvancedFSOps ops = mock(IAdvancedFSOps.class); @@ -132,15 +146,18 @@ public void testCleanup() throws Exception { ConcurrentMap lrArchives = new ConcurrentHashMap<>(); StormMetricsRegistry metricsRegistry = new StormMetricsRegistry(); // no reference to key1 - LocalizedResource localresource1 = new LocalizedResource("key1", Paths.get("./target/TESTING/testfile1"), false, ops, conf, + LocalizedResource localresource1 = new LocalizedResource("key1", Paths + .get("./target/TESTING/testfile1"), false, ops, conf, user, metricsRegistry); localresource1.setSize(10); // no reference to archive1 - LocalizedResource archiveresource1 = new LocalizedResource("archive1", Paths.get("./target/TESTING/testarchive1"), true, ops, + LocalizedResource archiveresource1 = new LocalizedResource("archive1", Paths + .get("./target/TESTING/testarchive1"), true, ops, conf, user, metricsRegistry); archiveresource1.setSize(20); // reference to key2 - LocalizedResource localresource2 = new LocalizedResource("key2", Paths.get("./target/TESTING/testfile2"), false, ops, conf, + LocalizedResource localresource2 = new LocalizedResource("key2", Paths + .get("./target/TESTING/testfile2"), false, ops, conf, user, metricsRegistry); localresource2.addReference(pna1, null); // check adding reference to local resource with topology of same name diff --git a/storm-server/src/test/java/org/apache/storm/localizer/LocallyCachedBlobTest.java b/storm-server/src/test/java/org/apache/storm/localizer/LocallyCachedBlobTest.java index 505a3eb0311..9947cb76808 100644 --- a/storm-server/src/test/java/org/apache/storm/localizer/LocallyCachedBlobTest.java +++ b/storm-server/src/test/java/org/apache/storm/localizer/LocallyCachedBlobTest.java @@ -1,17 +1,26 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.localizer; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + import java.nio.file.Path; import java.nio.file.Paths; import java.util.HashMap; @@ -26,12 +35,10 @@ import org.junit.jupiter.api.Test; import org.mockito.Mockito; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - public class LocallyCachedBlobTest { private static final ClientBlobStore blobStore = Mockito.mock(ClientBlobStore.class); - private static final PortAndAssignment pna = new PortAndAssignmentImpl(6077, new LocalAssignment()); + private static final PortAndAssignment pna = new PortAndAssignmentImpl(6077, + new LocalAssignment()); private static final Map conf = new HashMap<>(); @Test @@ -72,7 +79,8 @@ public void testOutOfDate() throws KeyNotFoundException, AuthorizationException // blob version differs from the remote blobstore assertTrue(blob.requiresUpdate(blobStore, 102L)); - // now validate we don't need any update as versions match, regardless of remote blobstore update time + // now validate we don't need any update as versions match, regardless of remote blobstore + // update time blob.localVersion = blob.getRemoteVersion(blobStore); assertFalse(blob.requiresUpdate(blobStore, -1L)); assertFalse(blob.requiresUpdate(blobStore, 101L)); @@ -82,7 +90,8 @@ public void testOutOfDate() throws KeyNotFoundException, AuthorizationException public class TestableBlob extends LocalizedResource { long localVersion = 9L; - TestableBlob(String key, Path localBaseDir, boolean shouldUncompress, IAdvancedFSOps fsOps, Map conf, String user, StormMetricsRegistry metricRegistry) { + TestableBlob(String key, Path localBaseDir, boolean shouldUncompress, IAdvancedFSOps fsOps, + Map conf, String user, StormMetricsRegistry metricRegistry) { super(key, localBaseDir, shouldUncompress, fsOps, conf, user, metricRegistry); } diff --git a/storm-server/src/test/java/org/apache/storm/metric/ClusterMetricsConsumerExecutorTest.java b/storm-server/src/test/java/org/apache/storm/metric/ClusterMetricsConsumerExecutorTest.java index aa79fdb3ef2..e28e3d20976 100644 --- a/storm-server/src/test/java/org/apache/storm/metric/ClusterMetricsConsumerExecutorTest.java +++ b/storm-server/src/test/java/org/apache/storm/metric/ClusterMetricsConsumerExecutorTest.java @@ -1,17 +1,26 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.metric; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.mock; + import java.util.Collection; import java.util.Collections; import org.apache.storm.metric.api.DataPoint; @@ -19,9 +28,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.Mockito.mock; - public class ClusterMetricsConsumerExecutorTest { @BeforeEach @@ -50,10 +56,12 @@ public void testHandleDataPointsWithClusterMetricsShouldSkipHandlingMetricsIfFai sut.prepare(); // no specific reason to mock... this is one of the easiest ways to make dummy instance - sut.handleDataPoints(mock(IClusterMetricsConsumer.ClusterInfo.class), Collections.emptyList()); + sut.handleDataPoints(mock(IClusterMetricsConsumer.ClusterInfo.class), Collections + .emptyList()); assertEquals(1, MockFailingClusterMetricsConsumer.getPrepareCallCount()); - assertEquals(0, MockFailingClusterMetricsConsumer.getHandleDataPointsWithClusterInfoCallCount()); + assertEquals(0, MockFailingClusterMetricsConsumer + .getHandleDataPointsWithClusterInfoCallCount()); } @Test @@ -65,10 +73,12 @@ public void testHandleDataPointsWithSupervisorMetricsShouldRetryInitializingClus sut.prepare(); // no specific reason to mock... this is one of the easiest ways to make dummy instance - sut.handleDataPoints(mock(IClusterMetricsConsumer.SupervisorInfo.class), Collections.emptyList()); + sut.handleDataPoints(mock(IClusterMetricsConsumer.SupervisorInfo.class), Collections + .emptyList()); assertEquals(1, MockFailingClusterMetricsConsumer.getPrepareCallCount()); - assertEquals(0, MockFailingClusterMetricsConsumer.getHandleDataPointsWithSupervisorInfoCallCount()); + assertEquals(0, MockFailingClusterMetricsConsumer + .getHandleDataPointsWithSupervisorInfoCallCount()); } public static class MockFailingClusterMetricsConsumer implements IClusterMetricsConsumer { @@ -113,7 +123,8 @@ public void handleDataPoints(ClusterInfo clusterInfo, Collection data } @Override - public void handleDataPoints(SupervisorInfo supervisorInfo, Collection dataPoints) { + public void handleDataPoints(SupervisorInfo supervisorInfo, + Collection dataPoints) { handleDataPointsWithSupervisorInfoCallCount++; } diff --git a/storm-server/src/test/java/org/apache/storm/metricstore/rocksdb/RocksDbKeyTest.java b/storm-server/src/test/java/org/apache/storm/metricstore/rocksdb/RocksDbKeyTest.java index 72d5a9430e4..bc83f34b412 100644 --- a/storm-server/src/test/java/org/apache/storm/metricstore/rocksdb/RocksDbKeyTest.java +++ b/storm-server/src/test/java/org/apache/storm/metricstore/rocksdb/RocksDbKeyTest.java @@ -8,7 +8,7 @@ * with the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, + *

      Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the @@ -18,12 +18,12 @@ package org.apache.storm.metricstore.rocksdb; -import org.apache.storm.metricstore.AggLevel; -import org.junit.jupiter.api.Test; - import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +import org.apache.storm.metricstore.AggLevel; +import org.junit.jupiter.api.Test; + public class RocksDbKeyTest { @Test diff --git a/storm-server/src/test/java/org/apache/storm/metricstore/rocksdb/RocksDbStoreTest.java b/storm-server/src/test/java/org/apache/storm/metricstore/rocksdb/RocksDbStoreTest.java index f34c4c2519c..a6e37deeee2 100644 --- a/storm-server/src/test/java/org/apache/storm/metricstore/rocksdb/RocksDbStoreTest.java +++ b/storm-server/src/test/java/org/apache/storm/metricstore/rocksdb/RocksDbStoreTest.java @@ -18,8 +18,20 @@ package org.apache.storm.metricstore.rocksdb; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import org.apache.commons.io.FileUtils; import org.apache.storm.DaemonConfig; +import org.apache.storm.metric.StormMetricsRegistry; import org.apache.storm.metricstore.AggLevel; import org.apache.storm.metricstore.FilterOptions; import org.apache.storm.metricstore.Metric; @@ -31,19 +43,6 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import org.apache.storm.metric.StormMetricsRegistry; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - public class RocksDbStoreTest { static MetricStore store; static Path tempDirForTest; @@ -54,7 +53,8 @@ public static void setUp() throws MetricException, IOException { StringMetadataCache.cleanUp(); tempDirForTest = Files.createTempDirectory("RocksDbStoreTest"); Map conf = new HashMap<>(); - conf.put(DaemonConfig.STORM_METRIC_STORE_CLASS, "org.apache.storm.metricstore.rocksdb.RocksDbStore"); + conf.put(DaemonConfig.STORM_METRIC_STORE_CLASS, + "org.apache.storm.metricstore.rocksdb.RocksDbStore"); conf.put(DaemonConfig.STORM_ROCKSDB_LOCATION, tempDirForTest.toString()); conf.put(DaemonConfig.STORM_ROCKSDB_CREATE_IF_MISSING, true); conf.put(DaemonConfig.STORM_ROCKSDB_METADATA_STRING_CACHE_CAPACITY, 4000); @@ -78,20 +78,20 @@ public void testAggregation() throws Exception { double sum10 = 0.0; double sum60 = 0.0; Metric toPopulate = null; - for (int i=0; i<20; i++) { + for (int i = 0; i < 20; i++) { double value = 5 + i; - long timestamp = 1L + i*60*1000; + long timestamp = 1L + i * 60 * 1000; Metric m = new Metric("cpu", timestamp, "myTopologyId123", value, "componentId1", "executorId1", "hostname1", "streamid1", 7777, AggLevel.AGG_LEVEL_NONE); toPopulate = new Metric(m); store.insert(m); - if (timestamp < 60*1000) { + if (timestamp < 60 * 1000) { sum0 += value; sum1 += value; sum10 += value; sum60 += value; - } else if (timestamp < 600*1000) { + } else if (timestamp < 600 * 1000) { sum10 += value; sum60 += value; } else { @@ -126,7 +126,7 @@ public void testAggregation() throws Exception { res = store.populateValue(toPopulate); assertTrue(res); assertEquals(sum10, toPopulate.getSum(), 0.001); - assertEquals(sum10/10.0, toPopulate.getValue(), 0.001); + assertEquals(sum10 / 10.0, toPopulate.getValue(), 0.001); assertEquals(5.0, toPopulate.getMin(), 0.001); assertEquals(14.0, toPopulate.getMax(), 0.001); assertEquals(10, toPopulate.getCount()); @@ -136,7 +136,7 @@ public void testAggregation() throws Exception { res = store.populateValue(toPopulate); assertTrue(res); assertEquals(sum60, toPopulate.getSum(), 0.001); - assertEquals(sum60/20.0, toPopulate.getValue(), 0.001); + assertEquals(sum60 / 20.0, toPopulate.getValue(), 0.001); assertEquals(5.0, toPopulate.getMin(), 0.001); assertEquals(24.0, toPopulate.getMax(), 0.001); assertEquals(20, toPopulate.getCount()); @@ -307,7 +307,8 @@ public void testMetricCleanup() throws Exception { assertTrue(list.size() >= 2); // delete anything older than an hour - MetricsCleaner cleaner = new MetricsCleaner((RocksDbStore)store, 1, 1, null, new StormMetricsRegistry()); + MetricsCleaner cleaner = new MetricsCleaner((RocksDbStore) store, 1, 1, null, + new StormMetricsRegistry()); cleaner.purgeMetrics(); list = getMetricsFromScan(filter); assertEquals(1, list.size()); diff --git a/storm-server/src/test/java/org/apache/storm/metricstore/rocksdb/RocksDbValueTest.java b/storm-server/src/test/java/org/apache/storm/metricstore/rocksdb/RocksDbValueTest.java index 7e9c1e0e7ac..6a3cb871f70 100644 --- a/storm-server/src/test/java/org/apache/storm/metricstore/rocksdb/RocksDbValueTest.java +++ b/storm-server/src/test/java/org/apache/storm/metricstore/rocksdb/RocksDbValueTest.java @@ -18,13 +18,13 @@ package org.apache.storm.metricstore.rocksdb; +import static org.junit.jupiter.api.Assertions.assertEquals; + import org.apache.storm.metricstore.AggLevel; import org.apache.storm.metricstore.Metric; import org.apache.storm.metricstore.MetricException; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; - public class RocksDbValueTest { @Test @@ -50,7 +50,7 @@ public void testMetadataConstructor() { @Test public void testMetricConstructor() throws MetricException { - Metric m = new Metric("cpu", 1L,"myTopologyId123", 1, + Metric m = new Metric("cpu", 1L, "myTopologyId123", 1, "componentId1", "executorId1", "hostname1", "streamid1", 7777, AggLevel.AGG_LEVEL_NONE); Metric m2 = new Metric(m); diff --git a/storm-server/src/test/java/org/apache/storm/metricstore/rocksdb/StringMetadataCacheTest.java b/storm-server/src/test/java/org/apache/storm/metricstore/rocksdb/StringMetadataCacheTest.java index fe6e63bc402..fe514703fb0 100644 --- a/storm-server/src/test/java/org/apache/storm/metricstore/rocksdb/StringMetadataCacheTest.java +++ b/storm-server/src/test/java/org/apache/storm/metricstore/rocksdb/StringMetadataCacheTest.java @@ -8,7 +8,7 @@ * with the License. You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, + *

      Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the @@ -18,18 +18,18 @@ package org.apache.storm.metricstore.rocksdb; -import org.apache.storm.metricstore.MetricException; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.rocksdb.RocksDB; - import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; +import org.apache.storm.metricstore.MetricException; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.rocksdb.RocksDB; + public class StringMetadataCacheTest { @BeforeEach diff --git a/storm-server/src/test/java/org/apache/storm/nimbus/InMemoryTopologyActionNotifier.java b/storm-server/src/test/java/org/apache/storm/nimbus/InMemoryTopologyActionNotifier.java index 83c46f63f7b..01feed4b1ae 100644 --- a/storm-server/src/test/java/org/apache/storm/nimbus/InMemoryTopologyActionNotifier.java +++ b/storm-server/src/test/java/org/apache/storm/nimbus/InMemoryTopologyActionNotifier.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -19,7 +25,8 @@ /** * In-memory topology action notifier for testing. - * Duplicated from storm-core test sources since storm-server cannot depend on storm-core test classes. + * Duplicated from storm-core test sources since storm-server cannot depend on storm-core test + * classes. */ public class InMemoryTopologyActionNotifier implements ITopologyActionNotifierPlugin { @@ -29,7 +36,7 @@ public class InMemoryTopologyActionNotifier implements ITopologyActionNotifierPl @Override public void prepare(Map stormConf) { - //no-op + // no-op } @Override @@ -46,6 +53,6 @@ public List getTopologyActions(String topologyName) { @Override public void cleanup() { - //no-op + // no-op } } diff --git a/storm-server/src/test/java/org/apache/storm/nimbus/LocalNimbusTest.java b/storm-server/src/test/java/org/apache/storm/nimbus/LocalNimbusTest.java index ad1137def32..b109735cde7 100644 --- a/storm-server/src/test/java/org/apache/storm/nimbus/LocalNimbusTest.java +++ b/storm-server/src/test/java/org/apache/storm/nimbus/LocalNimbusTest.java @@ -19,6 +19,8 @@ package org.apache.storm.nimbus; +import static org.junit.jupiter.api.Assertions.assertEquals; + import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -37,18 +39,19 @@ import org.apache.storm.utils.Utils; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; - /** - * Tests local cluster with nimbus and a plugin for {@link Config#STORM_TOPOLOGY_SUBMISSION_NOTIFIER_PLUGIN}. + * Tests local cluster with nimbus and a plugin for {@link + * Config#STORM_TOPOLOGY_SUBMISSION_NOTIFIER_PLUGIN}. */ public class LocalNimbusTest { public static StormTopology createTestTopology() { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("words", new TestWordSpout(), generateParallelismHint()); - builder.setBolt("count", new TestWordCounter(), generateParallelismHint()).shuffleGrouping("words"); - builder.setBolt("globalCount", new TestGlobalCount(), generateParallelismHint()).shuffleGrouping("count"); + builder.setBolt("count", new TestWordCounter(), generateParallelismHint()) + .shuffleGrouping("words"); + builder.setBolt("globalCount", new TestGlobalCount(), generateParallelismHint()) + .shuffleGrouping("count"); return builder.createTopology(); } @@ -67,8 +70,10 @@ public void testSubmitTopologyToLocalNimbus() throws Exception { Config topoConf = new Config(); topoConf.putAll(Utils.readDefaultConfig()); topoConf.setDebug(true); - topoConf.put("storm.cluster.mode", "local"); // default is aways "distributed" but here local cluster is being used. - topoConf.put(Config.STORM_TOPOLOGY_SUBMISSION_NOTIFIER_PLUGIN, InmemoryTopologySubmitterHook.class.getName()); + topoConf.put("storm.cluster.mode", + "local"); // default is aways "distributed" but here local cluster is being used. + topoConf.put(Config.STORM_TOPOLOGY_SUBMISSION_NOTIFIER_PLUGIN, + InmemoryTopologySubmitterHook.class.getName()); topoConf.put(Config.NIMBUS_THRIFT_PORT, port); List topologyNames = new ArrayList<>(); @@ -87,7 +92,8 @@ public static class InmemoryTopologySubmitterHook implements ISubmitterHook { public static final List submittedTopologies = new ArrayList<>(); @Override - public void notify(TopologyInfo topologyInfo, Map topoConf, StormTopology topology) { + public void notify(TopologyInfo topologyInfo, Map topoConf, + StormTopology topology) { submittedTopologies.add(new TopologyDetails(topologyInfo.get_name(), topology)); } } @@ -103,15 +109,21 @@ public TopologyDetails(String topologyName, StormTopology stormTopology) { @Override public boolean equals(Object o) { - if (this == o) return true; - if (!(o instanceof TopologyDetails)) return false; + if (this == o) { + return true; + } + if (!(o instanceof TopologyDetails)) { + return false; + } TopologyDetails that = (TopologyDetails) o; - if (topologyName != null ? !topologyName.equals(that.topologyName) : that.topologyName != null) { + if (topologyName != null ? !topologyName + .equals(that.topologyName) : that.topologyName != null) { return false; } - return !(stormTopology != null ? !stormTopology.equals(that.stormTopology) : that.stormTopology != null); + return !(stormTopology != null ? !stormTopology + .equals(that.stormTopology) : that.stormTopology != null); } @@ -124,10 +136,10 @@ public int hashCode() { @Override public String toString() { - return "TopologyDetails{" + - "topologyName='" + topologyName + '\'' + - ", stormTopology=" + stormTopology + - '}'; + return "TopologyDetails{" + + "topologyName='" + topologyName + '\'' + + ", stormTopology=" + stormTopology + + '}'; } } } diff --git a/storm-server/src/test/java/org/apache/storm/pacemaker/PacemakerServerTest.java b/storm-server/src/test/java/org/apache/storm/pacemaker/PacemakerServerTest.java index 1718d64ad9b..d9f815b0212 100644 --- a/storm-server/src/test/java/org/apache/storm/pacemaker/PacemakerServerTest.java +++ b/storm-server/src/test/java/org/apache/storm/pacemaker/PacemakerServerTest.java @@ -1,17 +1,35 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.pacemaker; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -30,8 +48,8 @@ import org.apache.storm.metric.StormMetricsRegistry; import org.apache.storm.pacemaker.codec.ThriftDecoder; import org.apache.storm.pacemaker.codec.ThriftEncoder; -import org.apache.storm.pacemaker.codec.ThriftNettyServerCodec; import org.apache.storm.pacemaker.codec.ThriftNettyServerCodec.AuthMethod; +import org.apache.storm.pacemaker.codec.ThriftNettyServerCodec; import org.apache.storm.shade.io.netty.buffer.ByteBuf; import org.apache.storm.shade.io.netty.buffer.Unpooled; import org.apache.storm.shade.io.netty.channel.Channel; @@ -41,18 +59,6 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.atLeastOnce; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - @SuppressWarnings("deprecation") public class PacemakerServerTest { @@ -85,10 +91,12 @@ private static ByteBuf controlFrame(ControlMessage controlMessage) { controlMessage.write(buf); byte[] blob = new byte[buf.readableBytes()]; buf.readBytes(blob); - return frame(new HBMessage(HBServerMessageType.CONTROL_MESSAGE, HBMessageData.message_blob(blob))); + return frame(new HBMessage(HBServerMessageType.CONTROL_MESSAGE, HBMessageData + .message_blob(blob))); } - private static ByteBuf saslTokenFrame(short identifier, int declaredPayloadLen, byte[] payload) { + private static ByteBuf saslTokenFrame(short identifier, int declaredPayloadLen, + byte[] payload) { ByteBuf buf = Unpooled.buffer(); buf.writeShort(identifier); buf.writeInt(declaredPayloadLen); @@ -97,7 +105,8 @@ private static ByteBuf saslTokenFrame(short identifier, int declaredPayloadLen, } byte[] blob = new byte[buf.readableBytes()]; buf.readBytes(blob); - return frame(new HBMessage(HBServerMessageType.SASL_MESSAGE_TOKEN, HBMessageData.message_blob(blob))); + return frame(new HBMessage(HBServerMessageType.SASL_MESSAGE_TOKEN, HBMessageData + .message_blob(blob))); } private static HBMessage readResponse(EmbeddedChannel serverChannel) { @@ -110,12 +119,14 @@ private static HBMessage readResponse(EmbeddedChannel serverChannel) { } private EmbeddedChannel pipeline() { - return new EmbeddedChannel(new ThriftNettyServerCodec(server, config(), AuthMethod.NONE, MAX_LENGTH)); + return new EmbeddedChannel(new ThriftNettyServerCodec(server, config(), AuthMethod.NONE, + MAX_LENGTH)); } @BeforeAll public static void setUp() { - server = new PacemakerServer(new Pacemaker(new ConcurrentHashMap<>(), new StormMetricsRegistry()), config()); + server = new PacemakerServer(new Pacemaker(new ConcurrentHashMap<>(), + new StormMetricsRegistry()), config()); } @AfterAll @@ -126,7 +137,8 @@ public static void tearDown() { @Test public void heartbeatRequestIsAnswered() { EmbeddedChannel channel = pipeline(); - HBMessage request = new HBMessage(HBServerMessageType.CREATE_PATH, HBMessageData.path("/path")); + HBMessage request = new HBMessage(HBServerMessageType.CREATE_PATH, HBMessageData + .path("/path")); request.set_message_id(7); channel.writeInbound(frame(request)); @@ -147,7 +159,8 @@ public void controlFrameClosesOnlyThatConnection() { assertFalse(channel.isActive()); assertNull(channel.readOutbound()); - other.writeInbound(frame(new HBMessage(HBServerMessageType.CREATE_PATH, HBMessageData.path("/path")))); + other.writeInbound(frame(new HBMessage(HBServerMessageType.CREATE_PATH, HBMessageData + .path("/path")))); assertEquals(HBServerMessageType.CREATE_PATH_RESPONSE, readResponse(other).get_type()); assertTrue(other.isActive()); } @@ -157,13 +170,15 @@ public void oversizedSaslTokenClosesOnlyThatConnection() { EmbeddedChannel other = pipeline(); EmbeddedChannel channel = pipeline(); - // A SASL token frame that declares a ~2GB payload previously OOM'd the decoder and terminated the daemon. + // A SASL token frame that declares a ~2GB payload previously OOM'd the decoder and + // terminated the daemon. channel.writeInbound(saslTokenFrame(SaslMessageToken.IDENTIFIER, Integer.MAX_VALUE, null)); assertFalse(channel.isActive()); assertNull(channel.readOutbound()); - other.writeInbound(frame(new HBMessage(HBServerMessageType.CREATE_PATH, HBMessageData.path("/path")))); + other.writeInbound(frame(new HBMessage(HBServerMessageType.CREATE_PATH, HBMessageData + .path("/path")))); assertEquals(HBServerMessageType.CREATE_PATH_RESPONSE, readResponse(other).get_type()); assertTrue(other.isActive()); } @@ -231,9 +246,11 @@ public void digestHandshakeAuthenticatesChannel() throws Exception { assertTrue(clientChannel.isActive()); verify(saslServer, never()).received(any(), anyString(), any(Channel.class)); - HBMessage request = new HBMessage(HBServerMessageType.GET_PULSE, HBMessageData.path("/path")); + HBMessage request = new HBMessage(HBServerMessageType.GET_PULSE, HBMessageData + .path("/path")); serverChannel.writeInbound(frame(request)); - verify(saslServer).received(request, serverChannel.remoteAddress().toString(), serverChannel); + verify(saslServer).received(request, serverChannel.remoteAddress().toString(), + serverChannel); // Control frames other than the handshake request are still refused once authenticated. serverChannel.writeInbound(controlFrame(ControlMessage.EOB_MESSAGE)); diff --git a/storm-server/src/test/java/org/apache/storm/scheduler/ClusterTest.java b/storm-server/src/test/java/org/apache/storm/scheduler/ClusterTest.java index 54280e62a97..6db6022a5e2 100644 --- a/storm-server/src/test/java/org/apache/storm/scheduler/ClusterTest.java +++ b/storm-server/src/test/java/org/apache/storm/scheduler/ClusterTest.java @@ -1,24 +1,30 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.scheduler; +import static org.junit.jupiter.api.Assertions.assertEquals; + import java.util.HashMap; import java.util.Map; import org.apache.storm.Config; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; - /** * Unit tests for {@link Cluster}. */ @@ -50,6 +56,7 @@ private Map getPopulatedConfig() { /** * Test Cluster.getAssignedMemoryForSlot with a single config value set. + * * @param key - the config key to set * @param value - the config value to set * @param expectedValue - the expected result @@ -62,7 +69,8 @@ private void singleValueTest(String key, String value, double expectedValue) { @Test public void getAssignedMemoryForSlot_allNull() { Map topConf = getEmptyConfig(); - assertEquals(TOPOLOGY_WORKER_DEFAULT_MEMORY_ALLOCATION, Cluster.getAssignedMemoryForSlot(topConf), 0); + assertEquals(TOPOLOGY_WORKER_DEFAULT_MEMORY_ALLOCATION, Cluster + .getAssignedMemoryForSlot(topConf), 0); } @Test diff --git a/storm-server/src/test/java/org/apache/storm/scheduler/IsolationSchedulerTest.java b/storm-server/src/test/java/org/apache/storm/scheduler/IsolationSchedulerTest.java index 6c8cd246436..736228e590b 100644 --- a/storm-server/src/test/java/org/apache/storm/scheduler/IsolationSchedulerTest.java +++ b/storm-server/src/test/java/org/apache/storm/scheduler/IsolationSchedulerTest.java @@ -1,17 +1,25 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.scheduler; +import static org.junit.jupiter.api.Assertions.assertEquals; + import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -24,8 +32,6 @@ import org.apache.storm.scheduler.resource.normalization.ResourceMetrics; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; - /** * Unit tests for {@link IsolationScheduler}. */ @@ -42,10 +48,12 @@ private static SupervisorDetails mkSupervisor(String id, String host, int numPor return new SupervisorDetails(id, host, null, ports, resources); } - private static Cluster mkCluster(Map supervisors, Topologies topologies) { + private static Cluster mkCluster(Map supervisors, + Topologies topologies) { INimbus iNimbus = new TestUtilsForBlacklistScheduler.INimbusTest(); ResourceMetrics resourceMetrics = new ResourceMetrics(new StormMetricsRegistry()); - return new Cluster(iNimbus, resourceMetrics, supervisors, new HashMap(), + return new Cluster(iNimbus, resourceMetrics, supervisors, new HashMap(), topologies, new HashMap()); } @@ -64,7 +72,8 @@ public void hostAssignableSlots_prefersHostWithMoreFreeSlots() { supervisors.put("sup-free", mkSupervisor("sup-free", "host-free", 2)); Map conf = new HashMap<>(); - TopologyDetails filler = TestUtilsForBlacklistScheduler.getTopology("filler", conf, 1, 0, 1, 0, 0, false); + TopologyDetails filler = TestUtilsForBlacklistScheduler.getTopology("filler", conf, 1, 0, 1, + 0, 0, false); Map topoMap = new HashMap<>(); topoMap.put(filler.getId(), filler); Topologies topologies = new Topologies(topoMap); @@ -102,4 +111,4 @@ public void hostAssignableSlots_breaksTiesByHostName() { assertEquals(2, ranked.get(1).getFreeSlots()); assertEquals(hostOrder(ranked), List.of("host-aaa", "host-bbb")); } -} \ No newline at end of file +} diff --git a/storm-server/src/test/java/org/apache/storm/scheduler/SchedulerModelTest.java b/storm-server/src/test/java/org/apache/storm/scheduler/SchedulerModelTest.java index 2ce32057ab2..90ae8fe0333 100644 --- a/storm-server/src/test/java/org/apache/storm/scheduler/SchedulerModelTest.java +++ b/storm-server/src/test/java/org/apache/storm/scheduler/SchedulerModelTest.java @@ -18,6 +18,11 @@ package org.apache.storm.scheduler; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -26,7 +31,6 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; - import org.apache.storm.Config; import org.apache.storm.daemon.nimbus.Nimbus.StandaloneINimbus; import org.apache.storm.generated.StormTopology; @@ -34,11 +38,6 @@ import org.apache.storm.scheduler.resource.normalization.ResourceMetrics; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; - public class SchedulerModelTest { @Test @@ -48,7 +47,8 @@ public void testSupervisorDetails() { executorToSlot.put(new ExecutorDetails(6, 10), new WorkerSlot("supervisor2", 2)); String topologyId = "topology1"; - SchedulerAssignmentImpl assignment = new SchedulerAssignmentImpl(topologyId, executorToSlot, null, null); + SchedulerAssignmentImpl assignment = new SchedulerAssignmentImpl(topologyId, executorToSlot, + null, null); // test assign assignment.assign(new WorkerSlot("supervisor1", 1), @@ -73,7 +73,8 @@ public void testSupervisorDetails() { assignment.unassignBySlot(new WorkerSlot("supervisor1", 1)); Map afterUnassign = assignment.getExecutorToSlot(); assertEquals(1, afterUnassign.size()); - assertEquals(new WorkerSlot("supervisor2", 2), afterUnassign.get(new ExecutorDetails(6, 10))); + assertEquals(new WorkerSlot("supervisor2", 2), afterUnassign.get(new ExecutorDetails(6, + 10))); } @Test @@ -92,7 +93,8 @@ public void testTopologies() { executorToComponent1, "user"); // test topology.selectExecutorToComponent - Map selected = topology1.selectExecutorToComponent(Arrays.asList(executor1)); + Map selected = topology1.selectExecutorToComponent(Arrays + .asList(executor1)); assertEquals(1, selected.size()); assertEquals("spout1", selected.get(executor1)); @@ -175,9 +177,12 @@ public void testCluster() { executorToSlot3.put(executor21, new WorkerSlot("supervisor1", 5)); executorToSlot3.put(executor22, new WorkerSlot("supervisor1", 5)); - SchedulerAssignmentImpl assignment1 = new SchedulerAssignmentImpl("topology1", executorToSlot1, null, null); - SchedulerAssignmentImpl assignment2 = new SchedulerAssignmentImpl("topology2", executorToSlot2, null, null); - SchedulerAssignmentImpl assignment3 = new SchedulerAssignmentImpl("topology3", executorToSlot3, null, null); + SchedulerAssignmentImpl assignment1 = new SchedulerAssignmentImpl("topology1", + executorToSlot1, null, null); + SchedulerAssignmentImpl assignment2 = new SchedulerAssignmentImpl("topology2", + executorToSlot2, null, null); + SchedulerAssignmentImpl assignment3 = new SchedulerAssignmentImpl("topology3", + executorToSlot3, null, null); Map supervisors = new HashMap<>(); supervisors.put("supervisor1", supervisor1); @@ -215,7 +220,8 @@ public void testCluster() { assertEquals(new HashSet<>(Arrays.asList("topology1", "topology3")), needsSchedulingIds); // test Cluster.getNeedsSchedulingExecutorToComponents - Map needsSched1 = cluster.getNeedsSchedulingExecutorToComponents(topology1); + Map needsSched1 = cluster + .getNeedsSchedulingExecutorToComponents(topology1); assertEquals(1, needsSched1.size()); assertEquals("bolt2", needsSched1.get(executor3)); assertTrue(cluster.getNeedsSchedulingExecutorToComponents(topology2).isEmpty()); @@ -251,7 +257,8 @@ public void testCluster() { Set availSlots2 = cluster.getAvailableSlots(supervisor2).stream() .map(s -> s.getNodeId() + ":" + s.getPort()) .collect(Collectors.toSet()); - assertEquals(new HashSet<>(Arrays.asList("supervisor2:6", "supervisor2:8", "supervisor2:10")), availSlots2); + assertEquals(new HashSet<>(Arrays.asList("supervisor2:6", "supervisor2:8", + "supervisor2:10")), availSlots2); // test Cluster.getAvailableSlots (all) Set allAvailSlots = cluster.getAvailableSlots().stream() @@ -306,11 +313,13 @@ public void testCluster() { // test Cluster.assign: if an executor is already assigned, there will be an exception assertThrows(Exception.class, - () -> cluster.assign(new WorkerSlot("supervisor1", 9), "topology1", Arrays.asList(executor1))); + () -> cluster.assign(new WorkerSlot("supervisor1", 9), "topology1", Arrays + .asList(executor1))); // test Cluster.assign: if a slot is occupied, there will be an exception assertThrows(Exception.class, - () -> cluster.assign(new WorkerSlot("supervisor2", 4), "topology1", Arrays.asList(executor3))); + () -> cluster.assign(new WorkerSlot("supervisor2", 4), "topology1", Arrays + .asList(executor3))); // test Cluster.freeSlot cluster.freeSlot(new WorkerSlot("supervisor1", 7)); diff --git a/storm-server/src/test/java/org/apache/storm/scheduler/TestEvenSchedulerIdleSupervisor.java b/storm-server/src/test/java/org/apache/storm/scheduler/TestEvenSchedulerIdleSupervisor.java index 32756a4ec7c..03e4f765b68 100644 --- a/storm-server/src/test/java/org/apache/storm/scheduler/TestEvenSchedulerIdleSupervisor.java +++ b/storm-server/src/test/java/org/apache/storm/scheduler/TestEvenSchedulerIdleSupervisor.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

      Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -17,12 +17,15 @@ */ package org.apache.storm.scheduler; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + import java.util.Collections; import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; - import org.apache.storm.Config; import org.apache.storm.DaemonConfig; import org.apache.storm.generated.StormTopology; @@ -34,17 +37,16 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - /** * Tests for the idle-supervisor rebalance behavior added to - * {@link EvenScheduler#redistributeOntoIdleSupervisors(Topologies, Cluster)} and its per-supervisor eligibility predicate + * {@link EvenScheduler#redistributeOntoIdleSupervisors(Topologies, Cluster)} and its per-supervisor + * eligibility predicate * {@link Cluster#isIdleSupervisorAvailableForEvenRebalance(SupervisorDetails)}. * - *

      Trigger condition is binary: at least one non-blacklisted supervisor with zero used slots must exist. The cluster - * being "almost balanced" never triggers the new logic, so a near-even distribution is preserved as-is. Each round only + *

      Trigger condition is binary: at least one non-blacklisted supervisor with zero used slots must + * exist. The cluster + * being "almost balanced" never triggers the new logic, so a near-even distribution is preserved + * as-is. Each round only * frees up to {@code nimbus.even.rebalance.max.free.per.topology} workers and never drains a supervisor down to zero. * The tests assert on the observable effect of {@code redistributeOntoIdleSupervisors} (the resulting assignment) rather * than on any internal boolean predicate. @@ -54,14 +56,17 @@ public class TestEvenSchedulerIdleSupervisor { private static final String TOPO_ID = "topo-1"; /** - * supA and supB host the topology; supC is freshly returned and idle. Topology has 2 workers on supA and 1 on supB. + * supA and supB host the topology; supC is freshly returned and idle. Topology has 2 workers on + * supA and 1 on supB. */ - private Cluster buildClusterWithIdleSupervisor(boolean enableRebalance, int maxFreePerTopology) { + private Cluster buildClusterWithIdleSupervisor(boolean enableRebalance, + int maxFreePerTopology) { return buildClusterWithIdleSupervisor(TestUtilsForBlacklistScheduler.genSupervisors(3, 4), evenRebalanceConf(enableRebalance, maxFreePerTopology)); } - private Cluster buildClusterWithIdleSupervisor(Map supMap, Map conf) { + private Cluster buildClusterWithIdleSupervisor(Map supMap, + Map conf) { // Build a topology and assign 3 workers: two on sup-0 and one on sup-1. sup-2 stays idle. TopologyDetails topology = makeTopologyDetails(TOPO_ID, 3); @@ -77,7 +82,8 @@ private Cluster buildClusterWithIdleSupervisor(Map su for (int i = 0; i < execs.size(); i++) { execToSlot.put(execs.get(i), slotRing[i % slotRing.length]); } - SchedulerAssignmentImpl assignment = new SchedulerAssignmentImpl(TOPO_ID, execToSlot, null, null); + SchedulerAssignmentImpl assignment = new SchedulerAssignmentImpl(TOPO_ID, execToSlot, null, + null); Map assignments = new HashMap<>(); assignments.put(TOPO_ID, assignment); @@ -98,7 +104,8 @@ private Map evenRebalanceConf(boolean enableRebalance, int maxFr return conf; } - private Map genSupervisorsWithUptime(int numSup, int numPorts, long uptimeSecs) { + private Map genSupervisorsWithUptime(int numSup, int numPorts, + long uptimeSecs) { Map supMap = new HashMap<>(); for (int i = 0; i < numSup; i++) { SupervisorDetails sup = supervisor("sup-" + i, "host-" + i, numPorts, uptimeSecs); @@ -121,7 +128,8 @@ private Cluster newCluster(Map supMap, Map conf) { StormMetricsRegistry metricsRegistry = new StormMetricsRegistry(); ResourceMetrics resourceMetrics = new ResourceMetrics(metricsRegistry); - return new Cluster(new TestUtilsForBlacklistScheduler.INimbusTest(), resourceMetrics, supMap, + return new Cluster(new TestUtilsForBlacklistScheduler.INimbusTest(), resourceMetrics, + supMap, assignments, topologies, conf); } @@ -132,10 +140,12 @@ private TopologyDetails makeTopologyDetails(String id, int numWorkers, int paral TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("spout-0", new TestUtilsForBlacklistScheduler.TestSpout(), parallelism); - builder.setBolt("bolt-0", new TestUtilsForBlacklistScheduler.TestBolt(), parallelism).shuffleGrouping("spout-0"); + builder.setBolt("bolt-0", new TestUtilsForBlacklistScheduler.TestBolt(), parallelism) + .shuffleGrouping("spout-0"); StormTopology stormTopology = builder.createTopology(); - Map execsAndComps = TestUtilsForBlacklistScheduler.genExecsAndComps( + Map execsAndComps = TestUtilsForBlacklistScheduler + .genExecsAndComps( stormTopology, parallelism, parallelism); return new TopologyDetails(id, conf, stormTopology, numWorkers, execsAndComps, 0, "user"); } @@ -159,30 +169,37 @@ public void disabledByDefault_doesNotTrigger() { EvenScheduler.redistributeOntoIdleSupervisors(cluster.getTopologies(), cluster); - // Disabled flag must short-circuit the rebalance even when an idle supervisor exists: nothing moves onto sup-2. + // Disabled flag must short-circuit the rebalance even when an idle supervisor exists: + // nothing moves onto sup-2. assertEquals(2, usedSlotCount(cluster, "sup-0")); assertEquals(1, usedSlotCount(cluster, "sup-1")); assertEquals(0, usedSlotCount(cluster, "sup-2")); assertFalse(cluster.needsScheduling(firstTopology(cluster)), - "needsScheduling must remain false when the new behavior is disabled and the topology is fully assigned"); + "needsScheduling must remain false when the new behavior is disabled and the " + + "topology is fully assigned"); } @Test public void enabledWithIdleSupervisor_doesNotChangeGenericNeedsScheduling() { Cluster cluster = buildClusterWithIdleSupervisor(true, 1); - // Enabling the opt-in idle rebalance must not leak into the generic scheduling triggers other schedulers use. + // Enabling the opt-in idle rebalance must not leak into the generic scheduling triggers + // other schedulers use. assertFalse(cluster.needsScheduling(firstTopology(cluster)), - "needsScheduling is used by schedulers other than EvenScheduler; the idle trigger stays out of that generic path"); + "needsScheduling is used by schedulers other than EvenScheduler; the idle trigger " + + "stays out of that generic path"); assertFalse(cluster.needsSchedulingRas(firstTopology(cluster)), - "ResourceAwareScheduler keeps using needsSchedulingRas, so this opt-in EvenScheduler feature is out of RAS scope"); + "ResourceAwareScheduler keeps using needsSchedulingRas, so this opt-in " + + "EvenScheduler feature is out of RAS scope"); EvenScheduler.redistributeOntoIdleSupervisors(cluster.getTopologies(), cluster); - // The feature itself still fires: one worker relocates onto the idle supervisor (the observable trigger)... + // The feature itself still fires: one worker relocates onto the idle supervisor (the + // observable trigger)... assertEquals(1, usedSlotCount(cluster, "sup-2")); assertEquals(3, cluster.getAssignedNumWorkers(firstTopology(cluster))); - // ...while the generic triggers stay false afterward -- the relocation kept the topology fully assigned. + // ...while the generic triggers stay false afterward -- the relocation kept the topology + // fully assigned. assertFalse(cluster.needsScheduling(firstTopology(cluster))); assertFalse(cluster.needsSchedulingRas(firstTopology(cluster))); } @@ -195,7 +212,7 @@ public void noIdleSupervisor_doesNotTrigger() { Map assignments = new HashMap<>(); assignments.put(TOPO_ID, buildAssignment(topology, new WorkerSlot[]{ - new WorkerSlot("sup-0", 0), new WorkerSlot("sup-0", 1), new WorkerSlot("sup-1", 0), + new WorkerSlot("sup-0", 0), new WorkerSlot("sup-0", 1), new WorkerSlot("sup-1", 0), })); Map topoMap = new HashMap<>(); @@ -206,7 +223,8 @@ public void noIdleSupervisor_doesNotTrigger() { EvenScheduler.redistributeOntoIdleSupervisors(cluster.getTopologies(), cluster); - // No supervisor has zero used slots, so the binary trigger never fires: the assignment is left untouched. + // No supervisor has zero used slots, so the binary trigger never fires: the assignment is + // left untouched. assertEquals(2, usedSlotCount(cluster, "sup-0")); assertEquals(1, usedSlotCount(cluster, "sup-1")); assertEquals(3, cluster.getAssignedNumWorkers(firstTopology(cluster))); @@ -222,7 +240,8 @@ public void redistributeRelocatesAtMostMaxFreeWorkersPerTopology() { EvenScheduler.redistributeOntoIdleSupervisors(cluster.getTopologies(), cluster); - // max-free=1 caps the topology to a single relocation; pulled from the most-loaded supervisor (sup-0) + // max-free=1 caps the topology to a single relocation; pulled from the most-loaded + // supervisor (sup-0) // and placed directly onto the idle supervisor. assertEquals(1, usedSlotCount(cluster, "sup-0")); assertEquals(1, usedSlotCount(cluster, "sup-1")); @@ -237,7 +256,7 @@ public void redistributeNeverDrainsSupervisorToZero() { Map assignments = new HashMap<>(); assignments.put(TOPO_ID, buildAssignment(topology, new WorkerSlot[]{ - new WorkerSlot("sup-0", 0), new WorkerSlot("sup-1", 0), + new WorkerSlot("sup-0", 0), new WorkerSlot("sup-1", 0), })); Map topoMap = new HashMap<>(); @@ -248,7 +267,8 @@ public void redistributeNeverDrainsSupervisorToZero() { EvenScheduler.redistributeOntoIdleSupervisors(topologies, cluster); - // floor(2/3)=0 → topology gets a budget of 0 and is skipped entirely. No source supervisor is drained. + // floor(2/3)=0 → topology gets a budget of 0 and is skipped entirely. No source supervisor + // is drained. assertEquals(1, usedSlotCount(cluster, "sup-0")); assertEquals(1, usedSlotCount(cluster, "sup-1")); assertEquals(0, usedSlotCount(cluster, "sup-2")); @@ -269,12 +289,14 @@ public void scheduleTopologiesEvenly_movesOneWorkerToIdleSupervisor() { + usedSlotCount(cluster, "sup-2"); assertEquals(3, total); assertEquals(3, cluster.getAssignedNumWorkers(firstTopology(cluster)), - "relocation must preserve the topology's declared worker count, not just keep 3 slots occupied"); + "relocation must preserve the topology's declared worker count, not just keep 3 " + + "slots occupied"); } /** * Single-worker topology + idle supervisors must produce no movement: {@code floor(1 / N) = 0} for any N >= 2, so the - * drain budget evaluates to zero regardless of how many idle supervisors exist. Without this guard a single-worker + * drain budget evaluates to zero regardless of how many idle supervisors exist. Without this + * guard a single-worker * topology would ping-pong between supervisors every monitor cycle. */ @Test @@ -284,7 +306,8 @@ public void singleWorkerTopology_doesNotMoveDespiteIdleSupervisors() { TopologyDetails topology = makeTopologyDetails(topoId, 1, 1); Map assignments = new HashMap<>(); - assignments.put(topoId, buildAssignment(topology, new WorkerSlot[]{ new WorkerSlot("sup-0", 0) })); + assignments.put(topoId, buildAssignment(topology, new WorkerSlot[]{ new WorkerSlot("sup-0", + 0) })); Map topoMap = new HashMap<>(); topoMap.put(topoId, topology); @@ -300,9 +323,12 @@ public void singleWorkerTopology_doesNotMoveDespiteIdleSupervisors() { } /** - * 8-worker topology starts at distribution (4, 4, 0). With max-free unbounded the budget targets - * floor(numWorkers / numSupervisors) = 2 workers for the idle supervisor, and the round ends at (3, 3, 2) - * — fully even — without disturbing topologies on the next round (no supervisor is idle anymore). + * 8-worker topology starts at distribution (4, 4, 0). With max-free unbounded the budget + * targets + * floor(numWorkers / numSupervisors) = 2 workers for the idle supervisor, and the round ends at + * (3, 3, 2) + * — fully even — without disturbing topologies on the next round (no supervisor is idle + * anymore). */ @Test public void evenDistributionInOneRound_unboundedMaxFree() { @@ -312,10 +338,10 @@ public void evenDistributionInOneRound_unboundedMaxFree() { Map assignments = new HashMap<>(); assignments.put(topoId, buildAssignment(topology, new WorkerSlot[]{ - new WorkerSlot("sup-0", 0), new WorkerSlot("sup-0", 1), - new WorkerSlot("sup-0", 2), new WorkerSlot("sup-0", 3), - new WorkerSlot("sup-1", 0), new WorkerSlot("sup-1", 1), - new WorkerSlot("sup-1", 2), new WorkerSlot("sup-1", 3), + new WorkerSlot("sup-0", 0), new WorkerSlot("sup-0", 1), + new WorkerSlot("sup-0", 2), new WorkerSlot("sup-0", 3), + new WorkerSlot("sup-1", 0), new WorkerSlot("sup-1", 1), + new WorkerSlot("sup-1", 2), new WorkerSlot("sup-1", 3), })); Map topoMap = new HashMap<>(); @@ -330,14 +356,16 @@ public void evenDistributionInOneRound_unboundedMaxFree() { EvenScheduler.scheduleTopologiesEvenly(topologies, cluster); - // Idle supervisor absorbs exactly floor(8/3) = 2 workers in one round; total worker count is preserved. + // Idle supervisor absorbs exactly floor(8/3) = 2 workers in one round; total worker count + // is preserved. assertEquals(2, usedSlotCount(cluster, "sup-2")); assertEquals(8, usedSlotCount(cluster, "sup-0") + usedSlotCount(cluster, "sup-1") + usedSlotCount(cluster, "sup-2")); assertEquals(8, cluster.getAssignedNumWorkers(cluster.getTopologies().getById(topoId)), "relocation must preserve the declared worker count of an 8-worker topology"); - // No supervisor is idle anymore, so a second pass relocates nothing -- the trigger will not refire next round. + // No supervisor is idle anymore, so a second pass relocates nothing -- the trigger will not + // refire next round. EvenScheduler.redistributeOntoIdleSupervisors(cluster.getTopologies(), cluster); assertEquals(2, usedSlotCount(cluster, "sup-2")); assertEquals(8, usedSlotCount(cluster, "sup-0") @@ -346,8 +374,10 @@ public void evenDistributionInOneRound_unboundedMaxFree() { } /** - * Two equally-sized topologies share the same returning supervisor round-robin: each contributes one worker, so - * sup-2 ends up hosting workers from both — restoring per-supervisor workload diversity the way a fresh submission + * Two equally-sized topologies share the same returning supervisor round-robin: each + * contributes one worker, so + * sup-2 ends up hosting workers from both — restoring per-supervisor workload diversity the way + * a fresh submission * would. */ @Test @@ -359,12 +389,12 @@ public void multipleTopologies_shareIdleSlotsRoundRobin() { Map assignments = new HashMap<>(); assignments.put("topo-A", buildAssignment(topoA, new WorkerSlot[]{ - new WorkerSlot("sup-0", 0), new WorkerSlot("sup-0", 1), - new WorkerSlot("sup-1", 0), new WorkerSlot("sup-1", 1), + new WorkerSlot("sup-0", 0), new WorkerSlot("sup-0", 1), + new WorkerSlot("sup-1", 0), new WorkerSlot("sup-1", 1), })); assignments.put("topo-B", buildAssignment(topoB, new WorkerSlot[]{ - new WorkerSlot("sup-0", 2), new WorkerSlot("sup-0", 3), - new WorkerSlot("sup-1", 2), new WorkerSlot("sup-1", 3), + new WorkerSlot("sup-0", 2), new WorkerSlot("sup-0", 3), + new WorkerSlot("sup-1", 2), new WorkerSlot("sup-1", 3), })); Map topoMap = new HashMap<>(); @@ -376,8 +406,10 @@ public void multipleTopologies_shareIdleSlotsRoundRobin() { EvenScheduler.redistributeOntoIdleSupervisors(topologies, cluster); - // floor(4/3)=1 per topology, two topologies → sup-2 hosts 1 worker from each, in round-robin order. - // Exact counts (not >= 1) are what actually enforce round-robin fairness: a broken inner loop that let the + // floor(4/3)=1 per topology, two topologies → sup-2 hosts 1 worker from each, in + // round-robin order. + // Exact counts (not >= 1) are what actually enforce round-robin fairness: a broken inner + // loop that let the // first topology grab both idle slots would leave topo-B at 0 here. assertEquals(2, usedSlotCount(cluster, "sup-2")); assertEquals(1, supervisorWorkerCount(cluster, "topo-A", "sup-2")); @@ -385,21 +417,23 @@ public void multipleTopologies_shareIdleSlotsRoundRobin() { // Each topology kept its total worker count; only one host moved. assertEquals(4, cluster.getAssignedNumWorkers(topoA)); assertEquals(4, cluster.getAssignedNumWorkers(topoB)); - // Donor supervisors are never drained to zero (which would make them the next round's idle target). + // Donor supervisors are never drained to zero (which would make them the next round's idle + // target). assertTrue(usedSlotCount(cluster, "sup-0") > 0); assertTrue(usedSlotCount(cluster, "sup-1") > 0); } /** - * Flap-guard boundary: with 3 stable rounds at a 3s monitor frequency a returning supervisor must have been up for + * Flap-guard boundary: with 3 stable rounds at a 3s monitor frequency a returning supervisor + * must have been up for * at least 9 seconds before it is eligible. {@code uptime == requiredUptime} is the first value that moves, making * the off-by-one contract explicit: {@code uptimeSecs >= minStableRounds * monitorFrequencySecs}. */ @ParameterizedTest @CsvSource({ - "8, false", // threshold - 1: too young, stays idle - "9, true", // exactly at threshold: eligible - "10, true", // threshold + 1: eligible + "8, false", // threshold - 1: too young, stays idle + "9, true", // exactly at threshold: eligible + "10, true", // threshold + 1: eligible }) public void flapGuardHonorsMinStableRoundBoundary(long sup2UptimeSecs, boolean expectMove) { Map supMap = genSupervisorsWithUptime(3, 4, 100); @@ -412,8 +446,10 @@ public void flapGuardHonorsMinStableRoundBoundary(long sup2UptimeSecs, boolean e EvenScheduler.scheduleTopologiesEvenly(cluster.getTopologies(), cluster); - // 3 stable rounds at a 3 second monitor frequency require at least 9 seconds of supervisor uptime before sup-2 - // becomes an eligible target; the placement below is the observable expression of that boundary. + // 3 stable rounds at a 3 second monitor frequency require at least 9 seconds of supervisor + // uptime before sup-2 + // becomes an eligible target; the placement below is the observable expression of that + // boundary. if (expectMove) { assertEquals(1, usedSlotCount(cluster, "sup-2")); assertEquals(3, cluster.getAssignedNumWorkers(firstTopology(cluster))); @@ -431,8 +467,8 @@ public void donorTieBreaksBySupervisorIdWhenWorkerCountsTie() { Map assignments = new HashMap<>(); assignments.put("topo-tie", buildAssignment(topology, new WorkerSlot[]{ - new WorkerSlot("sup-0", 0), new WorkerSlot("sup-0", 1), - new WorkerSlot("sup-1", 0), new WorkerSlot("sup-1", 1), + new WorkerSlot("sup-0", 0), new WorkerSlot("sup-0", 1), + new WorkerSlot("sup-1", 0), new WorkerSlot("sup-1", 1), })); Map topoMap = new HashMap<>(); @@ -443,7 +479,8 @@ public void donorTieBreaksBySupervisorIdWhenWorkerCountsTie() { EvenScheduler.redistributeOntoIdleSupervisors(topologies, cluster); assertEquals(1, supervisorWorkerCount(cluster, "topo-tie", "sup-0"), - "sup-0 and sup-1 started with two workers each; lexicographic tie-break chooses sup-0 as donor"); + "sup-0 and sup-1 started with two workers each; lexicographic tie-break chooses " + + "sup-0 as donor"); assertEquals(2, supervisorWorkerCount(cluster, "topo-tie", "sup-1")); assertEquals(1, supervisorWorkerCount(cluster, "topo-tie", "sup-2")); assertEquals(4, cluster.getAssignedNumWorkers(topology), @@ -455,7 +492,8 @@ public void donorTieBreaksBySupervisorIdWhenWorkerCountsTie() { * {@code floor(numWorkers / nonBlacklistedSupervisorCount) * idleSupervisorCount} workers in one round, exercising the * {@code * idleSupervisorCount} term of the budget formula. With 4 non-blacklisted supervisors (two busy, two idle) * and an 8-worker topology the budget is {@code floor(8 / 4) * 2 = 4}; a regression that dropped the multiplier would - * compute 2 and relocate only half as many workers. Every other test has exactly one usable idle supervisor, so this + * compute 2 and relocate only half as many workers. Every other test has exactly one usable + * idle supervisor, so this * fixture is the one that pins the multiplier down. */ @Test @@ -466,10 +504,10 @@ public void twoIdleSupervisors_budgetScalesWithIdleSupervisorCount() { Map assignments = new HashMap<>(); assignments.put(topoId, buildAssignment(topology, new WorkerSlot[]{ - new WorkerSlot("sup-0", 0), new WorkerSlot("sup-0", 1), - new WorkerSlot("sup-0", 2), new WorkerSlot("sup-0", 3), - new WorkerSlot("sup-1", 0), new WorkerSlot("sup-1", 1), - new WorkerSlot("sup-1", 2), new WorkerSlot("sup-1", 3), + new WorkerSlot("sup-0", 0), new WorkerSlot("sup-0", 1), + new WorkerSlot("sup-0", 2), new WorkerSlot("sup-0", 3), + new WorkerSlot("sup-1", 0), new WorkerSlot("sup-1", 1), + new WorkerSlot("sup-1", 2), new WorkerSlot("sup-1", 3), })); Map topoMap = new HashMap<>(); @@ -484,11 +522,14 @@ public void twoIdleSupervisors_budgetScalesWithIdleSupervisorCount() { EvenScheduler.redistributeOntoIdleSupervisors(topologies, cluster); - // budget = floor(8 / 4 non-blacklisted) * 2 idle = 4 relocations onto the idle supervisors. Asserting the sum - // (not a single supervisor) keeps this independent of the idle-slot fill order. Dropping the * idleSupervisorCount + // budget = floor(8 / 4 non-blacklisted) * 2 idle = 4 relocations onto the idle supervisors. + // Asserting the sum + // (not a single supervisor) keeps this independent of the idle-slot fill order. Dropping + // the * idleSupervisorCount // term would compute budget 2 and move only 2 workers, failing this assertion. assertEquals(4, usedSlotCount(cluster, "sup-2") + usedSlotCount(cluster, "sup-3"), - "budget must scale with the number of simultaneously-idle supervisors: floor(8/4) * 2 = 4"); + "budget must scale with the number of simultaneously-idle supervisors: floor(8/4) " + + "* 2 = 4"); assertEquals(4, usedSlotCount(cluster, "sup-0") + usedSlotCount(cluster, "sup-1")); assertEquals(8, cluster.getAssignedNumWorkers(cluster.getTopologies().getById(topoId)), "relocation preserves the declared worker count"); @@ -498,7 +539,8 @@ public void twoIdleSupervisors_budgetScalesWithIdleSupervisorCount() { * {@code max.free.per.topology} must be able to bind more tightly than the even-distribution budget. With 3 * supervisors and a 6-worker topology the even budget is {@code floor(6 / 3) * 1 = 2}, but {@code maxFree = 1} clamps * it to a single relocation. This is the only fixture where the {@code Math.min(target, maxFree)} clamp is the - * strictly binding constraint -- removing the clamp would relocate 2 workers and push sup-2 to 2, failing the + * strictly binding constraint -- removing the clamp would relocate 2 workers and push sup-2 to + * 2, failing the * assertion below. */ @Test @@ -509,8 +551,8 @@ public void maxFreePerTopologyClampsBelowEvenBudget() { Map assignments = new HashMap<>(); assignments.put(topoId, buildAssignment(topology, new WorkerSlot[]{ - new WorkerSlot("sup-0", 0), new WorkerSlot("sup-0", 1), new WorkerSlot("sup-0", 2), - new WorkerSlot("sup-1", 0), new WorkerSlot("sup-1", 1), new WorkerSlot("sup-1", 2), + new WorkerSlot("sup-0", 0), new WorkerSlot("sup-0", 1), new WorkerSlot("sup-0", 2), + new WorkerSlot("sup-1", 0), new WorkerSlot("sup-1", 1), new WorkerSlot("sup-1", 2), })); Map topoMap = new HashMap<>(); @@ -521,7 +563,8 @@ public void maxFreePerTopologyClampsBelowEvenBudget() { EvenScheduler.redistributeOntoIdleSupervisors(topologies, cluster); - // even budget = floor(6/3)*1 = 2, but maxFree=1 clamps it to a single relocation. Without Math.min(target, + // even budget = floor(6/3)*1 = 2, but maxFree=1 clamps it to a single relocation. Without + // Math.min(target, // maxFree) two workers would move and sup-2 would hold 2. assertEquals(1, usedSlotCount(cluster, "sup-2"), "max.free.per.topology must clamp the even-distribution budget of 2 down to 1"); @@ -535,8 +578,10 @@ public void blacklistedIdleSupervisorIsNotReusableTarget() { Cluster cluster = buildClusterWithIdleSupervisor(true, 1); cluster.blacklistHost("host-2"); - assertFalse(cluster.isIdleSupervisorAvailableForEvenRebalance(cluster.getSupervisorById("sup-2")), - "IsolationScheduler represents reserved hosts by blacklisting them before delegating to DefaultScheduler"); + assertFalse(cluster.isIdleSupervisorAvailableForEvenRebalance(cluster + .getSupervisorById("sup-2")), + "IsolationScheduler represents reserved hosts by blacklisting them before " + + "delegating to DefaultScheduler"); EvenScheduler.scheduleTopologiesEvenly(cluster.getTopologies(), cluster); @@ -548,32 +593,39 @@ public void blacklistedIdleSupervisorIsNotReusableTarget() { } /** - * Regression for the apache/storm#8778 follow-up: in the {@link DefaultScheduler} path the per-topology + * Regression for the apache/storm#8778 follow-up: in the {@link DefaultScheduler} path the + * per-topology * {@code max.free.per.topology} cap must bind once per scheduling round, not once per redistribute call. * - *

      {@link DefaultScheduler#defaultSchedule(Topologies, Cluster)} runs the idle-supervisor redistribute once over - * the full topology set, then delegates to {@link EvenScheduler#scheduleTopologiesEvenly(Topologies, Cluster, boolean)} + *

      {@link DefaultScheduler#defaultSchedule(Topologies, Cluster)} runs the idle-supervisor + * redistribute once over + * the full topology set, then delegates to {@link + * EvenScheduler#scheduleTopologiesEvenly(Topologies, Cluster, boolean)} * once per under-assigned topology (now passing {@code redistributeOntoIdle=false}) -- the 2-arg overload it used to * call ran the redistribute a second time. With two idle supervisors the - * full-set pass fills one of them (consuming the cap), leaving the second idle for the per-topology pass to fill + * full-set pass fills one of them (consuming the cap), leaving the second idle for the + * per-topology pass to fill * again, so an under-assigned topology relocated up to {@code 2 * maxFree} workers in a single round. * - *

      The fixture makes the topology under-assigned by declaring more workers than it has slots (8 declared, 4 + *

      The fixture makes the topology under-assigned by declaring more workers than it has slots + * (8 declared, 4 * executors on 4 slots) so {@code needsScheduling} stays true and the delegated call fires, while leaving no - * executor unassigned -- the ordinary even-scheduling pass is then a no-op and only the redistribute relocations are + * executor unassigned -- the ordinary even-scheduling pass is then a no-op and only the + * redistribute relocations are * observable. sup-0 and sup-1 are donors with two workers each; sup-2 and sup-3 start idle. */ @Test public void defaultSchedulerAppliesMaxFreeCapOncePerRound() { String topoId = "topo-double-cap"; Map supMap = TestUtilsForBlacklistScheduler.genSupervisors(4, 4); - // 8 declared workers but only 4 executors: needsScheduling stays true (8 > 4 assigned) with nothing to reassign. + // 8 declared workers but only 4 executors: needsScheduling stays true (8 > 4 assigned) with + // nothing to reassign. TopologyDetails topology = makeTopologyDetails(topoId, 8, 2); Map assignments = new HashMap<>(); assignments.put(topoId, buildAssignment(topology, new WorkerSlot[]{ - new WorkerSlot("sup-0", 0), new WorkerSlot("sup-0", 1), - new WorkerSlot("sup-1", 0), new WorkerSlot("sup-1", 1), + new WorkerSlot("sup-0", 0), new WorkerSlot("sup-0", 1), + new WorkerSlot("sup-1", 0), new WorkerSlot("sup-1", 1), })); Map topoMap = new HashMap<>(); @@ -588,16 +640,20 @@ public void defaultSchedulerAppliesMaxFreeCapOncePerRound() { DefaultScheduler.defaultSchedule(new Topologies(topology), cluster); - // maxFree=1 caps relocation at one existing worker per round. The old double redistribute moved two -- one onto - // each idle supervisor; with the cap applied once per round only one idle supervisor is filled and the other + // maxFree=1 caps relocation at one existing worker per round. The old double redistribute + // moved two -- one onto + // each idle supervisor; with the cap applied once per round only one idle supervisor is + // filled and the other // stays untouched. int relocatedOntoIdle = supervisorWorkerCount(cluster, topoId, "sup-2") + supervisorWorkerCount(cluster, topoId, "sup-3"); assertEquals(1, relocatedOntoIdle, - "max.free.per.topology must cap idle-supervisor relocation at 1 per round, not 2 (apache/storm#8778 follow-up)"); + "max.free.per.topology must cap idle-supervisor relocation at 1 per round, not 2 " + + "(apache/storm#8778 follow-up)"); assertEquals(0, supervisorWorkerCount(cluster, topoId, "sup-3"), "the second idle supervisor stays idle once the per-round cap is reached"); - // The relocation moves existing workers only -- the assigned worker count is unchanged and no executor is lost. + // The relocation moves existing workers only -- the assigned worker count is unchanged and + // no executor is lost. assertEquals(4, cluster.getAssignedNumWorkers(cluster.getTopologies().getById(topoId)), "relocation must preserve the topology's four assigned workers"); } @@ -610,27 +666,30 @@ public void defaultSchedulerIdleRebalanceHonorsLeftoverTopologySubset() { Map assignments = new HashMap<>(); assignments.put(isolated.getId(), buildAssignment(isolated, new WorkerSlot[]{ - new WorkerSlot("sup-0", 0), new WorkerSlot("sup-0", 1), + new WorkerSlot("sup-0", 0), new WorkerSlot("sup-0", 1), })); assignments.put(regular.getId(), buildAssignment(regular, new WorkerSlot[]{ - new WorkerSlot("sup-1", 0), new WorkerSlot("sup-1", 1), new WorkerSlot("sup-1", 2), + new WorkerSlot("sup-1", 0), new WorkerSlot("sup-1", 1), new WorkerSlot("sup-1", 2), })); Map topoMap = new HashMap<>(); topoMap.put(isolated.getId(), isolated); topoMap.put(regular.getId(), regular); Topologies allTopologies = new Topologies(topoMap); - Cluster cluster = newCluster(supMap, assignments, allTopologies, evenRebalanceConf(true, 0)); + Cluster cluster = newCluster(supMap, assignments, allTopologies, evenRebalanceConf(true, + 0)); cluster.blacklistHost("host-0"); DefaultScheduler.defaultSchedule(new Topologies(regular), cluster); assertEquals(2, supervisorWorkerCount(cluster, isolated.getId(), "sup-0"), - "the isolated topology is not in the leftover topology set, so DefaultScheduler must not move it"); + "the isolated topology is not in the leftover topology set, so DefaultScheduler " + + "must not move it"); assertEquals(0, supervisorWorkerCount(cluster, isolated.getId(), "sup-2")); assertEquals(2, supervisorWorkerCount(cluster, regular.getId(), "sup-1")); assertEquals(1, supervisorWorkerCount(cluster, regular.getId(), "sup-2")); - // Both topologies keep their declared worker counts: the leftover one is relocated, the excluded one untouched. + // Both topologies keep their declared worker counts: the leftover one is relocated, the + // excluded one untouched. assertEquals(3, cluster.getAssignedNumWorkers(regular)); assertEquals(2, cluster.getAssignedNumWorkers(isolated)); } @@ -643,10 +702,10 @@ public void isolationSchedulerOnlyRelocatesLeftoverTopologyOntoNonIsolatedIdleSu Map assignments = new HashMap<>(); assignments.put(isolated.getId(), buildAssignment(isolated, new WorkerSlot[]{ - new WorkerSlot("sup-0", 0), + new WorkerSlot("sup-0", 0), })); assignments.put(regular.getId(), buildAssignment(regular, new WorkerSlot[]{ - new WorkerSlot("sup-1", 0), new WorkerSlot("sup-1", 1), new WorkerSlot("sup-1", 2), + new WorkerSlot("sup-1", 0), new WorkerSlot("sup-1", 1), new WorkerSlot("sup-1", 2), })); Map topoMap = new HashMap<>(); @@ -655,7 +714,8 @@ public void isolationSchedulerOnlyRelocatesLeftoverTopologyOntoNonIsolatedIdleSu Topologies topologies = new Topologies(topoMap); Map conf = evenRebalanceConf(true, 0); - conf.put(DaemonConfig.ISOLATION_SCHEDULER_MACHINES, Collections.singletonMap(isolated.getName(), 1)); + conf.put(DaemonConfig.ISOLATION_SCHEDULER_MACHINES, Collections.singletonMap(isolated + .getName(), 1)); Cluster cluster = newCluster(supMap, assignments, topologies, conf); IsolationScheduler scheduler = new IsolationScheduler(); @@ -663,21 +723,28 @@ public void isolationSchedulerOnlyRelocatesLeftoverTopologyOntoNonIsolatedIdleSu scheduler.schedule(topologies, cluster); assertEquals(1, supervisorWorkerCount(cluster, isolated.getId(), "sup-0"), - "the already-isolated topology remains on its isolated host and is not selected as a donor"); + "the already-isolated topology remains on its isolated host and is not selected " + + "as a donor"); assertEquals(0, supervisorWorkerCount(cluster, isolated.getId(), "sup-2")); assertEquals(2, supervisorWorkerCount(cluster, regular.getId(), "sup-1")); assertEquals(1, supervisorWorkerCount(cluster, regular.getId(), "sup-2"), - "only the leftover regular topology is allowed to move onto the non-isolated idle supervisor"); - // The relocated leftover topology and the untouched isolated topology both keep their declared worker counts. + "only the leftover regular topology is allowed to move onto the non-isolated idle " + + "supervisor"); + // The relocated leftover topology and the untouched isolated topology both keep their + // declared worker counts. assertEquals(3, cluster.getAssignedNumWorkers(regular)); assertEquals(1, cluster.getAssignedNumWorkers(isolated)); } /** - * IsolationScheduler reserves a host by blacklisting it before delegating the remaining (non-isolated) topologies to - * {@link DefaultScheduler#defaultSchedule(Topologies, Cluster)}. Here the isolated topology is down -- it has no - * assigned workers at all -- yet its reserved host (sup-2) must not be treated as an idle relocation target even - * though it has zero used slots. The leftover regular topology is rebalanced onto the genuinely idle, non-reserved + * IsolationScheduler reserves a host by blacklisting it before delegating the remaining + * (non-isolated) topologies to + * {@link DefaultScheduler#defaultSchedule(Topologies, Cluster)}. Here the isolated topology is + * down -- it has no + * assigned workers at all -- yet its reserved host (sup-2) must not be treated as an idle + * relocation target even + * though it has zero used slots. The leftover regular topology is rebalanced onto the genuinely + * idle, non-reserved * sup-3 and never onto the reserved sup-2. */ @Test @@ -689,33 +756,41 @@ public void reservedHostForDownIsolatedTopologyIsNotTreatedAsIdle() { Map assignments = new HashMap<>(); // The isolated topology is down: it has no assignment at all. assignments.put(regular.getId(), buildAssignment(regular, new WorkerSlot[]{ - new WorkerSlot("sup-0", 0), new WorkerSlot("sup-0", 1), - new WorkerSlot("sup-1", 0), new WorkerSlot("sup-1", 1), + new WorkerSlot("sup-0", 0), new WorkerSlot("sup-0", 1), + new WorkerSlot("sup-1", 0), new WorkerSlot("sup-1", 1), })); Map topoMap = new HashMap<>(); topoMap.put(isolated.getId(), isolated); topoMap.put(regular.getId(), regular); Topologies allTopologies = new Topologies(topoMap); - Cluster cluster = newCluster(supMap, assignments, allTopologies, evenRebalanceConf(true, 0)); + Cluster cluster = newCluster(supMap, assignments, allTopologies, evenRebalanceConf(true, + 0)); - // sup-2 is reserved for the (down) isolated topology -- IsolationScheduler represents this by blacklisting it. + // sup-2 is reserved for the (down) isolated topology -- IsolationScheduler represents this + // by blacklisting it. cluster.blacklistHost("host-2"); - assertFalse(cluster.isIdleSupervisorAvailableForEvenRebalance(cluster.getSupervisorById("sup-2")), - "a blacklisted reserved host is never an even-rebalance target, even with zero used slots"); - assertTrue(cluster.isIdleSupervisorAvailableForEvenRebalance(cluster.getSupervisorById("sup-3")), + assertFalse(cluster.isIdleSupervisorAvailableForEvenRebalance(cluster + .getSupervisorById("sup-2")), + "a blacklisted reserved host is never an even-rebalance target, even with zero " + + "used slots"); + assertTrue(cluster.isIdleSupervisorAvailableForEvenRebalance(cluster + .getSupervisorById("sup-3")), "the non-reserved idle supervisor is available"); - // IsolationScheduler delegates the leftover (non-isolated) topologies to DefaultScheduler with the reserved + // IsolationScheduler delegates the leftover (non-isolated) topologies to DefaultScheduler + // with the reserved // host already blacklisted. DefaultScheduler.defaultSchedule(new Topologies(regular), cluster); assertEquals(0, usedSlotCount(cluster, "sup-2"), - "the reserved host stays idle: the down isolated topology's machine is not repopulated by rebalance"); + "the reserved host stays idle: the down isolated topology's machine is not " + + "repopulated by rebalance"); assertEquals(0, supervisorWorkerCount(cluster, regular.getId(), "sup-2")); assertEquals(1, supervisorWorkerCount(cluster, regular.getId(), "sup-3"), - "the leftover regular topology rebalances onto the genuinely idle, non-reserved supervisor"); + "the leftover regular topology rebalances onto the genuinely idle, non-reserved " + + "supervisor"); assertEquals(1, supervisorWorkerCount(cluster, regular.getId(), "sup-0")); assertEquals(2, supervisorWorkerCount(cluster, regular.getId(), "sup-1")); assertEquals(4, cluster.getAssignedNumWorkers(regular), diff --git a/storm-server/src/test/java/org/apache/storm/scheduler/blacklist/FaultGenerateUtils.java b/storm-server/src/test/java/org/apache/storm/scheduler/blacklist/FaultGenerateUtils.java index 9a34080a5a3..aec9863e88d 100644 --- a/storm-server/src/test/java/org/apache/storm/scheduler/blacklist/FaultGenerateUtils.java +++ b/storm-server/src/test/java/org/apache/storm/scheduler/blacklist/FaultGenerateUtils.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

      Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -17,33 +17,36 @@ */ package org.apache.storm.scheduler.blacklist; -import org.apache.storm.scheduler.Cluster; -import org.apache.storm.scheduler.INimbus; -import org.apache.storm.scheduler.SchedulerAssignmentImpl; -import org.apache.storm.scheduler.SupervisorDetails; -import org.apache.storm.scheduler.Topologies; - import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import org.apache.storm.metric.StormMetricsRegistry; +import org.apache.storm.scheduler.Cluster; +import org.apache.storm.scheduler.INimbus; +import org.apache.storm.scheduler.SchedulerAssignmentImpl; +import org.apache.storm.scheduler.SupervisorDetails; +import org.apache.storm.scheduler.Topologies; import org.apache.storm.scheduler.resource.normalization.ResourceMetrics; public class FaultGenerateUtils { - public static List> getSupervisorsList(int supervisorCount, int slotCount, List>> faultList) { + public static List> getSupervisorsList(int supervisorCount, + int slotCount, List>> faultList) { List> supervisorsList = new ArrayList<>(faultList.size()); for (Map> faults : faultList) { - Map supervisors = TestUtilsForBlacklistScheduler.genSupervisors(supervisorCount, slotCount); + Map supervisors = TestUtilsForBlacklistScheduler + .genSupervisors(supervisorCount, slotCount); for (Map.Entry> fault : faults.entrySet()) { int supervisor = fault.getKey(); List slots = fault.getValue(); if (slots.isEmpty()) { - supervisors = TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supervisors, "sup-" + supervisor); + supervisors = TestUtilsForBlacklistScheduler + .removeSupervisorFromSupervisors(supervisors, "sup-" + supervisor); } else { for (int slot : slots) { - supervisors = TestUtilsForBlacklistScheduler.removePortFromSupervisors(supervisors, "sup-" + supervisor, slot); + supervisors = TestUtilsForBlacklistScheduler + .removePortFromSupervisors(supervisors, "sup-" + supervisor, slot); } } } @@ -52,14 +55,17 @@ public static List> getSupervisorsList(int superv return supervisorsList; } - public static Cluster nextCluster(Cluster cluster, Map supervisors, INimbus iNimbus, Map config, + public static Cluster nextCluster(Cluster cluster, Map supervisors, + INimbus iNimbus, Map config, Topologies topologies) { Map assignment; if (cluster == null) { assignment = new HashMap<>(); } else { - assignment = TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()); + assignment = TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster + .getAssignments()); } - return new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supervisors, assignment, topologies, config); + return new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supervisors, + assignment, topologies, config); } } diff --git a/storm-server/src/test/java/org/apache/storm/scheduler/blacklist/TestBlacklistScheduler.java b/storm-server/src/test/java/org/apache/storm/scheduler/blacklist/TestBlacklistScheduler.java index abc5f25a289..0f3d997c8cf 100644 --- a/storm-server/src/test/java/org/apache/storm/scheduler/blacklist/TestBlacklistScheduler.java +++ b/storm-server/src/test/java/org/apache/storm/scheduler/blacklist/TestBlacklistScheduler.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

      Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -17,9 +17,20 @@ */ package org.apache.storm.scheduler.blacklist; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Sets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.core.config.Configurator; @@ -30,6 +41,7 @@ import org.apache.storm.generated.InvalidTopologyException; import org.apache.storm.generated.SpoutSpec; import org.apache.storm.generated.StormTopology; +import org.apache.storm.metric.StormMetricsRegistry; import org.apache.storm.scheduler.Cluster; import org.apache.storm.scheduler.DefaultScheduler; import org.apache.storm.scheduler.ExecutorDetails; @@ -41,6 +53,7 @@ import org.apache.storm.scheduler.TopologyDetails; import org.apache.storm.scheduler.resource.ResourceAwareScheduler; import org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler; +import org.apache.storm.scheduler.resource.normalization.ResourceMetrics; import org.apache.storm.scheduler.resource.strategies.scheduling.DefaultResourceAwareStrategy; import org.apache.storm.scheduler.resource.strategies.scheduling.DefaultResourceAwareStrategyOld; import org.apache.storm.scheduler.resource.strategies.scheduling.GenericResourceAwareStrategy; @@ -55,20 +68,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; -import java.util.List; -import java.util.ArrayList; -import org.apache.storm.metric.StormMetricsRegistry; -import org.apache.storm.scheduler.resource.normalization.ResourceMetrics; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - public class TestBlacklistScheduler { private static final Logger LOG = LoggerFactory.getLogger(TestBlacklistScheduler.class); @@ -87,7 +86,8 @@ public void cleanup() { @Test public void testBlacklistResumeWhenAckersWontFit() throws InvalidTopologyException { // 3 supervisors exist with 4 slots, 2 are blacklisted - // topology with given worker heap size would fit in 4 slots if ignoring ackers, needs 5 slots with ackers. + // topology with given worker heap size would fit in 4 slots if ignoring ackers, needs 5 + // slots with ackers. // verify that one of the supervisors will be resumed and topology will schedule. Config config = new Config(); @@ -100,11 +100,14 @@ public void testBlacklistResumeWhenAckersWontFit() throws InvalidTopologyExcepti config.put(DaemonConfig.BLACKLIST_SCHEDULER_ASSUME_SUPERVISOR_BAD_BASED_ON_BAD_SLOT, false); config.put(DaemonConfig.BLACKLIST_SCHEDULER_TOLERANCE_COUNT, 3); config.put(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB, 128); - config.put(DaemonConfig.BLACKLIST_SCHEDULER_STRATEGY, "org.apache.storm.scheduler.blacklist.strategies.RasBlacklistStrategy"); + config.put(DaemonConfig.BLACKLIST_SCHEDULER_STRATEGY, + "org.apache.storm.scheduler.blacklist.strategies.RasBlacklistStrategy"); config.put(Config.TOPOLOGY_RAS_ACKER_EXECUTORS_PER_WORKER, 1); config.setNumWorkers(1); config.put(Config.TOPOLOGY_ACKER_EXECUTORS, 4); - config.put(Config.TOPOLOGY_SCHEDULER_STRATEGY, "org.apache.storm.scheduler.resource.strategies.scheduling.GenericResourceAwareStrategy"); + config.put(Config.TOPOLOGY_SCHEDULER_STRATEGY, + "org.apache.storm.scheduler.resource.strategies.scheduling.GenericResourceAwareStr" + + "ategy"); config.put(Config.TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB, 512); config.put(Config.WORKER_HEAP_MEMORY_MB, 768); config.put(Config.TOPOLOGY_NAME, "testTopology"); @@ -112,19 +115,24 @@ public void testBlacklistResumeWhenAckersWontFit() throws InvalidTopologyExcepti INimbus iNimbus = new TestUtilsForBlacklistScheduler.INimbusTest(); StormMetricsRegistry metricsRegistry = new StormMetricsRegistry(); ResourceMetrics resourceMetrics = new ResourceMetrics(metricsRegistry); - Map supMap = TestUtilsForBlacklistScheduler.genSupervisors(3, 4, 400.0d, 4096.0d); + Map supMap = TestUtilsForBlacklistScheduler.genSupervisors(3, 4, + 400.0d, 4096.0d); Topologies noTopologies = new Topologies(); - Cluster cluster = new Cluster(iNimbus, resourceMetrics, supMap, new HashMap<>(), noTopologies, config); + Cluster cluster = new Cluster(iNimbus, resourceMetrics, supMap, new HashMap<>(), + noTopologies, config); scheduler = new BlacklistScheduler(new ResourceAwareScheduler()); scheduler.prepare(config, metricsRegistry); scheduler.schedule(noTopologies, cluster); - Map removedSup0 = TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap, "sup-0"); - Map removedSup0Sup1 = TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(removedSup0, "sup-1"); + Map removedSup0 = TestUtilsForBlacklistScheduler + .removeSupervisorFromSupervisors(supMap, "sup-0"); + Map removedSup0Sup1 = TestUtilsForBlacklistScheduler + .removeSupervisorFromSupervisors(removedSup0, "sup-1"); cluster = new Cluster(iNimbus, resourceMetrics, removedSup0Sup1, - TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), noTopologies, config); + TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster + .getAssignments()), noTopologies, config); scheduler.schedule(noTopologies, cluster); scheduler.schedule(noTopologies, cluster); scheduler.schedule(noTopologies, cluster); @@ -137,7 +145,8 @@ public void testBlacklistResumeWhenAckersWontFit() throws InvalidTopologyExcepti Topologies topologies = new Topologies(topoMap); cluster = new Cluster(iNimbus, resourceMetrics, supMap, - TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config); + TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster + .getAssignments()), topologies, config); boolean enableTraceLogging = false; // for scheduling debug if (enableTraceLogging) { Configurator.setAllLevels(LogManager.getRootLogger().getName(), Level.TRACE); @@ -164,7 +173,8 @@ public TopologyDetails createResourceTopo(Config conf) throws InvalidTopologyExc StormTopology stormTopology = builder.createTopology(); TopologyDetails topo = new TopologyDetails("testTopology-id", conf, stormTopology, 0, - genExecsAndComps(StormCommon.systemTopology(conf, stormTopology)), currentTime, "user"); + genExecsAndComps(StormCommon.systemTopology(conf, + stormTopology)), currentTime, "user"); return topo; } @@ -210,21 +220,28 @@ public void TestBadSupervisor() { Map topoMap = new HashMap<>(); - TopologyDetails topo1 = TestUtilsForBlacklistScheduler.getTopology("topo-1", config, 5, 15, 1, 1, currentTime - 2, true); + TopologyDetails topo1 = TestUtilsForBlacklistScheduler.getTopology("topo-1", config, 5, 15, + 1, 1, currentTime - 2, true); topoMap.put(topo1.getId(), topo1); Topologies topologies = new Topologies(topoMap); StormMetricsRegistry metricsRegistry = new StormMetricsRegistry(); ResourceMetrics resourceMetrics = new ResourceMetrics(metricsRegistry); - Cluster cluster = new Cluster(iNimbus, resourceMetrics, supMap, new HashMap<>(), topologies, config); + Cluster cluster = new Cluster(iNimbus, resourceMetrics, supMap, new HashMap<>(), topologies, + config); scheduler = new BlacklistScheduler(new DefaultScheduler()); scheduler.prepare(config, metricsRegistry); scheduler.schedule(topologies, cluster); - cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap, "sup-0"), TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config); + cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler + .removeSupervisorFromSupervisors(supMap, "sup-0"), TestUtilsForBlacklistScheduler + .assignmentMapToImpl(cluster.getAssignments()), topologies, config); scheduler.schedule(topologies, cluster); - cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap, "sup-0"), TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config); + cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler + .removeSupervisorFromSupervisors(supMap, "sup-0"), TestUtilsForBlacklistScheduler + .assignmentMapToImpl(cluster.getAssignments()), topologies, config); scheduler.schedule(topologies, cluster); - cluster = new Cluster(iNimbus, resourceMetrics, supMap, TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config); + cluster = new Cluster(iNimbus, resourceMetrics, supMap, TestUtilsForBlacklistScheduler + .assignmentMapToImpl(cluster.getAssignments()), topologies, config); scheduler.schedule(topologies, cluster); assertEquals(Collections.singleton("host-0"), cluster.getBlacklistedHosts(), "blacklist"); } @@ -241,32 +258,41 @@ public void TestBadSlot(boolean blacklistOnBadSlot) { config.put(DaemonConfig.BLACKLIST_SCHEDULER_TOLERANCE_TIME, 200); config.put(DaemonConfig.BLACKLIST_SCHEDULER_TOLERANCE_COUNT, 2); config.put(DaemonConfig.BLACKLIST_SCHEDULER_RESUME_TIME, 300); - config.put(DaemonConfig.BLACKLIST_SCHEDULER_ASSUME_SUPERVISOR_BAD_BASED_ON_BAD_SLOT, blacklistOnBadSlot); + config.put(DaemonConfig.BLACKLIST_SCHEDULER_ASSUME_SUPERVISOR_BAD_BASED_ON_BAD_SLOT, + blacklistOnBadSlot); Map topoMap = new HashMap<>(); - TopologyDetails topo1 = TestUtilsForBlacklistScheduler.getTopology("topo-1", config, 5, 15, 1, 1, currentTime - 2, true); + TopologyDetails topo1 = TestUtilsForBlacklistScheduler.getTopology("topo-1", config, 5, 15, + 1, 1, currentTime - 2, true); topoMap.put(topo1.getId(), topo1); Topologies topologies = new Topologies(topoMap); StormMetricsRegistry metricsRegistry = new StormMetricsRegistry(); ResourceMetrics resourceMetrics = new ResourceMetrics(metricsRegistry); - Cluster cluster = new Cluster(iNimbus, resourceMetrics, supMap, new HashMap(), topologies, config); + Cluster cluster = new Cluster(iNimbus, resourceMetrics, supMap, new HashMap(), topologies, config); scheduler = new BlacklistScheduler(new DefaultScheduler()); scheduler.prepare(config, metricsRegistry); scheduler.schedule(topologies, cluster); - cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.removePortFromSupervisors(supMap, - "sup-0", 0), TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config); + cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler + .removePortFromSupervisors(supMap, + "sup-0", 0), TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster + .getAssignments()), topologies, config); scheduler.schedule(topologies, cluster); - cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.removePortFromSupervisors(supMap, "sup-0", 0), - TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config); + cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler + .removePortFromSupervisors(supMap, "sup-0", 0), + TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster + .getAssignments()), topologies, config); scheduler.schedule(topologies, cluster); - cluster = new Cluster(iNimbus, resourceMetrics, supMap, new HashMap(), topologies, config); + cluster = new Cluster(iNimbus, resourceMetrics, supMap, new HashMap(), topologies, config); scheduler.schedule(topologies, cluster); if (blacklistOnBadSlot) { - assertEquals(Collections.singleton("host-0"), cluster.getBlacklistedHosts(), "blacklist"); + assertEquals(Collections.singleton("host-0"), cluster.getBlacklistedHosts(), + "blacklist"); } else { assertEquals(Collections.emptySet(), cluster.getBlacklistedHosts(), "blacklist"); } @@ -286,21 +312,28 @@ public void TestResumeBlacklist() { Map topoMap = new HashMap<>(); - TopologyDetails topo1 = TestUtilsForBlacklistScheduler.getTopology("topo-1", config, 5, 15, 1, 1, currentTime - 2, true); + TopologyDetails topo1 = TestUtilsForBlacklistScheduler.getTopology("topo-1", config, 5, 15, + 1, 1, currentTime - 2, true); topoMap.put(topo1.getId(), topo1); Topologies topologies = new Topologies(topoMap); StormMetricsRegistry metricsRegistry = new StormMetricsRegistry(); ResourceMetrics resourceMetrics = new ResourceMetrics(metricsRegistry); - Cluster cluster = new Cluster(iNimbus, resourceMetrics, supMap, new HashMap(), topologies, config); + Cluster cluster = new Cluster(iNimbus, resourceMetrics, supMap, new HashMap(), topologies, config); scheduler = new BlacklistScheduler(new DefaultScheduler()); scheduler.prepare(config, metricsRegistry); scheduler.schedule(topologies, cluster); - cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap, "sup-0"), TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config); + cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler + .removeSupervisorFromSupervisors(supMap, "sup-0"), TestUtilsForBlacklistScheduler + .assignmentMapToImpl(cluster.getAssignments()), topologies, config); scheduler.schedule(topologies, cluster); - cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap, "sup-0"), TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config); + cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler + .removeSupervisorFromSupervisors(supMap, "sup-0"), TestUtilsForBlacklistScheduler + .assignmentMapToImpl(cluster.getAssignments()), topologies, config); scheduler.schedule(topologies, cluster); - cluster = new Cluster(iNimbus, resourceMetrics, supMap, new HashMap(), topologies, config); + cluster = new Cluster(iNimbus, resourceMetrics, supMap, new HashMap(), topologies, config); scheduler.schedule(topologies, cluster); assertEquals(Collections.singleton("host-0"), cluster.getBlacklistedHosts(), "blacklist"); for (int i = 0; i < 300 / 10 - 2; i++) { @@ -325,31 +358,42 @@ public void TestReleaseBlacklist() { Map topoMap = new HashMap<>(); - TopologyDetails topo1 = TestUtilsForBlacklistScheduler.getTopology("topo-1", config, 5, 15, 1, 1, currentTime - 2, true); - TopologyDetails topo2 = TestUtilsForBlacklistScheduler.getTopology("topo-2", config, 5, 15, 1, 1, currentTime - 8, true); - TopologyDetails topo3 = TestUtilsForBlacklistScheduler.getTopology("topo-3", config, 5, 15, 1, 1, currentTime - 16, true); - TopologyDetails topo4 = TestUtilsForBlacklistScheduler.getTopology("topo-4", config, 5, 15, 1, 1, currentTime - 32, true); + TopologyDetails topo1 = TestUtilsForBlacklistScheduler.getTopology("topo-1", config, 5, 15, + 1, 1, currentTime - 2, true); + TopologyDetails topo2 = TestUtilsForBlacklistScheduler.getTopology("topo-2", config, 5, 15, + 1, 1, currentTime - 8, true); + TopologyDetails topo3 = TestUtilsForBlacklistScheduler.getTopology("topo-3", config, 5, 15, + 1, 1, currentTime - 16, true); + TopologyDetails topo4 = TestUtilsForBlacklistScheduler.getTopology("topo-4", config, 5, 15, + 1, 1, currentTime - 32, true); topoMap.put(topo1.getId(), topo1); Topologies topologies = new Topologies(topoMap); StormMetricsRegistry metricsRegistry = new StormMetricsRegistry(); ResourceMetrics resourceMetrics = new ResourceMetrics(metricsRegistry); - Cluster cluster = new Cluster(iNimbus, resourceMetrics, supMap, new HashMap(), topologies, config); + Cluster cluster = new Cluster(iNimbus, resourceMetrics, supMap, new HashMap(), topologies, config); scheduler = new BlacklistScheduler(new DefaultScheduler()); scheduler.prepare(config, metricsRegistry); scheduler.schedule(topologies, cluster); - cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap, "sup-0"), TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config); + cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler + .removeSupervisorFromSupervisors(supMap, "sup-0"), TestUtilsForBlacklistScheduler + .assignmentMapToImpl(cluster.getAssignments()), topologies, config); scheduler.schedule(topologies, cluster); - cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap, "sup-0"), TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config); + cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler + .removeSupervisorFromSupervisors(supMap, "sup-0"), TestUtilsForBlacklistScheduler + .assignmentMapToImpl(cluster.getAssignments()), topologies, config); scheduler.schedule(topologies, cluster); - cluster = new Cluster(iNimbus, resourceMetrics, supMap, TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config); + cluster = new Cluster(iNimbus, resourceMetrics, supMap, TestUtilsForBlacklistScheduler + .assignmentMapToImpl(cluster.getAssignments()), topologies, config); scheduler.schedule(topologies, cluster); assertEquals(Collections.singleton("host-0"), cluster.getBlacklistedHosts(), "blacklist"); topoMap.put(topo2.getId(), topo2); topoMap.put(topo3.getId(), topo3); topoMap.put(topo4.getId(), topo4); topologies = new Topologies(topoMap); - cluster = new Cluster(iNimbus, resourceMetrics, supMap, TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config); + cluster = new Cluster(iNimbus, resourceMetrics, supMap, TestUtilsForBlacklistScheduler + .assignmentMapToImpl(cluster.getAssignments()), topologies, config); scheduler.schedule(topologies, cluster); assertEquals(Collections.emptySet(), cluster.getBlacklistedHosts(), "blacklist"); } @@ -371,50 +415,69 @@ public void TestGreylist() { config.put(Config.TOPOLOGY_RAS_ONE_EXECUTOR_PER_WORKER, true); Class[] strategyClasses = { - DefaultResourceAwareStrategy.class, - DefaultResourceAwareStrategyOld.class, - RoundRobinResourceAwareStrategy.class, - GenericResourceAwareStrategy.class, - GenericResourceAwareStrategyOld.class, + DefaultResourceAwareStrategy.class, + DefaultResourceAwareStrategyOld.class, + RoundRobinResourceAwareStrategy.class, + GenericResourceAwareStrategy.class, + GenericResourceAwareStrategyOld.class, }; - for (Class strategyClass: strategyClasses) { + for (Class strategyClass : strategyClasses) { String strategyClassName = strategyClass.getName(); config.put(Config.TOPOLOGY_SCHEDULER_STRATEGY, strategyClassName); { Map topoMap = new HashMap<>(); - TopologyDetails topo1 = TestUtilsForBlacklistScheduler.getTopology("topo-1", config, 1, 1, 1, 1, currentTime - 2, true); - TopologyDetails topo2 = TestUtilsForBlacklistScheduler.getTopology("topo-2", config, 1, 1, 1, 1, currentTime - 8, true); + TopologyDetails topo1 = TestUtilsForBlacklistScheduler.getTopology("topo-1", config, + 1, 1, 1, 1, currentTime - 2, true); + TopologyDetails topo2 = TestUtilsForBlacklistScheduler.getTopology("topo-2", config, + 1, 1, 1, 1, currentTime - 8, true); Topologies topologies = new Topologies(topoMap); StormMetricsRegistry metricsRegistry = new StormMetricsRegistry(); ResourceMetrics resourceMetrics = new ResourceMetrics(metricsRegistry); - Cluster cluster = new Cluster(iNimbus, resourceMetrics, supMap, new HashMap(), topologies, config); + Cluster cluster = new Cluster(iNimbus, resourceMetrics, supMap, new HashMap(), topologies, config); scheduler = new BlacklistScheduler(new ResourceAwareScheduler()); scheduler.prepare(config, metricsRegistry); scheduler.schedule(topologies, cluster); - cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap, "sup-0"), TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config); + cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler + .removeSupervisorFromSupervisors(supMap, + "sup-0"), TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster + .getAssignments()), topologies, config); scheduler.schedule(topologies, cluster); - cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap, "sup-0"), TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config); + cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler + .removeSupervisorFromSupervisors(supMap, + "sup-0"), TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster + .getAssignments()), topologies, config); scheduler.schedule(topologies, cluster); - cluster = new Cluster(iNimbus, resourceMetrics, supMap, TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config); + cluster = new Cluster(iNimbus, resourceMetrics, supMap, + TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster + .getAssignments()), topologies, config); scheduler.schedule(topologies, cluster); - assertEquals(Collections.singleton("host-0"), cluster.getBlacklistedHosts(), "blacklist"); + assertEquals(Collections.singleton("host-0"), cluster.getBlacklistedHosts(), + "blacklist"); topoMap.put(topo1.getId(), topo1); topoMap.put(topo2.getId(), topo2); topologies = new Topologies(topoMap); - cluster = new Cluster(iNimbus, resourceMetrics, supMap, TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config); + cluster = new Cluster(iNimbus, resourceMetrics, supMap, + TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster + .getAssignments()), topologies, config); scheduler.schedule(topologies, cluster); - assertEquals(Collections.emptySet(), cluster.getBlacklistedHosts(), "blacklist using " + strategyClassName); - assertEquals(Collections.singletonList("sup-0"), cluster.getGreyListedSupervisors(), "greylist using" + strategyClassName); - LOG.debug("{}: Now only these slots remain available: {}", strategyClassName, cluster.getAvailableSlots()); + assertEquals(Collections.emptySet(), cluster.getBlacklistedHosts(), + "blacklist using " + strategyClassName); + assertEquals(Collections.singletonList("sup-0"), cluster.getGreyListedSupervisors(), + "greylist using" + strategyClassName); + LOG.debug("{}: Now only these slots remain available: {}", strategyClassName, + cluster.getAvailableSlots()); if (strategyClass == RoundRobinResourceAwareStrategy.class) { // available slots will be across supervisors - assertFalse(cluster.getAvailableSlots(supMap.get("sup-0")).containsAll(cluster.getAvailableSlots()), "using " + strategyClassName); + assertFalse(cluster.getAvailableSlots(supMap.get("sup-0")).containsAll(cluster + .getAvailableSlots()), "using " + strategyClassName); } else { - assertTrue(cluster.getAvailableSlots(supMap.get("sup-0")).containsAll(cluster.getAvailableSlots()), "using " + strategyClassName); + assertTrue(cluster.getAvailableSlots(supMap.get("sup-0")).containsAll(cluster + .getAvailableSlots()), "using " + strategyClassName); } } } @@ -430,8 +493,10 @@ public void TestList() { config.put(DaemonConfig.BLACKLIST_SCHEDULER_RESUME_TIME, 300); Map topoMap = new HashMap<>(); - TopologyDetails topo1 = TestUtilsForBlacklistScheduler.getTopology("topo-1", config, 5, 15, 1, 1, currentTime - 2, true); - TopologyDetails topo2 = TestUtilsForBlacklistScheduler.getTopology("topo-2", config, 5, 15, 1, 1, currentTime - 2, true); + TopologyDetails topo1 = TestUtilsForBlacklistScheduler.getTopology("topo-1", config, 5, 15, + 1, 1, currentTime - 2, true); + TopologyDetails topo2 = TestUtilsForBlacklistScheduler.getTopology("topo-2", config, 5, 15, + 1, 1, currentTime - 2, true); topoMap.put(topo1.getId(), topo1); topoMap.put(topo2.getId(), topo2); Topologies topologies = new Topologies(topoMap); @@ -457,11 +522,13 @@ public void TestList() { faultList.add(new HashMap<>()); } - List> supervisorsList = FaultGenerateUtils.getSupervisorsList(3, 4, faultList); + List> supervisorsList = FaultGenerateUtils + .getSupervisorsList(3, 4, faultList); Cluster cluster = null; int count = 0; for (Map supervisors : supervisorsList) { - cluster = FaultGenerateUtils.nextCluster(cluster, supervisors, iNimbus, config, topologies); + cluster = FaultGenerateUtils.nextCluster(cluster, supervisors, iNimbus, config, + topologies); scheduler.schedule(topologies, cluster); if (count == 0) { Set hosts = new HashSet<>(); @@ -505,43 +572,50 @@ public void TestList() { } @Test - public void removeLongTimeDisappearFromCache(){ + public void removeLongTimeDisappearFromCache() { INimbus iNimbus = new TestUtilsForBlacklistScheduler.INimbusTest(); - Map supMap = TestUtilsForBlacklistScheduler.genSupervisors(3,4); + Map supMap = TestUtilsForBlacklistScheduler.genSupervisors(3, 4); Config config = new Config(); config.putAll(Utils.readDefaultConfig()); - config.put(DaemonConfig.BLACKLIST_SCHEDULER_TOLERANCE_TIME,200); - config.put(DaemonConfig.BLACKLIST_SCHEDULER_TOLERANCE_COUNT,2); - config.put(DaemonConfig.BLACKLIST_SCHEDULER_RESUME_TIME,300); + config.put(DaemonConfig.BLACKLIST_SCHEDULER_TOLERANCE_TIME, 200); + config.put(DaemonConfig.BLACKLIST_SCHEDULER_TOLERANCE_COUNT, 2); + config.put(DaemonConfig.BLACKLIST_SCHEDULER_RESUME_TIME, 300); Map topoMap = new HashMap<>(); - TopologyDetails topo1 = TestUtilsForBlacklistScheduler.getTopology("topo-1", config, 5, 15, 1, 1, currentTime - 2,true); + TopologyDetails topo1 = TestUtilsForBlacklistScheduler.getTopology("topo-1", config, 5, 15, + 1, 1, currentTime - 2, true); topoMap.put(topo1.getId(), topo1); Topologies topologies = new Topologies(topoMap); StormMetricsRegistry metricsRegistry = new StormMetricsRegistry(); ResourceMetrics resourceMetrics = new ResourceMetrics(metricsRegistry); - Cluster cluster = new Cluster(iNimbus, resourceMetrics, supMap, new HashMap(), topologies, config); + Cluster cluster = new Cluster(iNimbus, resourceMetrics, supMap, new HashMap(), topologies, config); BlacklistScheduler bs = new BlacklistScheduler(new DefaultScheduler()); scheduler = bs; bs.prepare(config, metricsRegistry); - bs.schedule(topologies,cluster); - cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap, "sup-0"), - TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config); - for (int i = 0 ; i < 20 ; i++){ - bs.schedule(topologies,cluster); + bs.schedule(topologies, cluster); + cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler + .removeSupervisorFromSupervisors(supMap, "sup-0"), + TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster + .getAssignments()), topologies, config); + for (int i = 0; i < 20; i++) { + bs.schedule(topologies, cluster); } Set cached = new HashSet<>(); cached.add("sup-1"); cached.add("sup-2"); assertEquals(cached, bs.cachedSupervisors.keySet()); - cluster = new Cluster(iNimbus, resourceMetrics, supMap, new HashMap(), topologies, config); - bs.schedule(topologies,cluster); - cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.removePortFromSupervisors(supMap, "sup-0", 0), - TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config); - for (int i = 0 ;i < 20 ; i++){ + cluster = new Cluster(iNimbus, resourceMetrics, supMap, new HashMap(), topologies, config); + bs.schedule(topologies, cluster); + cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler + .removePortFromSupervisors(supMap, "sup-0", 0), + TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster + .getAssignments()), topologies, config); + for (int i = 0; i < 20; i++) { bs.schedule(topologies, cluster); } Set cachedPorts = Sets.newHashSet(1, 2, 3); @@ -552,9 +626,9 @@ public void removeLongTimeDisappearFromCache(){ public void blacklistSupervisorWithAddedPort() { Config config = new Config(); config.putAll(Utils.readDefaultConfig()); - config.put(DaemonConfig.BLACKLIST_SCHEDULER_TOLERANCE_TIME,10); - config.put(DaemonConfig.BLACKLIST_SCHEDULER_TOLERANCE_COUNT,2); - config.put(DaemonConfig.BLACKLIST_SCHEDULER_RESUME_TIME,300); + config.put(DaemonConfig.BLACKLIST_SCHEDULER_TOLERANCE_TIME, 10); + config.put(DaemonConfig.BLACKLIST_SCHEDULER_TOLERANCE_COUNT, 2); + config.put(DaemonConfig.BLACKLIST_SCHEDULER_RESUME_TIME, 300); StormMetricsRegistry metricsRegistry = new StormMetricsRegistry(); scheduler = new BlacklistScheduler(new DefaultScheduler()); @@ -568,21 +642,25 @@ public void blacklistSupervisorWithAddedPort() { INimbus iNimbus = new TestUtilsForBlacklistScheduler.INimbusTest(); ResourceMetrics resourceMetrics = new ResourceMetrics(metricsRegistry); - Map supMap = TestUtilsForBlacklistScheduler.genSupervisors(3,4); - Cluster cluster = new Cluster(iNimbus, resourceMetrics, supMap, new HashMap(), + Map supMap = TestUtilsForBlacklistScheduler.genSupervisors(3, 4); + Cluster cluster = new Cluster(iNimbus, resourceMetrics, supMap, new HashMap(), topologies, config); // allow blacklist scheduler to cache the supervisor scheduler.schedule(topologies, cluster); - cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.addPortToSupervisors(supMap, - "sup-0", 4),TestUtilsForBlacklistScheduler.assignmentMapToImpl( + cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler + .addPortToSupervisors(supMap, + "sup-0", 4), TestUtilsForBlacklistScheduler.assignmentMapToImpl( cluster.getAssignments()), topologies, config); // allow blacklist scheduler to cache the supervisor with an added port scheduler.schedule(topologies, cluster); - // remove the port from the supervisor and make sure the blacklist scheduler can remove the port without + // remove the port from the supervisor and make sure the blacklist scheduler can remove the + // port without // throwing an exception - cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.removePortFromSupervisors(supMap, - "sup-0", 4),TestUtilsForBlacklistScheduler.assignmentMapToImpl( + cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler + .removePortFromSupervisors(supMap, + "sup-0", 4), TestUtilsForBlacklistScheduler.assignmentMapToImpl( cluster.getAssignments()), topologies, config); scheduler.schedule(topologies, cluster); } diff --git a/storm-server/src/test/java/org/apache/storm/scheduler/blacklist/TestUtilsForBlacklistScheduler.java b/storm-server/src/test/java/org/apache/storm/scheduler/blacklist/TestUtilsForBlacklistScheduler.java index 403cb99e722..16a495919fb 100644 --- a/storm-server/src/test/java/org/apache/storm/scheduler/blacklist/TestUtilsForBlacklistScheduler.java +++ b/storm-server/src/test/java/org/apache/storm/scheduler/blacklist/TestUtilsForBlacklistScheduler.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

      Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -17,6 +17,14 @@ */ package org.apache.storm.scheduler.blacklist; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Set; import org.apache.storm.Config; import org.apache.storm.Constants; import org.apache.storm.generated.Bolt; @@ -47,30 +55,23 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.util.Set; - - public class TestUtilsForBlacklistScheduler { private static final Logger LOG = LoggerFactory.getLogger(TestUtilsForBlacklistScheduler.class); - public static Map removeSupervisorFromSupervisors(Map supervisorDetailsMap, String supervisor) { + public static Map removeSupervisorFromSupervisors(Map supervisorDetailsMap, String supervisor) { Map retList = new HashMap(); retList.putAll(supervisorDetailsMap); retList.remove(supervisor); return retList; } - public static Map removePortFromSupervisors(Map supervisorDetailsMap, String supervisor, int port) { + public static Map removePortFromSupervisors(Map supervisorDetailsMap, String supervisor, int port) { Map retList = new HashMap(); - for (Map.Entry supervisorDetailsEntry : supervisorDetailsMap.entrySet()) { + for (Map.Entry supervisorDetailsEntry : supervisorDetailsMap + .entrySet()) { String supervisorKey = supervisorDetailsEntry.getKey(); SupervisorDetails supervisorDetails = supervisorDetailsEntry.getValue(); Set ports = new HashSet<>(); @@ -78,15 +79,18 @@ public static Map removePortFromSupervisors(Map addPortToSupervisors(Map supervisorDetailsMap, String supervisor, int port) { + public static Map addPortToSupervisors(Map supervisorDetailsMap, String supervisor, int port) { Map retList = new HashMap(); - for (Map.Entry supervisorDetailsEntry : supervisorDetailsMap.entrySet()) { + for (Map.Entry supervisorDetailsEntry : supervisorDetailsMap + .entrySet()) { String supervisorKey = supervisorDetailsEntry.getKey(); SupervisorDetails supervisorDetails = supervisorDetailsEntry.getValue(); Set ports = new HashSet<>(); @@ -94,7 +98,8 @@ public static Map addPortToSupervisors(Map genSupervisors(int numSup, int numP for (int j = 0; j < numPorts; j++) { ports.add(j); } - SupervisorDetails sup = new SupervisorDetails("sup-" + i, "host-" + i, null, ports, null); + SupervisorDetails sup = new SupervisorDetails("sup-" + i, "host-" + i, null, ports, + null); retList.put(sup.getId(), sup); } return retList; } - public static Map genSupervisors(int numSup, int numPorts, double cpu, double memory) { + public static Map genSupervisors(int numSup, int numPorts, + double cpu, double memory) { Map totalResources = new HashMap<>(); totalResources.put(Constants.COMMON_CPU_RESOURCE_NAME, cpu); totalResources.put(Constants.COMMON_TOTAL_MEMORY_RESOURCE_NAME, memory); @@ -123,26 +130,30 @@ public static Map genSupervisors(int numSup, int numP for (int j = 0; j < numPorts; j++) { ports.add(j); } - SupervisorDetails sup = new SupervisorDetails("sup-" + i, "host-" + i, null, ports, totalResources); + SupervisorDetails sup = new SupervisorDetails("sup-" + i, "host-" + i, null, ports, + totalResources); retList.put(sup.getId(), sup); } return retList; } - - public static TopologyDetails getTopology(String name, Map config, int numSpout, int numBolt, + public static TopologyDetails getTopology(String name, Map config, int numSpout, + int numBolt, int spoutParallelism, int boltParallelism, int launchTime, boolean blacklistEnable) { Config conf = new Config(); conf.putAll(config); conf.put(Config.TOPOLOGY_NAME, name); - StormTopology topology = buildTopology(numSpout, numBolt, spoutParallelism, boltParallelism); + StormTopology topology = buildTopology(numSpout, numBolt, spoutParallelism, + boltParallelism); TopologyDetails topo = new TopologyDetails(name + "-" + launchTime, conf, topology, - 3, genExecsAndComps(topology, spoutParallelism, boltParallelism), launchTime, "user"); + 3, genExecsAndComps(topology, spoutParallelism, + boltParallelism), launchTime, "user"); return topo; } - public static Map genExecsAndComps(StormTopology topology, int spoutParallelism, int boltParallelism) { + public static Map genExecsAndComps(StormTopology topology, + int spoutParallelism, int boltParallelism) { Map retMap = new HashMap<>(); int startTask = 0; int endTask = 1; @@ -200,7 +211,8 @@ public TestSpout(boolean isDistributed) { } @Override - public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { + public void open(Map conf, TopologyContext context, + SpoutOutputCollector collector) { _collector = collector; } @@ -274,12 +286,14 @@ public Collection allSlotsAvailableForScheduling(Collection> newSlotsByTopologyId) { + public void assignSlots(Topologies topologies, Map> newSlotsByTopologyId) { } @Override - public String getHostName(Map existingSupervisors, String nodeId) { + public String getHostName(Map existingSupervisors, + String nodeId) { if (existingSupervisors.containsKey(nodeId)) { return existingSupervisors.get(nodeId).getHost(); } @@ -292,7 +306,8 @@ public IScheduler getForcedScheduler() { } } - public static Map assignmentMapToImpl(Map assignmentMap) { + public static Map assignmentMapToImpl(Map assignmentMap) { Map impl = new HashMap<>(); for (Map.Entry entry : assignmentMap.entrySet()) { impl.put(entry.getKey(), (SchedulerAssignmentImpl) entry.getValue()); diff --git a/storm-server/src/test/java/org/apache/storm/scheduler/multitenant/MultitenantSchedulerTest.java b/storm-server/src/test/java/org/apache/storm/scheduler/multitenant/MultitenantSchedulerTest.java index 90522a617e3..4f8974f1d02 100644 --- a/storm-server/src/test/java/org/apache/storm/scheduler/multitenant/MultitenantSchedulerTest.java +++ b/storm-server/src/test/java/org/apache/storm/scheduler/multitenant/MultitenantSchedulerTest.java @@ -19,9 +19,9 @@ package org.apache.storm.scheduler.multitenant; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.assertFalse; import java.util.Arrays; import java.util.Collection; @@ -31,11 +31,10 @@ import java.util.LinkedHashMap; import java.util.Map; import java.util.Set; - import org.apache.storm.Config; import org.apache.storm.DaemonConfig; -import org.apache.storm.generated.StormTopology; import org.apache.storm.daemon.nimbus.Nimbus.StandaloneINimbus; +import org.apache.storm.generated.StormTopology; import org.apache.storm.metric.StormMetricsRegistry; import org.apache.storm.scheduler.Cluster; import org.apache.storm.scheduler.ExecutorDetails; @@ -100,12 +99,15 @@ private static ResourceMetrics newResourceMetrics() { @Test public void testNode() { Map supers = genSupervisors(5); - TopologyDetails topology1 = new TopologyDetails("topology1", new HashMap<>(), null, 1, "user"); - TopologyDetails topology2 = new TopologyDetails("topology2", new HashMap<>(), null, 1, "user"); + TopologyDetails topology1 = new TopologyDetails("topology1", new HashMap<>(), null, 1, + "user"); + TopologyDetails topology2 = new TopologyDetails("topology2", new HashMap<>(), null, 1, + "user"); Map topMap = new HashMap<>(); topMap.put("topology1", topology1); topMap.put("topology2", topology2); - Cluster cluster = new Cluster(new StandaloneINimbus(), newResourceMetrics(), supers, new HashMap<>(), + Cluster cluster = new Cluster(new StandaloneINimbus(), newResourceMetrics(), supers, + new HashMap<>(), new Topologies(topMap), new HashMap<>()); Map nodeMap = Node.getAllNodesFrom(cluster); @@ -159,16 +161,19 @@ public void testNode() { @Test public void testFreePool() { Map supers = genSupervisors(5); - TopologyDetails topology1 = new TopologyDetails("topology1", new HashMap<>(), null, 1, "user"); + TopologyDetails topology1 = new TopologyDetails("topology1", new HashMap<>(), null, 1, + "user"); Map topMap = new HashMap<>(); topMap.put("topology1", topology1); - Cluster cluster = new Cluster(new StandaloneINimbus(), newResourceMetrics(), supers, new HashMap<>(), + Cluster cluster = new Cluster(new StandaloneINimbus(), newResourceMetrics(), supers, + new HashMap<>(), new Topologies(topMap), new HashMap<>()); Map nodeMap = Node.getAllNodesFrom(cluster); FreePool freePool = new FreePool(); // assign one node so it is not in the pool - nodeMap.get("super0").assign("topology1", Arrays.asList(new ExecutorDetails(1, 1)), cluster); + nodeMap.get("super0").assign("topology1", Arrays.asList(new ExecutorDetails(1, 1)), + cluster); freePool.init(cluster, nodeMap); assertEquals(4, freePool.nodesAvailable()); @@ -217,9 +222,11 @@ public void testDefaultPoolSimple() { execMap1.put(executor1, "spout1"); execMap1.put(executor2, "bolt1"); execMap1.put(executor3, "bolt2"); - TopologyDetails topology1 = new TopologyDetails("topology1", conf1, new StormTopology(), 2, execMap1, "user"); + TopologyDetails topology1 = new TopologyDetails("topology1", conf1, new StormTopology(), 2, + execMap1, "user"); Topologies topologies = new Topologies(toTopMap(topology1)); - Cluster cluster = new Cluster(new StandaloneINimbus(), newResourceMetrics(), supers, new HashMap<>(), + Cluster cluster = new Cluster(new StandaloneINimbus(), newResourceMetrics(), supers, + new HashMap<>(), topologies, new HashMap<>()); Map nodeMap = Node.getAllNodesFrom(cluster); @@ -261,9 +268,11 @@ public void testDefaultPoolBigRequest() { execMap1.put(executor1, "spout1"); execMap1.put(executor2, "bolt1"); execMap1.put(executor3, "bolt2"); - TopologyDetails topology1 = new TopologyDetails("topology1", conf1, new StormTopology(), 5, execMap1, "user"); + TopologyDetails topology1 = new TopologyDetails("topology1", conf1, new StormTopology(), 5, + execMap1, "user"); Topologies topologies = new Topologies(toTopMap(topology1)); - Cluster cluster = new Cluster(new StandaloneINimbus(), newResourceMetrics(), supers, new HashMap<>(), + Cluster cluster = new Cluster(new StandaloneINimbus(), newResourceMetrics(), supers, + new HashMap<>(), topologies, new HashMap<>()); Map nodeMap = Node.getAllNodesFrom(cluster); @@ -309,8 +318,10 @@ public void testDefaultPoolBigRequest2() { execMap1.put(executor3, "bolt1"); execMap1.put(executor4, "bolt1"); execMap1.put(executor5, "bolt2"); - TopologyDetails topology1 = new TopologyDetails("topology1", conf1, new StormTopology(), 5, execMap1, "user"); - Cluster cluster = new Cluster(new StandaloneINimbus(), newResourceMetrics(), supers, new HashMap<>(), + TopologyDetails topology1 = new TopologyDetails("topology1", conf1, new StormTopology(), 5, + execMap1, "user"); + Cluster cluster = new Cluster(new StandaloneINimbus(), newResourceMetrics(), supers, + new HashMap<>(), new Topologies(toTopMap(topology1)), new HashMap<>()); Map nodeMap = Node.getAllNodesFrom(cluster); @@ -359,9 +370,11 @@ public void testDefaultPoolFull() { execMap1.put(executor3, "bolt2"); execMap1.put(executor4, "bolt3"); execMap1.put(executor5, "bolt4"); - TopologyDetails topology1 = new TopologyDetails("topology1", conf1, new StormTopology(), 5, execMap1, "user"); + TopologyDetails topology1 = new TopologyDetails("topology1", conf1, new StormTopology(), 5, + execMap1, "user"); Topologies topologies = new Topologies(toTopMap(topology1)); - Cluster singleCluster = new Cluster(new StandaloneINimbus(), newResourceMetrics(), singleSuper, + Cluster singleCluster = new Cluster(new StandaloneINimbus(), newResourceMetrics(), + singleSuper, new HashMap<>(), topologies, new HashMap<>()); { @@ -413,7 +426,8 @@ public void testDefaultPoolComplex() { execMap1.put(executor1, "spout1"); execMap1.put(executor2, "bolt1"); execMap1.put(executor3, "bolt2"); - TopologyDetails topology1 = new TopologyDetails("topology1", conf1, new StormTopology(), 2, execMap1, "user"); + TopologyDetails topology1 = new TopologyDetails("topology1", conf1, new StormTopology(), 2, + execMap1, "user"); Map conf2 = new HashMap<>(); conf2.put(Config.TOPOLOGY_NAME, "topology-name-2"); @@ -422,10 +436,12 @@ public void testDefaultPoolComplex() { execMap2.put(executor12, "bolt12"); execMap2.put(executor13, "bolt13"); execMap2.put(executor14, "bolt14"); - TopologyDetails topology2 = new TopologyDetails("topology2", conf2, new StormTopology(), 4, execMap2, "user"); + TopologyDetails topology2 = new TopologyDetails("topology2", conf2, new StormTopology(), 4, + execMap2, "user"); Topologies topologies = new Topologies(toTopMap(topology1, topology2)); - Cluster cluster = new Cluster(new StandaloneINimbus(), newResourceMetrics(), supers, new HashMap<>(), + Cluster cluster = new Cluster(new StandaloneINimbus(), newResourceMetrics(), supers, + new HashMap<>(), topologies, new HashMap<>()); Map nodeMap = Node.getAllNodesFrom(cluster); @@ -505,9 +521,11 @@ public void testIsolatedPoolSimple() { execMap1.put(executor2, "bolt1"); execMap1.put(executor3, "bolt2"); execMap1.put(executor4, "bolt4"); - TopologyDetails topology1 = new TopologyDetails("topology1", conf1, new StormTopology(), 4, execMap1, "user"); + TopologyDetails topology1 = new TopologyDetails("topology1", conf1, new StormTopology(), 4, + execMap1, "user"); Topologies topologies = new Topologies(toTopMap(topology1)); - Cluster cluster = new Cluster(new StandaloneINimbus(), newResourceMetrics(), supers, new HashMap<>(), + Cluster cluster = new Cluster(new StandaloneINimbus(), newResourceMetrics(), supers, + new HashMap<>(), topologies, new HashMap<>()); Map nodeMap = Node.getAllNodesFrom(cluster); @@ -558,9 +576,11 @@ public void testIsolatedPoolBigAsk() { execMap1.put(executor2, "bolt1"); execMap1.put(executor3, "bolt2"); execMap1.put(executor4, "bolt4"); - TopologyDetails topology1 = new TopologyDetails("topology1", conf1, new StormTopology(), 10, execMap1, "user"); + TopologyDetails topology1 = new TopologyDetails("topology1", conf1, new StormTopology(), 10, + execMap1, "user"); Topologies topologies = new Topologies(toTopMap(topology1)); - Cluster cluster = new Cluster(new StandaloneINimbus(), newResourceMetrics(), supers, new HashMap<>(), + Cluster cluster = new Cluster(new StandaloneINimbus(), newResourceMetrics(), supers, + new HashMap<>(), topologies, new HashMap<>()); Map nodeMap = Node.getAllNodesFrom(cluster); @@ -614,7 +634,8 @@ public void testIsolatedPoolComplex() { execMap1.put(executor2, "bolt1"); execMap1.put(executor3, "bolt2"); execMap1.put(executor4, "bolt4"); - TopologyDetails topology1 = new TopologyDetails("topology1", conf1, new StormTopology(), 4, execMap1, "user"); + TopologyDetails topology1 = new TopologyDetails("topology1", conf1, new StormTopology(), 4, + execMap1, "user"); Map conf2 = new HashMap<>(); conf2.put(Config.TOPOLOGY_NAME, "topology-name-2"); @@ -624,10 +645,12 @@ public void testIsolatedPoolComplex() { execMap2.put(executor12, "bolt12"); execMap2.put(executor13, "bolt13"); execMap2.put(executor14, "bolt14"); - TopologyDetails topology2 = new TopologyDetails("topology2", conf2, new StormTopology(), 4, execMap2, "user"); + TopologyDetails topology2 = new TopologyDetails("topology2", conf2, new StormTopology(), 4, + execMap2, "user"); Topologies topologies = new Topologies(toTopMap(topology1, topology2)); - Cluster cluster = new Cluster(new StandaloneINimbus(), newResourceMetrics(), supers, new HashMap<>(), + Cluster cluster = new Cluster(new StandaloneINimbus(), newResourceMetrics(), supers, + new HashMap<>(), topologies, new HashMap<>()); Map nodeMap = Node.getAllNodesFrom(cluster); @@ -650,7 +673,8 @@ public void testIsolatedPoolComplex() { assertNull(cluster.getAssignmentById("topology2")); isolatedPool.scheduleAsNeeded(new NodePool[]{freePool}); - // We steal 2 nodes from the free pool to handle the extra (but still only 1 node for the non-isolated top) + // We steal 2 nodes from the free pool to handle the extra (but still only 1 node for the + // non-isolated top) assertEquals(4, isolatedPool.slotsAvailable()); assertEquals(1, isolatedPool.nodesAvailable()); assertEquals(2 * 4, freePool.slotsAvailable()); @@ -696,7 +720,8 @@ public void testIsolatedPoolComplex() { assertEquals(4, assignedSlots2After.size()); assertEquals(2, getNodeIds(assignedSlots2After).size()); - Collection nodes2 = isolatedPool.takeNodes(3); // Cannot steal from the isolated scheduler + Collection nodes2 = isolatedPool + .takeNodes(3); // Cannot steal from the isolated scheduler assertEquals(0, nodes2.size()); assertEquals(0, Node.countFreeSlotsAlive(nodes2)); assertEquals(0, Node.countTotalSlotsAlive(nodes2)); @@ -729,7 +754,8 @@ public void testIsolatedPoolComplex2() { execMap1.put(executor2, "bolt1"); execMap1.put(executor3, "bolt2"); execMap1.put(executor4, "bolt4"); - TopologyDetails topology1 = new TopologyDetails("topology1", conf1, new StormTopology(), 4, execMap1, "user"); + TopologyDetails topology1 = new TopologyDetails("topology1", conf1, new StormTopology(), 4, + execMap1, "user"); Map conf2 = new HashMap<>(); conf2.put(Config.TOPOLOGY_NAME, "topology-name-2"); @@ -739,10 +765,12 @@ public void testIsolatedPoolComplex2() { execMap2.put(executor12, "bolt12"); execMap2.put(executor13, "bolt13"); execMap2.put(executor14, "bolt14"); - TopologyDetails topology2 = new TopologyDetails("topology2", conf2, new StormTopology(), 4, execMap2, "user"); + TopologyDetails topology2 = new TopologyDetails("topology2", conf2, new StormTopology(), 4, + execMap2, "user"); Topologies topologies = new Topologies(toTopMap(topology1, topology2)); - Cluster cluster = new Cluster(new StandaloneINimbus(), newResourceMetrics(), supers, new HashMap<>(), + Cluster cluster = new Cluster(new StandaloneINimbus(), newResourceMetrics(), supers, + new HashMap<>(), topologies, new HashMap<>()); Map nodeMap = Node.getAllNodesFrom(cluster); @@ -783,7 +811,8 @@ public void testIsolatedPoolComplex2() { // The text can be off for a bit until we schedule again isolatedPool.scheduleAsNeeded(new NodePool[]{freePool}); - assertEquals("Max Nodes(2) for this user would be exceeded. 1 more nodes needed to run topology.", + assertEquals("Max Nodes(2) for this user would be exceeded. 1 more nodes needed to run " + + "topology.", cluster.getStatusMap().get("topology1")); assertEquals("Scheduled Isolated on 2 Nodes", cluster.getStatusMap().get("topology2")); } @@ -812,7 +841,8 @@ public void testMultitenantScheduler() { new Object[]{"bolt23", 20, 30}, new Object[]{"bolt24", 30, 40}), "userB"); Topologies topologies = new Topologies(toTopMap(topology1, topology2, topology3)); - Cluster cluster = new Cluster(new StandaloneINimbus(), newResourceMetrics(), supers, new HashMap<>(), + Cluster cluster = new Cluster(new StandaloneINimbus(), newResourceMetrics(), supers, + new HashMap<>(), topologies, new HashMap<>()); Map nodeMap = Node.getAllNodesFrom(cluster); @@ -859,10 +889,12 @@ public void testForceFreeSlotInBadState() { executorToSlot.put(new ExecutorDetails(10, 15), new WorkerSlot("super0", 1)); executorToSlot.put(new ExecutorDetails(15, 20), new WorkerSlot("super0", 1)); Map existingAssignments = new HashMap<>(); - existingAssignments.put("topology1", new SchedulerAssignmentImpl("topology1", executorToSlot, null, null)); + existingAssignments.put("topology1", new SchedulerAssignmentImpl("topology1", + executorToSlot, null, null)); Topologies topologies = new Topologies(toTopMap(topology1)); - Cluster cluster = new Cluster(new StandaloneINimbus(), newResourceMetrics(), supers, existingAssignments, + Cluster cluster = new Cluster(new StandaloneINimbus(), newResourceMetrics(), supers, + existingAssignments, topologies, new HashMap<>()); Map nodeMap = Node.getAllNodesFrom(cluster); @@ -915,13 +947,16 @@ public void testMultitenantSchedulerBadStartingState() { Map existingAssignments = new HashMap<>(); Map execToSlot2 = new HashMap<>(); execToSlot2.put(new ExecutorDetails(1, 1), workerSlotWithMultipleAssignments); - existingAssignments.put("topology2", new SchedulerAssignmentImpl("topology2", execToSlot2, null, null)); + existingAssignments.put("topology2", new SchedulerAssignmentImpl("topology2", execToSlot2, + null, null)); Map execToSlot3 = new HashMap<>(); execToSlot3.put(new ExecutorDetails(2, 2), workerSlotWithMultipleAssignments); - existingAssignments.put("topology3", new SchedulerAssignmentImpl("topology3", execToSlot3, null, null)); + existingAssignments.put("topology3", new SchedulerAssignmentImpl("topology3", execToSlot3, + null, null)); Topologies topologies = new Topologies(toTopMap(topology1, topology2, topology3)); - Cluster cluster = new Cluster(new StandaloneINimbus(), newResourceMetrics(), supers, existingAssignments, + Cluster cluster = new Cluster(new StandaloneINimbus(), newResourceMetrics(), supers, + existingAssignments, topologies, new HashMap<>()); Map conf = new HashMap<>(); @@ -958,12 +993,15 @@ public void testExistingAssignmentSlotNotFoundInSupervisor() { mkEdMap(new Object[]{"spout11", 0, 1}), "userA"); Map execToSlot = new HashMap<>(); - execToSlot.put(new ExecutorDetails(0, 0), new WorkerSlot("super0", portNotReportedBySupervisor)); + execToSlot.put(new ExecutorDetails(0, 0), new WorkerSlot("super0", + portNotReportedBySupervisor)); Map existingAssignments = new HashMap<>(); - existingAssignments.put("topology1", new SchedulerAssignmentImpl("topology1", execToSlot, null, null)); + existingAssignments.put("topology1", new SchedulerAssignmentImpl("topology1", execToSlot, + null, null)); Topologies topologies = new Topologies(toTopMap(topology1)); - Cluster cluster = new Cluster(new StandaloneINimbus(), newResourceMetrics(), supers, existingAssignments, + Cluster cluster = new Cluster(new StandaloneINimbus(), newResourceMetrics(), supers, + existingAssignments, topologies, new HashMap<>()); Map conf = new HashMap<>(); @@ -1004,15 +1042,19 @@ public void testExistingAssignmentSlotOnDeadSupervisor() { Map execToSlot1 = new HashMap<>(); execToSlot1.put(new ExecutorDetails(0, 0), workerSlotWithMultipleAssignments); execToSlot1.put(new ExecutorDetails(1, 1), new WorkerSlot(deadSupervisor, 3)); - existingAssignments.put("topology1", new SchedulerAssignmentImpl("topology1", execToSlot1, null, null)); + existingAssignments.put("topology1", new SchedulerAssignmentImpl("topology1", execToSlot1, + null, null)); Map execToSlot2 = new HashMap<>(); execToSlot2.put(new ExecutorDetails(4, 4), workerSlotWithMultipleAssignments); - execToSlot2.put(new ExecutorDetails(5, 5), new WorkerSlot(deadSupervisor, portNotReportedBySupervisor)); - existingAssignments.put("topology2", new SchedulerAssignmentImpl("topology2", execToSlot2, null, null)); + execToSlot2.put(new ExecutorDetails(5, 5), new WorkerSlot(deadSupervisor, + portNotReportedBySupervisor)); + existingAssignments.put("topology2", new SchedulerAssignmentImpl("topology2", execToSlot2, + null, null)); Topologies topologies = new Topologies(toTopMap(topology1, topology2)); - Cluster cluster = new Cluster(new StandaloneINimbus(), newResourceMetrics(), supers, existingAssignments, + Cluster cluster = new Cluster(new StandaloneINimbus(), newResourceMetrics(), supers, + existingAssignments, topologies, new HashMap<>()); Map conf = new HashMap<>(); @@ -1059,10 +1101,12 @@ public void testIsolatedPoolSchedulingWithNodesWithDifferentNumberOfSlots() { execToSlot.put(new ExecutorDetails(4, 4), new WorkerSlot("super2", 1)); execToSlot.put(new ExecutorDetails(5, 5), new WorkerSlot("super2", 2)); Map existingAssignments = new HashMap<>(); - existingAssignments.put("topology1", new SchedulerAssignmentImpl("topology1", execToSlot, null, null)); + existingAssignments.put("topology1", new SchedulerAssignmentImpl("topology1", execToSlot, + null, null)); Topologies topologies = new Topologies(toTopMap(topology1)); - Cluster cluster = new Cluster(new StandaloneINimbus(), newResourceMetrics(), supers, existingAssignments, + Cluster cluster = new Cluster(new StandaloneINimbus(), newResourceMetrics(), supers, + existingAssignments, topologies, new HashMap<>()); Map conf = new HashMap<>(); diff --git a/storm-server/src/test/java/org/apache/storm/scheduler/resource/TestResourceAwareScheduler.java b/storm-server/src/test/java/org/apache/storm/scheduler/resource/TestResourceAwareScheduler.java index a4804202abc..4aa543aeaa4 100644 --- a/storm-server/src/test/java/org/apache/storm/scheduler/resource/TestResourceAwareScheduler.java +++ b/storm-server/src/test/java/org/apache/storm/scheduler/resource/TestResourceAwareScheduler.java @@ -1,17 +1,46 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

      - * http://www.apache.org/licenses/LICENSE-2.0 - *

      - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

      http://www.apache.org/licenses/LICENSE-2.0 + * + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.scheduler.resource; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.INimbusTest; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.TestBolt; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.TestSpout; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.addTopologies; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.assertTopologiesFullyScheduled; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.assertTopologiesNotScheduled; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.createCSSClusterConfig; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.createClusterConfig; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.genExecsAndComps; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.genSupervisors; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.genSupervisorsWithRacks; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.genTopology; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.getSupervisorToCpuUsage; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.getSupervisorToMemoryUsage; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.supervisorIdToRackName; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.topoToTopologyDetails; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.topologyBuilder; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.userRes; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.userResourcePool; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; @@ -21,12 +50,11 @@ import java.util.HashSet; import java.util.LinkedList; import java.util.List; -import java.util.Map; import java.util.Map.Entry; +import java.util.Map; import java.util.Set; import java.util.TreeMap; import java.util.concurrent.atomic.AtomicLong; - import org.apache.storm.Config; import org.apache.storm.DaemonConfig; import org.apache.storm.generated.StormTopology; @@ -43,6 +71,7 @@ import org.apache.storm.scheduler.TopologyDetails; import org.apache.storm.scheduler.WorkerSlot; import org.apache.storm.scheduler.resource.normalization.NormalizedResources; +import org.apache.storm.scheduler.resource.normalization.ResourceMetrics; import org.apache.storm.scheduler.resource.strategies.scheduling.BaseResourceAwareStrategy; import org.apache.storm.scheduler.resource.strategies.scheduling.ConstraintSolverStrategy; import org.apache.storm.scheduler.resource.strategies.scheduling.DefaultResourceAwareStrategy; @@ -50,7 +79,6 @@ import org.apache.storm.scheduler.resource.strategies.scheduling.GenericResourceAwareStrategy; import org.apache.storm.scheduler.resource.strategies.scheduling.GenericResourceAwareStrategyOld; import org.apache.storm.scheduler.resource.strategies.scheduling.RoundRobinResourceAwareStrategy; -import org.apache.storm.scheduler.resource.normalization.ResourceMetrics; import org.apache.storm.testing.PerformanceTest; import org.apache.storm.testing.TestWordCounter; import org.apache.storm.testing.TestWordSpout; @@ -67,33 +95,30 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.*; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; - - public class TestResourceAwareScheduler { private static final Logger LOG = LoggerFactory.getLogger(TestResourceAwareScheduler.class); private static final Class[] strategyClasses = { - DefaultResourceAwareStrategy.class, - RoundRobinResourceAwareStrategy.class, - GenericResourceAwareStrategy.class, + DefaultResourceAwareStrategy.class, + RoundRobinResourceAwareStrategy.class, + GenericResourceAwareStrategy.class, }; private final Config defaultTopologyConf; private int currentTime = 1450418597; private IScheduler scheduler = null; public TestResourceAwareScheduler() { - defaultTopologyConf = createClusterConfig(DefaultResourceAwareStrategy.class, 10, 128, 0, null); + defaultTopologyConf = createClusterConfig(DefaultResourceAwareStrategy.class, 10, 128, 0, + null); defaultTopologyConf.put(Config.TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB, 8192.0); defaultTopologyConf.put(Config.TOPOLOGY_PRIORITY, 0); } - private Config createClusterConfig(Class strategyClass, double compPcore, double compOnHeap, double compOffHeap, + private Config createClusterConfig(Class strategyClass, double compPcore, double compOnHeap, + double compOffHeap, Map> pools) { - Config config = TestUtilsForResourceAwareScheduler.createClusterConfig(compPcore, compOnHeap, compOffHeap, pools); + Config config = TestUtilsForResourceAwareScheduler.createClusterConfig(compPcore, + compOnHeap, compOffHeap, pools); config.put(Config.TOPOLOGY_SCHEDULER_STRATEGY, strategyClass.getName()); return config; } @@ -116,7 +141,8 @@ public void testRASNodeSlotAssign() { TopologyDetails topology1 = genTopology("topology1", config, 1, 0, 2, 0, 0, 0, "user"); TopologyDetails topology2 = genTopology("topology2", config, 1, 0, 2, 0, 0, 0, "user"); Topologies topologies = new Topologies(topology1, topology2); - Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config); + Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), + supMap, new HashMap<>(), topologies, config); Map nodes = RasNodes.getAllNodesFrom(cluster); assertEquals(5, nodes.size()); RasNode node = nodes.get("r000s000"); @@ -186,7 +212,8 @@ public void sanityTestOfScheduling() { TopologyDetails topology1 = genTopology("topology1", config, 1, 1, 1, 1, 0, 0, "user"); Topologies topologies = new Topologies(topology1); - Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config); + Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), + supMap, new HashMap<>(), topologies, config); scheduler.prepare(config, new StormMetricsRegistry()); scheduler.schedule(topologies, cluster); @@ -203,7 +230,8 @@ public void sanityTestOfScheduling() { assertEquals(1, nodesIDs.size()); assertEquals(2, executors.size()); assertFalse(cluster.needsSchedulingRas(topology1)); - assertTrue(cluster.getStatusMap().get(topology1.getId()).startsWith("Running - Fully Scheduled by DefaultResourceAwareStrategy")); + assertTrue(cluster.getStatusMap().get(topology1.getId()) + .startsWith("Running - Fully Scheduled by DefaultResourceAwareStrategy")); } @Test @@ -214,29 +242,37 @@ public void testTopologyWithMultipleSpouts() { TopologyBuilder builder1 = new TopologyBuilder(); // a topology with multiple spouts builder1.setSpout("wordSpout1", new TestWordSpout(), 1); builder1.setSpout("wordSpout2", new TestWordSpout(), 1); - builder1.setBolt("wordCountBolt1", new TestWordCounter(), 1).shuffleGrouping("wordSpout1").shuffleGrouping("wordSpout2"); - builder1.setBolt("wordCountBolt2", new TestWordCounter(), 1).shuffleGrouping("wordCountBolt1"); - builder1.setBolt("wordCountBolt3", new TestWordCounter(), 1).shuffleGrouping("wordCountBolt1"); - builder1.setBolt("wordCountBolt4", new TestWordCounter(), 1).shuffleGrouping("wordCountBolt2"); + builder1.setBolt("wordCountBolt1", new TestWordCounter(), 1).shuffleGrouping("wordSpout1") + .shuffleGrouping("wordSpout2"); + builder1.setBolt("wordCountBolt2", new TestWordCounter(), 1) + .shuffleGrouping("wordCountBolt1"); + builder1.setBolt("wordCountBolt3", new TestWordCounter(), 1) + .shuffleGrouping("wordCountBolt1"); + builder1.setBolt("wordCountBolt4", new TestWordCounter(), 1) + .shuffleGrouping("wordCountBolt2"); builder1.setBolt("wordCountBolt5", new TestWordCounter(), 1).shuffleGrouping("wordSpout2"); StormTopology stormTopology1 = builder1.createTopology(); Config config = new Config(); config.putAll(defaultTopologyConf); Map executorMap1 = genExecsAndComps(stormTopology1); - TopologyDetails topology1 = new TopologyDetails("topology1", config, stormTopology1, 0, executorMap1, 0, "user"); + TopologyDetails topology1 = new TopologyDetails("topology1", config, stormTopology1, 0, + executorMap1, 0, "user"); - TopologyBuilder builder2 = new TopologyBuilder(); // a topology with two unconnected partitions + TopologyBuilder builder2 = + new TopologyBuilder(); // a topology with two unconnected partitions builder2.setSpout("wordSpoutX", new TestWordSpout(), 1); builder2.setSpout("wordSpoutY", new TestWordSpout(), 1); StormTopology stormTopology2 = builder2.createTopology(); Map executorMap2 = genExecsAndComps(stormTopology2); - TopologyDetails topology2 = new TopologyDetails("topology2", config, stormTopology2, 0, executorMap2, 0, "user"); + TopologyDetails topology2 = new TopologyDetails("topology2", config, stormTopology2, 0, + executorMap2, 0, "user"); scheduler = new ResourceAwareScheduler(); Topologies topologies = new Topologies(topology1, topology2); - Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config); + Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), + supMap, new HashMap<>(), topologies, config); scheduler.prepare(config, new StormMetricsRegistry()); scheduler.schedule(topologies, cluster); @@ -253,7 +289,8 @@ public void testTopologyWithMultipleSpouts() { assertEquals(1, nodesIDs1.size()); assertEquals(7, executors1.size()); assertFalse(cluster.needsSchedulingRas(topology1)); - assertTrue(cluster.getStatusMap().get(topology1.getId()).startsWith("Running - Fully Scheduled by DefaultResourceAwareStrategy")); + assertTrue(cluster.getStatusMap().get(topology1.getId()) + .startsWith("Running - Fully Scheduled by DefaultResourceAwareStrategy")); SchedulerAssignment assignment2 = cluster.getAssignmentById(topology2.getId()); Set assignedSlots2 = assignment2.getSlots(); @@ -267,7 +304,8 @@ public void testTopologyWithMultipleSpouts() { assertEquals(1, nodesIDs2.size()); assertEquals(2, executors2.size()); assertFalse(cluster.needsSchedulingRas(topology2)); - assertTrue(cluster.getStatusMap().get(topology2.getId()).startsWith("Running - Fully Scheduled by DefaultResourceAwareStrategy")); + assertTrue(cluster.getStatusMap().get(topology2.getId()) + .startsWith("Running - Fully Scheduled by DefaultResourceAwareStrategy")); } @Test @@ -276,19 +314,23 @@ public void testTopologySetCpuAndMemLoad() { Map supMap = genSupervisors(2, 2, 400, 2000); TopologyBuilder builder1 = new TopologyBuilder(); // a topology with multiple spouts - builder1.setSpout("wordSpout", new TestWordSpout(), 1).setCPULoad(20.0).setMemoryLoad(200.0); - builder1.setBolt("wordCountBolt", new TestWordCounter(), 1).shuffleGrouping("wordSpout").setCPULoad(20.0).setMemoryLoad(200.0); + builder1.setSpout("wordSpout", new TestWordSpout(), 1).setCPULoad(20.0) + .setMemoryLoad(200.0); + builder1.setBolt("wordCountBolt", new TestWordCounter(), 1).shuffleGrouping("wordSpout") + .setCPULoad(20.0).setMemoryLoad(200.0); StormTopology stormTopology1 = builder1.createTopology(); Config config = new Config(); config.putAll(defaultTopologyConf); Map executorMap1 = genExecsAndComps(stormTopology1); - TopologyDetails topology1 = new TopologyDetails("topology1", config, stormTopology1, 0, executorMap1, 0, "user"); + TopologyDetails topology1 = new TopologyDetails("topology1", config, stormTopology1, 0, + executorMap1, 0, "user"); ResourceAwareScheduler rs = new ResourceAwareScheduler(); scheduler = rs; Topologies topologies = new Topologies(topology1); - Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config); + Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), + supMap, new HashMap<>(), topologies, config); rs.prepare(config, new StormMetricsRegistry()); rs.schedule(topologies, cluster); @@ -322,20 +364,24 @@ public void testResourceLimitation() { Map supMap = genSupervisors(2, 2, 400, 2000); TopologyBuilder builder1 = new TopologyBuilder(); // a topology with multiple spouts - builder1.setSpout("wordSpout", new TestWordSpout(), 2).setCPULoad(250.0).setMemoryLoad(1000.0, 200.0); - builder1.setBolt("wordCountBolt", new TestWordCounter(), 1).shuffleGrouping("wordSpout").setCPULoad(100.0) + builder1.setSpout("wordSpout", new TestWordSpout(), 2).setCPULoad(250.0) + .setMemoryLoad(1000.0, 200.0); + builder1.setBolt("wordCountBolt", new TestWordCounter(), 1).shuffleGrouping("wordSpout") + .setCPULoad(100.0) .setMemoryLoad(500.0, 100.0); StormTopology stormTopology1 = builder1.createTopology(); Config config = new Config(); config.putAll(defaultTopologyConf); Map executorMap1 = genExecsAndComps(stormTopology1); - TopologyDetails topology1 = new TopologyDetails("topology1", config, stormTopology1, 2, executorMap1, 0, "user"); + TopologyDetails topology1 = new TopologyDetails("topology1", config, stormTopology1, 2, + executorMap1, 0, "user"); ResourceAwareScheduler rs = new ResourceAwareScheduler(); scheduler = rs; Topologies topologies = new Topologies(topology1); - Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config); + Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), + supMap, new HashMap<>(), topologies, config); rs.prepare(config, new StormMetricsRegistry()); rs.schedule(topologies, cluster); @@ -361,15 +407,19 @@ public void testResourceLimitation() { Map cpuAvailableToUsed = new HashMap<>(); Map memoryAvailableToUsed = new HashMap<>(); - for (Map.Entry entry : assignment1.getExecutorToSlot().entrySet()) { - executorToSupervisor.put(entry.getKey(), cluster.getSupervisorById(entry.getValue().getNodeId())); + for (Map.Entry entry : assignment1.getExecutorToSlot() + .entrySet()) { + executorToSupervisor.put(entry.getKey(), cluster.getSupervisorById(entry.getValue() + .getNodeId())); } - for (Map.Entry entry : executorToSupervisor.entrySet()) { + for (Map.Entry entry : executorToSupervisor + .entrySet()) { supervisorToExecutors .computeIfAbsent(entry.getValue(), k -> new ArrayList<>()) .add(entry.getKey()); } - for (Map.Entry> entry : supervisorToExecutors.entrySet()) { + for (Map.Entry> entry : supervisorToExecutors + .entrySet()) { Double supervisorTotalCpu = entry.getKey().getTotalCpu(); Double supervisorTotalMemory = entry.getKey().getTotalMemory(); Double supervisorUsedCpu = 0.0; @@ -381,7 +431,8 @@ public void testResourceLimitation() { cpuAvailableToUsed.put(supervisorTotalCpu, supervisorUsedCpu); memoryAvailableToUsed.put(supervisorTotalMemory, supervisorUsedMemory); } - // executor0 resides one one worker (on one), executor1 and executor2 on another worker (on the other node) + // executor0 resides one one worker (on one), executor1 and executor2 on another worker (on + // the other node) assertEquals(2, assignedSlots1.size()); assertEquals(2, nodesIDs1.size()); assertEquals(3, executors1.size()); @@ -400,7 +451,8 @@ public void testResourceLimitation() { assertTrue(entry.getKey() - entry.getValue() >= 0); } assertFalse(cluster.needsSchedulingRas(topology1)); - assertTrue(cluster.getStatusMap().get(topology1.getId()).startsWith("Running - Fully Scheduled by DefaultResourceAwareStrategy")); + assertTrue(cluster.getStatusMap().get(topology1.getId()) + .startsWith("Running - Fully Scheduled by DefaultResourceAwareStrategy")); } @Test @@ -414,22 +466,26 @@ public void testScheduleResilience() { Config config1 = new Config(); config1.putAll(defaultTopologyConf); Map executorMap1 = genExecsAndComps(stormTopology1); - TopologyDetails topology1 = new TopologyDetails("topology1", config1, stormTopology1, 3, executorMap1, 0, "user"); + TopologyDetails topology1 = new TopologyDetails("topology1", config1, stormTopology1, 3, + executorMap1, 0, "user"); TopologyBuilder builder2 = new TopologyBuilder(); builder2.setSpout("wordSpout2", new TestWordSpout(), 2); StormTopology stormTopology2 = builder2.createTopology(); Config config2 = new Config(); config2.putAll(defaultTopologyConf); - // memory requirement is large enough so that two executors can not be fully assigned to one node + // memory requirement is large enough so that two executors can not be fully assigned to one + // node config2.put(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB, 1280.0); Map executorMap2 = genExecsAndComps(stormTopology2); - TopologyDetails topology2 = new TopologyDetails("topology2", config2, stormTopology2, 2, executorMap2, 0, "user"); + TopologyDetails topology2 = new TopologyDetails("topology2", config2, stormTopology2, 2, + executorMap2, 0, "user"); // Test1: When a worker fails, RAS does not alter existing assignments on healthy workers scheduler = new ResourceAwareScheduler(); Topologies topologies = new Topologies(topology2); - Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config1); + Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), + supMap, new HashMap<>(), topologies, config1); scheduler.prepare(config1, new StormMetricsRegistry()); scheduler.schedule(topologies, cluster); @@ -445,7 +501,8 @@ public void testScheduleResilience() { } } for (ExecutorDetails executor : failedExecutors) { - executorToSlot.remove(executor); // remove executor details assigned to the failed worker + executorToSlot + .remove(executor); // remove executor details assigned to the failed worker } Map copyOfOldMapping = new HashMap<>(executorToSlot); Set healthyExecutors = copyOfOldMapping.keySet(); @@ -458,7 +515,8 @@ public void testScheduleResilience() { assertEquals(copyOfOldMapping.get(executor), newExecutorToSlot.get(executor)); } assertFalse(cluster.needsSchedulingRas(topology2)); - assertTrue(cluster.getStatusMap().get(topology2.getId()).startsWith("Running - Fully Scheduled by DefaultResourceAwareStrategy")); + assertTrue(cluster.getStatusMap().get(topology2.getId()) + .startsWith("Running - Fully Scheduled by DefaultResourceAwareStrategy")); // end of Test1 // Test2: When a supervisor fails, RAS does not alter existing assignments @@ -475,7 +533,8 @@ public void testScheduleResilience() { supMap1.remove("r000s000"); // mock the supervisor r000s000 as a failed supervisor topologies = new Topologies(topology1); - Cluster cluster1 = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap1, existingAssignments, topologies, config1); + Cluster cluster1 = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), + supMap1, existingAssignments, topologies, config1); scheduler.schedule(topologies, cluster1); newAssignment = cluster1.getAssignmentById(topology1.getId()); @@ -487,24 +546,31 @@ public void testScheduleResilience() { assertEquals("Fully Scheduled", cluster1.getStatusMap().get(topology1.getId())); // end of Test2 - // Test3: When a supervisor and a worker on it fails, RAS does not alter existing assignments + // Test3: When a supervisor and a worker on it fails, RAS does not alter existing + // assignments executorToSlot = new HashMap<>(); - executorToSlot.put(new ExecutorDetails(0, 0), new WorkerSlot("r000s000", 1)); // the worker to orphan - executorToSlot.put(new ExecutorDetails(1, 1), new WorkerSlot("r000s000", 2)); // the worker that fails - executorToSlot.put(new ExecutorDetails(2, 2), new WorkerSlot("r000s001", 1)); // the healthy worker + executorToSlot.put(new ExecutorDetails(0, 0), new WorkerSlot("r000s000", + 1)); // the worker to orphan + executorToSlot.put(new ExecutorDetails(1, 1), new WorkerSlot("r000s000", + 2)); // the worker that fails + executorToSlot.put(new ExecutorDetails(2, 2), new WorkerSlot("r000s001", + 1)); // the healthy worker existingAssignments = new HashMap<>(); assignment = new SchedulerAssignmentImpl(topology1.getId(), executorToSlot, null, null); existingAssignments.put(topology1.getId(), assignment); - // delete one worker of r000s000 (failed) from topo1 assignment to enable actual schedule for testing + // delete one worker of r000s000 (failed) from topo1 assignment to enable actual schedule + // for testing executorToSlot.remove(new ExecutorDetails(1, 1)); copyOfOldMapping = new HashMap<>(executorToSlot); - existingExecutors = copyOfOldMapping.keySet(); // namely the two eds on the orphaned worker and the healthy worker + existingExecutors = copyOfOldMapping + .keySet(); // namely the two eds on the orphaned worker and the healthy worker supMap1 = new HashMap<>(supMap); supMap1.remove("r000s000"); // mock the supervisor r000s000 as a failed supervisor topologies = new Topologies(topology1); - cluster1 = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap1, existingAssignments, topologies, config1); + cluster1 = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap1, + existingAssignments, topologies, config1); scheduler.schedule(topologies, cluster1); newAssignment = cluster1.getAssignmentById(topology1.getId()); @@ -519,14 +585,16 @@ public void testScheduleResilience() { // Test4: Scheduling a new topology does not disturb other assignments unnecessarily topologies = new Topologies(topology1); - cluster1 = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config1); + cluster1 = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, + new HashMap<>(), topologies, config1); scheduler.schedule(topologies, cluster1); assignment = cluster1.getAssignmentById(topology1.getId()); executorToSlot = assignment.getExecutorToSlot(); copyOfOldMapping = new HashMap<>(executorToSlot); topologies = addTopologies(topologies, topology2); - cluster1 = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config1); + cluster1 = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, + new HashMap<>(), topologies, config1); scheduler.schedule(topologies, cluster1); newAssignment = cluster1.getAssignmentById(topology1.getId()); @@ -552,8 +620,10 @@ public void testHeterogeneousCluster(Config topologyConf, String strategyName) { resourceMap2.put(Config.SUPERVISOR_CPU_CAPACITY, 200.0); resourceMap2.put(Config.SUPERVISOR_MEMORY_CAPACITY_MB, 1024.0); - resourceMap1 = NormalizedResources.RESOURCE_NAME_NORMALIZER.normalizedResourceMap(resourceMap1); - resourceMap2 = NormalizedResources.RESOURCE_NAME_NORMALIZER.normalizedResourceMap(resourceMap2); + resourceMap1 = NormalizedResources.RESOURCE_NAME_NORMALIZER + .normalizedResourceMap(resourceMap1); + resourceMap2 = NormalizedResources.RESOURCE_NAME_NORMALIZER + .normalizedResourceMap(resourceMap2); Map supMap = new HashMap<>(); for (int i = 0; i < 2; i++) { @@ -561,61 +631,75 @@ public void testHeterogeneousCluster(Config topologyConf, String strategyName) { for (int j = 0; j < 4; j++) { ports.add(j); } - SupervisorDetails sup = new SupervisorDetails("r00s00" + i, "host-" + i, null, ports, i == 0 ? resourceMap1 : resourceMap2); + SupervisorDetails sup = new SupervisorDetails("r00s00" + i, "host-" + i, null, ports, + i == 0 ? resourceMap1 : resourceMap2); supMap.put(sup.getId(), sup); } LOG.info("SUPERVISORS = {}", supMap); // topo1 has one single huge task that can not be handled by the small-super TopologyBuilder builder1 = new TopologyBuilder(); - builder1.setSpout("wordSpout1", new TestWordSpout(), 1).setCPULoad(300.0).setMemoryLoad(2000.0, 48.0); + builder1.setSpout("wordSpout1", new TestWordSpout(), 1).setCPULoad(300.0) + .setMemoryLoad(2000.0, 48.0); StormTopology stormTopology1 = builder1.createTopology(); Config config1 = new Config(); config1.putAll(topologyConf); Map executorMap1 = genExecsAndComps(stormTopology1); - TopologyDetails topology1 = new TopologyDetails("topology1", config1, stormTopology1, 1, executorMap1, 0, "user"); + TopologyDetails topology1 = new TopologyDetails("topology1", config1, stormTopology1, 1, + executorMap1, 0, "user"); // topo2 has 4 large tasks TopologyBuilder builder2 = new TopologyBuilder(); - builder2.setSpout("wordSpout2", new TestWordSpout(), 4).setCPULoad(100.0).setMemoryLoad(500.0, 12.0); + builder2.setSpout("wordSpout2", new TestWordSpout(), 4).setCPULoad(100.0) + .setMemoryLoad(500.0, 12.0); StormTopology stormTopology2 = builder2.createTopology(); Config config2 = new Config(); config2.putAll(topologyConf); Map executorMap2 = genExecsAndComps(stormTopology2); - TopologyDetails topology2 = new TopologyDetails("topology2", config2, stormTopology2, 1, executorMap2, 0, "user"); + TopologyDetails topology2 = new TopologyDetails("topology2", config2, stormTopology2, 1, + executorMap2, 0, "user"); // topo3 has 4 large tasks TopologyBuilder builder3 = new TopologyBuilder(); - builder3.setSpout("wordSpout3", new TestWordSpout(), 4).setCPULoad(20.0).setMemoryLoad(200.0, 56.0); + builder3.setSpout("wordSpout3", new TestWordSpout(), 4).setCPULoad(20.0) + .setMemoryLoad(200.0, 56.0); StormTopology stormTopology3 = builder3.createTopology(); Config config3 = new Config(); config3.putAll(topologyConf); Map executorMap3 = genExecsAndComps(stormTopology3); - TopologyDetails topology3 = new TopologyDetails("topology3", config3, stormTopology3, 1, executorMap3, 0, "user"); + TopologyDetails topology3 = new TopologyDetails("topology3", config3, stormTopology3, 1, + executorMap3, 0, "user"); // topo4 has 12 small tasks, whose mem usage does not exactly divide a node's mem capacity TopologyBuilder builder4 = new TopologyBuilder(); - builder4.setSpout("wordSpout4", new TestWordSpout(), 12).setCPULoad(30.0).setMemoryLoad(100.0, 0.0); + builder4.setSpout("wordSpout4", new TestWordSpout(), 12).setCPULoad(30.0) + .setMemoryLoad(100.0, 0.0); StormTopology stormTopology4 = builder4.createTopology(); Config config4 = new Config(); config4.putAll(topologyConf); Map executorMap4 = genExecsAndComps(stormTopology4); - TopologyDetails topology4 = new TopologyDetails("topology4", config4, stormTopology4, 1, executorMap4, 0, "user"); + TopologyDetails topology4 = new TopologyDetails("topology4", config4, stormTopology4, 1, + executorMap4, 0, "user"); - // topo5 has 40 small tasks, it should be able to exactly use up both the cpu and mem in the cluster + // topo5 has 40 small tasks, it should be able to exactly use up both the cpu and mem in the + // cluster TopologyBuilder builder5 = new TopologyBuilder(); - builder5.setSpout("wordSpout5", new TestWordSpout(), 40).setCPULoad(25.0).setMemoryLoad(100.0, 28.0); + builder5.setSpout("wordSpout5", new TestWordSpout(), 40).setCPULoad(25.0) + .setMemoryLoad(100.0, 28.0); StormTopology stormTopology5 = builder5.createTopology(); Config config5 = new Config(); config5.putAll(topologyConf); Map executorMap5 = genExecsAndComps(stormTopology5); - TopologyDetails topology5 = new TopologyDetails("topology5", config5, stormTopology5, 1, executorMap5, 0, "user"); + TopologyDetails topology5 = new TopologyDetails("topology5", config5, stormTopology5, 1, + executorMap5, 0, "user"); - // Test1: Launch topo 1-3 together, it should be able to use up either mem or cpu resource due to exact division + // Test1: Launch topo 1-3 together, it should be able to use up either mem or cpu resource + // due to exact division ResourceAwareScheduler rs = new ResourceAwareScheduler(); LOG.info("\n\n\t\tScheduling topologies 1, 2 and 3"); Topologies topologies = new Topologies(topology1, topology2, topology3); - Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config1); + Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), + supMap, new HashMap<>(), topologies, config1); rs.prepare(config1, new StormMetricsRegistry()); Map superToCpu = null; @@ -643,8 +727,10 @@ public void testHeterogeneousCluster(Config topologyConf, String strategyName) { Double cpuUsed = superToCpu.get(supervisor); Double memUsed = superToMem.get(supervisor); - assertTrue((Math.abs(memAvailable - memUsed) < EPSILON) || (Math.abs(cpuAvailable - cpuUsed) < EPSILON), - supervisor.getId() + " MEM: " + memAvailable + " == " + memUsed + " OR CPU: " + cpuAvailable + " == " + cpuUsed); + assertTrue((Math.abs(memAvailable - memUsed) < EPSILON) || (Math + .abs(cpuAvailable - cpuUsed) < EPSILON), + supervisor.getId() + " MEM: " + memAvailable + " == " + memUsed + + " OR CPU: " + cpuAvailable + " == " + cpuUsed); } } finally { rs.cleanup(); @@ -653,23 +739,28 @@ public void testHeterogeneousCluster(Config topologyConf, String strategyName) { // end of Test1 LOG.warn("\n\n\t\tSwitching to topologies 1, 2 and 4"); - // Test2: Launch topo 1, 2 and 4, they together request a little more mem than available, so one of the 3 topos will not be + // Test2: Launch topo 1, 2 and 4, they together request a little more mem than available, so + // one of the 3 topos will not be // scheduled topologies = new Topologies(topology1, topology2, topology4); - cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config1); + cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, + new HashMap<>(), topologies, config1); rs.prepare(config1, new StormMetricsRegistry()); try { rs.schedule(topologies, cluster); int numTopologiesAssigned = 0; - if (cluster.getStatusMap().get(topology1.getId()).startsWith("Running - Fully Scheduled by " + strategyName)) { + if (cluster.getStatusMap().get(topology1.getId()) + .startsWith("Running - Fully Scheduled by " + strategyName)) { LOG.info("TOPO 1 scheduled"); numTopologiesAssigned++; } - if (cluster.getStatusMap().get(topology2.getId()).startsWith("Running - Fully Scheduled by " + strategyName)) { + if (cluster.getStatusMap().get(topology2.getId()) + .startsWith("Running - Fully Scheduled by " + strategyName)) { LOG.info("TOPO 2 scheduled"); numTopologiesAssigned++; } - if (cluster.getStatusMap().get(topology4.getId()).startsWith("Running - Fully Scheduled by " + strategyName)) { + if (cluster.getStatusMap().get(topology4.getId()) + .startsWith("Running - Fully Scheduled by " + strategyName)) { LOG.info("TOPO 3 scheduled"); numTopologiesAssigned++; } @@ -677,12 +768,13 @@ public void testHeterogeneousCluster(Config topologyConf, String strategyName) { } finally { rs.cleanup(); } - //end of Test2 + // end of Test2 LOG.info("\n\n\t\tScheduling just topo 5"); - //Test3: "Launch topo5 only, both mem and cpu should be exactly used up" + // Test3: "Launch topo5 only, both mem and cpu should be exactly used up" topologies = new Topologies(topology5); - cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config1); + cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, + new HashMap<>(), topologies, config1); rs.prepare(config1, new StormMetricsRegistry()); try { rs.schedule(topologies, cluster); @@ -699,24 +791,28 @@ public void testHeterogeneousCluster(Config topologyConf, String strategyName) { } finally { rs.cleanup(); } - //end of Test3 + // end of Test3 } @Test public void testHeterogeneousClusterwithDefaultRas() { - testHeterogeneousCluster(defaultTopologyConf, DefaultResourceAwareStrategy.class.getSimpleName()); + testHeterogeneousCluster(defaultTopologyConf, DefaultResourceAwareStrategy.class + .getSimpleName()); } @Test public void testHeterogeneousClusterwithGras() { Config grasClusterConfig = (Config) defaultTopologyConf.clone(); - grasClusterConfig.put(Config.TOPOLOGY_SCHEDULER_STRATEGY, GenericResourceAwareStrategy.class.getName()); - testHeterogeneousCluster(grasClusterConfig, GenericResourceAwareStrategy.class.getSimpleName()); + grasClusterConfig.put(Config.TOPOLOGY_SCHEDULER_STRATEGY, GenericResourceAwareStrategy.class + .getName()); + testHeterogeneousCluster(grasClusterConfig, GenericResourceAwareStrategy.class + .getSimpleName()); } @Test public void testTopologyWorkerMaxHeapSize() { - // Test1: If RAS spreads executors across multiple workers based on the set limit for a worker used by the topology + // Test1: If RAS spreads executors across multiple workers based on the set limit for a + // worker used by the topology INimbus iNimbus = new INimbusTest(); Map supMap = genSupervisors(2, 2, 400, 2000); @@ -727,24 +823,31 @@ public void testTopologyWorkerMaxHeapSize() { config1.putAll(defaultTopologyConf); config1.put(Config.TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB, 128.0); Map executorMap1 = genExecsAndComps(stormTopology1); - TopologyDetails topology1 = new TopologyDetails("topology1", config1, stormTopology1, 1, executorMap1, 0, "user"); + TopologyDetails topology1 = new TopologyDetails("topology1", config1, stormTopology1, 1, + executorMap1, 0, "user"); ResourceAwareScheduler rs = new ResourceAwareScheduler(); Topologies topologies = new Topologies(topology1); - Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config1); + Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), + supMap, new HashMap<>(), topologies, config1); rs.prepare(config1, new StormMetricsRegistry()); try { rs.schedule(topologies, cluster); assertFalse(cluster.needsSchedulingRas(topology1)); - assertTrue(cluster.getStatusMap().get(topology1.getId()).startsWith("Running - Fully Scheduled by DefaultResourceAwareStrategy")); + assertTrue(cluster.getStatusMap().get(topology1.getId()) + .startsWith("Running - Fully Scheduled by DefaultResourceAwareStrategy")); assertEquals(4, cluster.getAssignedNumWorkers(topology1)); } finally { rs.cleanup(); } - // Test2: test when no more workers are available due to topology worker max heap size limit but there is memory is still available - // wordSpout2 is going to contain 5 executors that needs scheduling. Each of those executors has a memory requirement of 128.0 MB - // The cluster contains 4 free WorkerSlots. For this topolology each worker is limited to a max heap size of 128.0 - // Thus, one executor not going to be able to get scheduled thus failing the scheduling of this topology and no executors of this + // Test2: test when no more workers are available due to topology worker max heap size limit + // but there is memory is still available + // wordSpout2 is going to contain 5 executors that needs scheduling. Each of those executors + // has a memory requirement of 128.0 MB + // The cluster contains 4 free WorkerSlots. For this topolology each worker is limited to a + // max heap size of 128.0 + // Thus, one executor not going to be able to get scheduled thus failing the scheduling of + // this topology and no executors of this // topology will be scheduled TopologyBuilder builder2 = new TopologyBuilder(); builder2.setSpout("wordSpout2", new TestWordSpout(), 5); @@ -753,16 +856,19 @@ public void testTopologyWorkerMaxHeapSize() { config2.putAll(defaultTopologyConf); config2.put(Config.TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB, 128.0); Map executorMap2 = genExecsAndComps(stormTopology2); - TopologyDetails topology2 = new TopologyDetails("topology2", config2, stormTopology2, 1, executorMap2, 0, "user"); + TopologyDetails topology2 = new TopologyDetails("topology2", config2, stormTopology2, 1, + executorMap2, 0, "user"); topologies = new Topologies(topology2); - cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config2); + cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, + new HashMap<>(), topologies, config2); rs.prepare(config2, new StormMetricsRegistry()); try { rs.schedule(topologies, cluster); assertTrue(cluster.needsSchedulingRas(topology2)); String status = cluster.getStatusMap().get(topology2.getId()); String expectedStatusPrefix = "Not enough resources to schedule"; - assertTrue(status.startsWith(expectedStatusPrefix), "Expected status to start with \"" + expectedStatusPrefix + "\" but status is: " + status); + assertTrue(status.startsWith(expectedStatusPrefix), "Expected status to start with \"" + + expectedStatusPrefix + "\" but status is: " + status); assertEquals(5, cluster.getUnassignedExecutors(topology2).size()); } finally { rs.cleanup(); @@ -771,7 +877,8 @@ public void testTopologyWorkerMaxHeapSize() { @Test public void testReadInResourceAwareSchedulerUserPools() { - Map fromFile = Utils.findAndReadConfigFile("user-resource-pools.yaml", false); + Map fromFile = Utils.findAndReadConfigFile("user-resource-pools.yaml", + false); LOG.info("fromFile: {}", fromFile); ConfigValidation.validateFields(fromFile); } @@ -783,7 +890,7 @@ public void testSubmitUsersWithNoGuarantees() { Map> resourceUserPool = userResourcePool( userRes("jerry", 200, 2000)); - for (Class strategyClass: strategyClasses) { + for (Class strategyClass : strategyClasses) { Config config = createClusterConfig(strategyClass, 100, 500, 500, resourceUserPool); Topologies topologies = new Topologies( @@ -792,19 +899,21 @@ public void testSubmitUsersWithNoGuarantees() { genTopology("topo-3", config, 1, 0, 1, 0, currentTime - 2, 20, "jerry"), genTopology("topo-4", config, 1, 0, 1, 0, currentTime - 2, 10, "bobby"), genTopology("topo-5", config, 1, 0, 1, 0, currentTime - 2, 20, "bobby")); - Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config); + Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), + supMap, new HashMap<>(), topologies, config); scheduler = new ResourceAwareScheduler(); scheduler.prepare(config, new StormMetricsRegistry()); scheduler.schedule(topologies, cluster); - assertTopologiesFullyScheduled(cluster, strategyClass, "topo-1", "topo-2", "topo-3", "topo-4"); + assertTopologiesFullyScheduled(cluster, strategyClass, "topo-1", "topo-2", "topo-3", + "topo-4"); assertTopologiesNotScheduled(cluster, strategyClass, "topo-5"); } } @Test public void testMultipleUsers() { - for (Class strategyClass: strategyClasses) { + for (Class strategyClass : strategyClasses) { if (strategyClass.getName().equals(RoundRobinResourceAwareStrategy.class.getName())) { continue; // exclude RoundRbin from this test } @@ -833,7 +942,8 @@ public void testMultipleUsers() { genTopology("topo-15", config, 5, 15, 1, 1, currentTime - 24, 29, "derek"), }; Topologies topologies = new Topologies(topos); - Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config); + Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), + supMap, new HashMap<>(), topologies, config); scheduler = new ResourceAwareScheduler(); scheduler.prepare(config, new StormMetricsRegistry()); @@ -844,7 +954,7 @@ public void testMultipleUsers() { @Test public void testHandlingClusterSubscription() { - for (Class strategyClass: strategyClasses) { + for (Class strategyClass : strategyClasses) { INimbus iNimbus = new INimbusTest(); Map supMap = genSupervisors(1, 4, 200, 1024 * 10); Map> resourceUserPool = userResourcePool( @@ -856,23 +966,25 @@ public void testHandlingClusterSubscription() { Topologies topologies = new Topologies( genTopology("topo-1", config, 5, 15, 1, 1, currentTime - 2, 20, "jerry"), genTopology("topo-2", config, 5, 15, 1, 1, currentTime - 8, 29, "jerry")); - Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config); + Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), + supMap, new HashMap<>(), topologies, config); scheduler = new ResourceAwareScheduler(); scheduler.prepare(config, new StormMetricsRegistry()); scheduler.schedule(topologies, cluster); - assertTopologiesFullyScheduled(cluster, strategyClass,"topo-1"); + assertTopologiesFullyScheduled(cluster, strategyClass, "topo-1"); assertTopologiesNotScheduled(cluster, strategyClass, "topo-2"); } } /** - * Test correct behavior when a supervisor dies. Check if the scheduler handles it correctly and evicts the correct + * Test correct behavior when a supervisor dies. Check if the scheduler handles it correctly and + * evicts the correct * topology when rescheduling the executors from the died supervisor */ @Test public void testFaultTolerance() { - for (Class strategyClass: strategyClasses) { + for (Class strategyClass : strategyClasses) { INimbus iNimbus = new INimbusTest(); Map supMap = genSupervisors(6, 4, 100, 1000); Map> resourceUserPool = userResourcePool( @@ -888,49 +1000,56 @@ public void testFaultTolerance() { genTopology("topo-4", config, 1, 0, 1, 0, currentTime - 2, 10, "bobby"), genTopology("topo-5", config, 1, 0, 1, 0, currentTime - 2, 29, "derek"), genTopology("topo-6", config, 1, 0, 1, 0, currentTime - 2, 10, "derek")); - Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config); + Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), + supMap, new HashMap<>(), topologies, config); scheduler = new ResourceAwareScheduler(); scheduler.prepare(config, new StormMetricsRegistry()); scheduler.schedule(topologies, cluster); - assertTopologiesFullyScheduled(cluster, strategyClass, "topo-1", "topo-2", "topo-3", "topo-4", "topo-5", "topo-6"); + assertTopologiesFullyScheduled(cluster, strategyClass, "topo-1", "topo-2", "topo-3", + "topo-4", "topo-5", "topo-6"); - //fail supervisor + // fail supervisor SupervisorDetails supFailed = cluster.getSupervisors().values().iterator().next(); LOG.info("/***** failing supervisor: {} ****/", supFailed.getHost()); supMap.remove(supFailed.getId()); Map newAssignments = new HashMap<>(); - for (Map.Entry topoToAssignment : cluster.getAssignments().entrySet()) { + for (Map.Entry topoToAssignment : cluster.getAssignments() + .entrySet()) { String topoId = topoToAssignment.getKey(); SchedulerAssignment assignment = topoToAssignment.getValue(); Map executorToSlots = new HashMap<>(); - for (Map.Entry execToWorker : assignment.getExecutorToSlot().entrySet()) { + for (Map.Entry execToWorker : assignment + .getExecutorToSlot().entrySet()) { ExecutorDetails exec = execToWorker.getKey(); WorkerSlot ws = execToWorker.getValue(); if (!ws.getNodeId().equals(supFailed.getId())) { executorToSlots.put(exec, ws); } } - newAssignments.put(topoId, new SchedulerAssignmentImpl(topoId, executorToSlots, null, null)); + newAssignments.put(topoId, new SchedulerAssignmentImpl(topoId, executorToSlots, + null, null)); } Map statusMap = cluster.getStatusMap(); LOG.warn("Rescheduling with removed Supervisor...."); - cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, newAssignments, topologies, config); + cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, + newAssignments, topologies, config); cluster.setStatusMap(statusMap); scheduler.schedule(topologies, cluster); - assertTopologiesFullyScheduled(cluster, strategyClass, "topo-2", "topo-3", "topo-4", "topo-5", "topo-6"); + assertTopologiesFullyScheduled(cluster, strategyClass, "topo-2", "topo-3", "topo-4", + "topo-5", "topo-6"); assertTopologiesNotScheduled(cluster, strategyClass, "topo-1"); } } /** - * test if free slots on nodes work correctly + * Test if free slots on nodes work correctly. */ @Test public void testNodeFreeSlot() { - for (Class strategyClass: strategyClasses) { + for (Class strategyClass : strategyClasses) { INimbus iNimbus = new INimbusTest(); Map supMap = genSupervisors(4, 4, 100, 1000); Config config = createClusterConfig(strategyClass, 100, 500, 500, null); @@ -938,7 +1057,8 @@ public void testNodeFreeSlot() { Topologies topologies = new Topologies( genTopology("topo-1", config, 1, 0, 2, 0, currentTime - 2, 29, "user"), genTopology("topo-2", config, 1, 0, 2, 0, currentTime - 2, 10, "user")); - Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config); + Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), + supMap, new HashMap<>(), topologies, config); scheduler = new ResourceAwareScheduler(); scheduler.prepare(config, new StormMetricsRegistry()); @@ -947,42 +1067,53 @@ public void testNodeFreeSlot() { Map nodes = RasNodes.getAllNodesFrom(cluster); for (SchedulerAssignment assignment : cluster.getAssignments().values()) { - for (Entry entry : new HashMap<>(assignment.getScheduledResources()).entrySet()) { + for (Entry entry : new HashMap<>(assignment + .getScheduledResources()).entrySet()) { WorkerSlot ws = entry.getKey(); WorkerResources wr = entry.getValue(); double memoryBefore = nodes.get(ws.getNodeId()).getAvailableMemoryResources(); double cpuBefore = nodes.get(ws.getNodeId()).getAvailableCpuResources(); double memoryUsedByWorker = wr.get_mem_on_heap() + wr.get_mem_off_heap(); - assertEquals(1000.0, memoryUsedByWorker, 0.001, "Check if memory used by worker is calculated correctly"); + assertEquals(1000.0, memoryUsedByWorker, 0.001, + "Check if memory used by worker is calculated correctly"); double cpuUsedByWorker = wr.get_cpu(); - assertEquals(100.0, cpuUsedByWorker, 0.001, "Check if CPU used by worker is calculated correctly"); + assertEquals(100.0, cpuUsedByWorker, 0.001, + "Check if CPU used by worker is calculated correctly"); nodes.get(ws.getNodeId()).free(ws); double memoryAfter = nodes.get(ws.getNodeId()).getAvailableMemoryResources(); double cpuAfter = nodes.get(ws.getNodeId()).getAvailableCpuResources(); - assertEquals(memoryBefore + memoryUsedByWorker, memoryAfter, 0.001, "Check if free correctly frees amount of memory"); - assertEquals(cpuBefore + cpuUsedByWorker, cpuAfter, 0.001, "Check if free correctly frees amount of memory"); - assertFalse(assignment.getSlotToExecutors().containsKey(ws), "Check if worker was removed from assignments"); + assertEquals(memoryBefore + memoryUsedByWorker, memoryAfter, 0.001, + "Check if free correctly frees amount of memory"); + assertEquals(cpuBefore + cpuUsedByWorker, cpuAfter, 0.001, + "Check if free correctly frees amount of memory"); + assertFalse(assignment.getSlotToExecutors().containsKey(ws), + "Check if worker was removed from assignments"); } } } } /** - * When the first topology failed to be scheduled make sure subsequent schedulings can still succeed + * When the first topology failed to be scheduled make sure subsequent schedulings can still + * succeed. */ @Test public void testSchedulingAfterFailedScheduling() { - for (Class strategyClass: strategyClasses) { + for (Class strategyClass : strategyClasses) { INimbus iNimbus = new INimbusTest(); Map supMap = genSupervisors(8, 4, 100, 1000); Config config = createClusterConfig(strategyClass, 100, 500, 500, null); - TopologyDetails topo1 = genTopology("topo-1", config, 8, 0, 2, 0, currentTime - 2, 10, "jerry"); - TopologyDetails topo2 = genTopology("topo-2", config, 2, 0, 2, 0, currentTime - 2, 20, "jerry"); - TopologyDetails topo3 = genTopology("topo-3", config, 1, 2, 1, 1, currentTime - 2, 20, "jerry"); + TopologyDetails topo1 = genTopology("topo-1", config, 8, 0, 2, 0, currentTime - 2, 10, + "jerry"); + TopologyDetails topo2 = genTopology("topo-2", config, 2, 0, 2, 0, currentTime - 2, 20, + "jerry"); + TopologyDetails topo3 = genTopology("topo-3", config, 1, 2, 1, 1, currentTime - 2, 20, + "jerry"); Topologies topologies = new Topologies(topo1, topo2, topo3); - Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap(), topologies, config); + Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), + supMap, new HashMap(), topologies, config); scheduler = new ResourceAwareScheduler(); scheduler.prepare(config, new StormMetricsRegistry()); @@ -990,9 +1121,11 @@ public void testSchedulingAfterFailedScheduling() { assertFalse(cluster.getAssignmentById(topo1.getId()) != null, "Topo-1 unscheduled?"); assertTrue(cluster.getAssignmentById(topo2.getId()) != null, "Topo-2 scheduled?"); - assertEquals(4, cluster.getAssignmentById(topo2.getId()).getExecutorToSlot().size(), "Topo-2 all executors scheduled?"); + assertEquals(4, cluster.getAssignmentById(topo2.getId()).getExecutorToSlot().size(), + "Topo-2 all executors scheduled?"); assertTrue(cluster.getAssignmentById(topo3.getId()) != null, "Topo-3 scheduled?"); - assertEquals(3, cluster.getAssignmentById(topo3.getId()).getExecutorToSlot().size(), "Topo-3 all executors scheduled?"); + assertEquals(3, cluster.getAssignmentById(topo3.getId()).getExecutorToSlot().size(), + "Topo-3 all executors scheduled?"); } } @@ -1002,14 +1135,16 @@ public void testSchedulingAfterFailedScheduling() { */ @Test public void minCpuWorkerJustFits() { - for (Class strategyClass: strategyClasses) { + for (Class strategyClass : strategyClasses) { INimbus iNimbus = new INimbusTest(); Map supMap = genSupervisors(1, 4, 100, 60000); Config config = createClusterConfig(strategyClass, 10, 500, 500, null); config.put(DaemonConfig.STORM_WORKER_MIN_CPU_PCORE_PERCENT, 50.0); - TopologyDetails topo1 = genTopology("topo-1", config, 10, 0, 1, 1, currentTime - 2, 20, "jerry"); + TopologyDetails topo1 = genTopology("topo-1", config, 10, 0, 1, 1, currentTime - 2, 20, + "jerry"); Topologies topologies = new Topologies(topo1); - Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap(), topologies, config); + Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), + supMap, new HashMap(), topologies, config); scheduler = new ResourceAwareScheduler(); scheduler.prepare(config, new StormMetricsRegistry()); scheduler.schedule(topologies, cluster); @@ -1020,11 +1155,12 @@ public void minCpuWorkerJustFits() { /** * Min CPU for worker set to 40%. 1 supervisor with 100% CPU. - * 2 topologies with 2 10% components should schedule. A third topology should then fail scheduling due to lack of CPU. + * 2 topologies with 2 10% components should schedule. A third topology should then fail + * scheduling due to lack of CPU. */ @Test public void minCpuPreventsThirdTopo() { - for (Class strategyClass: strategyClasses) { + for (Class strategyClass : strategyClasses) { if (strategyClass.getName().equals(RoundRobinResourceAwareStrategy.class.getName())) { continue; // exclude RoundRbin from this test } @@ -1032,20 +1168,27 @@ public void minCpuPreventsThirdTopo() { Map supMap = genSupervisors(1, 4, 100, 60000); Config config = createClusterConfig(strategyClass, 10, 500, 500, null); config.put(DaemonConfig.STORM_WORKER_MIN_CPU_PCORE_PERCENT, 40.0); - TopologyDetails topo1 = genTopology("topo-1", config, 2, 0, 1, 1, currentTime - 2, 20, "jerry"); - TopologyDetails topo2 = genTopology("topo-2", config, 2, 0, 1, 1, currentTime - 2, 20, "jerry"); - TopologyDetails topo3 = genTopology("topo-3", config, 2, 0, 1, 1, currentTime - 2, 20, "jerry"); + TopologyDetails topo1 = genTopology("topo-1", config, 2, 0, 1, 1, currentTime - 2, 20, + "jerry"); + TopologyDetails topo2 = genTopology("topo-2", config, 2, 0, 1, 1, currentTime - 2, 20, + "jerry"); + TopologyDetails topo3 = genTopology("topo-3", config, 2, 0, 1, 1, currentTime - 2, 20, + "jerry"); Topologies topologies = new Topologies(topo1, topo2, topo3); - Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap(), topologies, config); + Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), + supMap, new HashMap(), topologies, config); scheduler = new ResourceAwareScheduler(); scheduler.prepare(config, new StormMetricsRegistry()); scheduler.schedule(topologies, cluster); assertFalse(cluster.needsSchedulingRas(topo1), "using " + strategyClass); assertFalse(cluster.needsSchedulingRas(topo2), "using " + strategyClass); assertTrue(cluster.needsSchedulingRas(topo3), "using " + strategyClass); - assertTrue(cluster.getAssignmentById(topo1.getId()) != null, "topo-1 scheduled? using " + strategyClass); - assertTrue(cluster.getAssignmentById(topo2.getId()) != null, "topo-2 scheduled? using " + strategyClass); - assertFalse(cluster.getAssignmentById(topo3.getId()) != null, "topo-3 unscheduled? using " + strategyClass); + assertTrue(cluster.getAssignmentById(topo1.getId()) != null, "topo-1 scheduled? using " + + strategyClass); + assertTrue(cluster.getAssignmentById(topo2.getId()) != null, "topo-2 scheduled? using " + + strategyClass); + assertFalse(cluster.getAssignmentById(topo3.getId()) != null, + "topo-3 unscheduled? using " + strategyClass); SchedulerAssignment assignment1 = cluster.getAssignmentById(topo1.getId()); assertEquals(1, assignment1.getSlots().size()); @@ -1072,7 +1215,7 @@ public void minCpuPreventsThirdTopo() { @Test public void testMinCpuMaxMultipleSupervisors() { int topoCnt = 10; - for (Class strategyClass: strategyClasses) { + for (Class strategyClass : strategyClasses) { if (strategyClass.getName().equals(RoundRobinResourceAwareStrategy.class.getName())) { continue; // exclude RoundRbin from this test } @@ -1081,12 +1224,13 @@ public void testMinCpuMaxMultipleSupervisors() { Config config = createClusterConfig(strategyClass, 5, 50, 50, null); config.put(DaemonConfig.STORM_WORKER_MIN_CPU_PCORE_PERCENT, 100.0); TopologyDetails[] topos = new TopologyDetails[topoCnt]; - for (int i = 0 ; i < topoCnt ; i++) { + for (int i = 0; i < topoCnt; i++) { String topoName = "topo-" + i; topos[i] = genTopology(topoName, config, 4, 5, 1, 1, currentTime - 2, 20, "jerry"); } Topologies topologies = new Topologies(topos); - Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap(), topologies, config); + Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), + supMap, new HashMap(), topologies, config); scheduler = new ResourceAwareScheduler(); scheduler.prepare(config, new StormMetricsRegistry()); scheduler.schedule(topologies, cluster); @@ -1102,7 +1246,7 @@ public void testMinCpuMaxMultipleSupervisors() { */ @Test public void minCpuWorkerSplitFails() { - for (Class strategyClass: strategyClasses) { + for (Class strategyClass : strategyClasses) { INimbus iNimbus = new INimbusTest(); Map supMap = genSupervisors(1, 4, 100, 60000); Config config = createClusterConfig(strategyClass, 10, 500, 500, null); @@ -1110,7 +1254,8 @@ public void minCpuWorkerSplitFails() { TopologyDetails topo1 = genTopology("topo-1", config, 10, 0, 1, 1, currentTime - 2, 20, "jerry", 2000.0); Topologies topologies = new Topologies(topo1); - Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap(), topologies, config); + Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), + supMap, new HashMap(), topologies, config); scheduler = new ResourceAwareScheduler(); scheduler.prepare(config, new StormMetricsRegistry()); scheduler.schedule(topologies, cluster); @@ -1136,7 +1281,8 @@ void append(TimeBlockResult other) { private long getMedianValue(List values) { final int numValues = values.size(); - assertEquals(1, (numValues % 2), "Expecting odd number of values to compute median, got " + numValues); + assertEquals(1, (numValues % 2), "Expecting odd number of values to compute median, got " + + numValues); List sortedValues = new ArrayList<>(); sortedValues.addAll(values); Collections.sort(sortedValues); @@ -1146,11 +1292,15 @@ private long getMedianValue(List values) { } /** - * Check time to schedule a fragmented cluster using different strategies + * Check time to schedule a fragmented cluster using different strategies. * - * Simulate scheduling on a large production cluster. Find the ratio of time to schedule a set of topologies when - * the cluster is empty and when the cluster is nearly full. While the cluster has sufficient resources to schedule - * all topologies, when nearly full the cluster becomes fragmented and some topologies fail to schedule. + *

      Simulate scheduling on a large production cluster. Find the ratio of time to schedule a + * set + * of topologies when + * the cluster is empty and when the cluster is nearly full. While the cluster has sufficient + * resources to schedule + * all topologies, when nearly full the cluster becomes fragmented and some topologies fail to + * schedule. */ @Test public void TestLargeFragmentedClusterScheduling() { @@ -1202,13 +1352,15 @@ public void TestLargeFragmentedClusterScheduling() { Map strategyToConfigs = new HashMap<>(); // AcceptedBlockTimeRatios obtained by empirical testing (see comment block above) Map strategyToAcceptedBlockTimeRatios = new HashMap<>(); - for (Class strategyClass: strategyClasses) { - strategyToConfigs.put(strategyClass.getName(), createClusterConfig(strategyClass, 10, 10, 0, null)); + for (Class strategyClass : strategyClasses) { + strategyToConfigs.put(strategyClass.getName(), createClusterConfig(strategyClass, 10, + 10, 0, null)); strategyToAcceptedBlockTimeRatios.put(strategyClass.getName(), 6.96); } strategyToAcceptedBlockTimeRatios.put(DefaultResourceAwareStrategy.class.getName(), 6.96); strategyToAcceptedBlockTimeRatios.put(GenericResourceAwareStrategy.class.getName(), 7.78); - strategyToConfigs.put(ConstraintSolverStrategy.class.getName(), createCSSClusterConfig(10, 10, 0, null)); + strategyToConfigs.put(ConstraintSolverStrategy.class.getName(), createCSSClusterConfig(10, + 10, 0, null)); strategyToAcceptedBlockTimeRatios.put(ConstraintSolverStrategy.class.getName(), 7.75); Map strategyToTimeBlockResults = new HashMap<>(); @@ -1216,34 +1368,43 @@ public void TestLargeFragmentedClusterScheduling() { // Get first and last block times for multiple runs and strategies long startTime = Time.currentTimeMillis(); for (Entry strategyConfig : strategyToConfigs.entrySet()) { - TimeBlockResult strategyTimeBlockResult = strategyToTimeBlockResults.computeIfAbsent(strategyConfig.getKey(), (k) -> new TimeBlockResult()); + TimeBlockResult strategyTimeBlockResult = strategyToTimeBlockResults + .computeIfAbsent(strategyConfig.getKey(), (k) -> new TimeBlockResult()); for (int run = 0; run < numRuns; ++run) { - TimeBlockResult result = testLargeClusterSchedulingTiming(numNodes, strategyConfig.getValue()); + TimeBlockResult result = testLargeClusterSchedulingTiming(numNodes, strategyConfig + .getValue()); strategyTimeBlockResult.append(result); } } // Log median ratios for different strategies - LOG.info("TestLargeFragmentedClusterScheduling took {} ms", Time.currentTimeMillis() - startTime); - for (Entry strategyResult : strategyToTimeBlockResults.entrySet()) { + LOG.info("TestLargeFragmentedClusterScheduling took {} ms", Time + .currentTimeMillis() - startTime); + for (Entry strategyResult : strategyToTimeBlockResults + .entrySet()) { TimeBlockResult strategyTimeBlockResult = strategyResult.getValue(); double medianFirstBlockTime = getMedianValue(strategyTimeBlockResult.firstBlockTime); double medianLastBlockTime = getMedianValue(strategyTimeBlockResult.lastBlockTime); double ratio = medianLastBlockTime / medianFirstBlockTime; - LOG.info("{}, FirstBlock {}, LastBlock {} ratio {}", strategyResult.getKey(), medianFirstBlockTime, medianLastBlockTime, ratio); + LOG.info("{}, FirstBlock {}, LastBlock {} ratio {}", strategyResult.getKey(), + medianFirstBlockTime, medianLastBlockTime, ratio); } // Check last block scheduling time does not get significantly slower - for (Entry strategyResult : strategyToTimeBlockResults.entrySet()) { + for (Entry strategyResult : strategyToTimeBlockResults + .entrySet()) { TimeBlockResult strategyTimeBlockResult = strategyResult.getValue(); double medianFirstBlockTime = getMedianValue(strategyTimeBlockResult.firstBlockTime); double medianLastBlockTime = getMedianValue(strategyTimeBlockResult.lastBlockTime); double ratio = medianLastBlockTime / medianFirstBlockTime; double slowSchedulingThreshold = 1.5; - String msg = "Strategy " + strategyResult.getKey() + " scheduling is significantly slower for mostly full fragmented cluster\n"; + String msg = "Strategy " + strategyResult.getKey() + + " scheduling is significantly slower for mostly full fragmented cluster\n"; double ratioAccepted = strategyToAcceptedBlockTimeRatios.get(strategyResult.getKey()); - msg += String.format("Ratio was %.2f (high/low=%.2f/%.2f), max allowed is %.2f (%.2f * %.2f)", + msg += String + .format("Ratio was %.2f (high/low=%.2f/%.2f), max allowed is %.2f (%.2f * " + + "%.2f)", ratio, medianLastBlockTime, medianFirstBlockTime, ratioAccepted * slowSchedulingThreshold, ratioAccepted, slowSchedulingThreshold); assertTrue(ratio < slowSchedulingThreshold * ratioAccepted, msg); @@ -1251,7 +1412,8 @@ public void TestLargeFragmentedClusterScheduling() { } // Create multiple copies of a test topology - private void addTopologyBlockToMap(Map topologyMap, String baseName, Config config, + private void addTopologyBlockToMap(Map topologyMap, String baseName, + Config config, double spoutMemoryLoad, int[] blockIndices) { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("testSpout", new TestSpout(), 1).setMemoryLoad(spoutMemoryLoad); @@ -1259,7 +1421,8 @@ private void addTopologyBlockToMap(Map topologyMap, Str Map executorMap = genExecsAndComps(stormTopology); for (int i = blockIndices[0]; i <= blockIndices[1]; ++i) { - TopologyDetails topo = new TopologyDetails(baseName + i, config, stormTopology, 0, executorMap, 0, "user"); + TopologyDetails topo = new TopologyDetails(baseName + i, config, stormTopology, 0, + executorMap, 0, "user"); topologyMap.put(topo.getId(), topo); } } @@ -1268,16 +1431,18 @@ private void addTopologyBlockToMap(Map topologyMap, Str * Test time to schedule large cluster scheduling with fragmentation */ private TimeBlockResult testLargeClusterSchedulingTiming(int numNodes, Config config) { - // Attempt to schedule multiple copies of 2 different topologies (topo-t0 and topo-t1) in 3 blocks. - // Without fragmentation it is possible to schedule all topologies, but fragmentation causes topologies to not + // Attempt to schedule multiple copies of 2 different topologies (topo-t0 and topo-t1) in 3 + // blocks. + // Without fragmentation it is possible to schedule all topologies, but fragmentation causes + // topologies to not // schedule for the last block. // Get start/end indices for blocks int numTopologyPairs = numNodes; int increment = (int) Math.floor(numTopologyPairs * 0.1); - int firstBlockIndices[] = {0, increment - 1}; - int midBlockIndices[] = {increment, numTopologyPairs - increment - 1}; - int lastBlockIndices[] = {numTopologyPairs - increment, numTopologyPairs - 1}; + int[] firstBlockIndices = {0, increment - 1}; + int[] midBlockIndices = {increment, numTopologyPairs - increment - 1}; + int[] lastBlockIndices = {numTopologyPairs - increment, numTopologyPairs - 1}; // Memory is the constraining resource. double t0Mem = 70; // memory required by topo-t0 @@ -1291,7 +1456,8 @@ private TimeBlockResult testLargeClusterSchedulingTiming(int numNodes, Config co Topologies topologies = new Topologies(topologyMap); Map supMap = genSupervisors(numNodes, 7, 3500, nodeMem); - Cluster cluster = new Cluster(new INimbusTest(), new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap(), topologies, config); + Cluster cluster = new Cluster(new INimbusTest(), + new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap(), topologies, config); TimeBlockResult timeBlockResult = new TimeBlockResult(); // schedule first block (0% - 10%) @@ -1332,11 +1498,11 @@ private TimeBlockResult testLargeClusterSchedulingTiming(int numNodes, Config co } /** - * Test multiple spouts and cyclic topologies + * Test multiple spouts and cyclic topologies. */ @Test public void testMultipleSpoutsAndCyclicTopologies() { - for (Class strategyClass: strategyClasses) { + for (Class strategyClass : strategyClasses) { String strategyClassName = strategyClass.getName(); TopologyBuilder builder = new TopologyBuilder(); @@ -1362,20 +1528,22 @@ public void testMultipleSpoutsAndCyclicTopologies() { 0, genExecsAndComps(stormTopology), 0, "jerry"); Topologies topologies = new Topologies(topo); - Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap(), topologies, config); + Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), + supMap, new HashMap(), topologies, config); scheduler = new ResourceAwareScheduler(); scheduler.prepare(config, new StormMetricsRegistry()); scheduler.schedule(topologies, cluster); assertTrue(cluster.getAssignmentById(topo.getId()) != null, "Topo scheduled?"); - assertEquals(25, cluster.getAssignmentById(topo.getId()).getExecutorToSlot().size(), "Topo all executors scheduled?"); + assertEquals(25, cluster.getAssignmentById(topo.getId()).getExecutorToSlot().size(), + "Topo all executors scheduled?"); } } @Test public void testSchedulerStrategyWhitelist() { Map config = ConfigUtils.readStormConfig(); - for (Class strategyClass: strategyClasses) { + for (Class strategyClass : strategyClasses) { String strategyClassName = strategyClass.getName(); String allowed = strategyClassName; config.put(Config.NIMBUS_SCHEDULER_STRATEGY_CLASS_WHITELIST, Arrays.asList(allowed)); @@ -1388,20 +1556,22 @@ public void testSchedulerStrategyWhitelist() { @Test public void testSchedulerStrategyWhitelistException() { Map config = ConfigUtils.readStormConfig(); - String allowed = "org.apache.storm.scheduler.resource.strategies.scheduling.SomeNonExistantStrategy"; - for (Class strategyClass: strategyClasses) { + String allowed = + "org.apache.storm.scheduler.resource.strategies.scheduling.SomeNonExistantStrategy"; + for (Class strategyClass : strategyClasses) { String strategyClassName = strategyClass.getName(); String notAllowed = strategyClassName; config.put(Config.NIMBUS_SCHEDULER_STRATEGY_CLASS_WHITELIST, Arrays.asList(allowed)); - Assertions.assertThrows(DisallowedStrategyException.class, () -> ReflectionUtils.newSchedulerStrategyInstance(notAllowed, config)); + Assertions.assertThrows(DisallowedStrategyException.class, () -> ReflectionUtils + .newSchedulerStrategyInstance(notAllowed, config)); } } @Test public void testSchedulerStrategyEmptyWhitelist() { Map config = ConfigUtils.readStormConfig(); - for (Class strategyClass: strategyClasses) { + for (Class strategyClass : strategyClasses) { String strategyClassName = strategyClass.getName(); String allowed = strategyClassName; Object sched = ReflectionUtils.newSchedulerStrategyInstance(allowed, config); @@ -1412,7 +1582,7 @@ public void testSchedulerStrategyEmptyWhitelist() { @PerformanceTest @Test public void testLargeTopologiesOnLargeClusters() { - for (Class strategyClass: strategyClasses) { + for (Class strategyClass : strategyClasses) { String strategyClassName = strategyClass.getName(); Assertions.assertTimeoutPreemptively(Duration.ofSeconds(30), () -> testLargeTopologiesCommon(strategyClassName, false, 1)); @@ -1424,14 +1594,16 @@ public void testLargeTopologiesOnLargeClusters() { @Test public void testLargeTopologiesOnLargeClustersGras() { Assertions.assertTimeoutPreemptively(Duration.ofSeconds(75), - () -> testLargeTopologiesCommon(GenericResourceAwareStrategy.class.getName(), true, 1)); + () -> testLargeTopologiesCommon(GenericResourceAwareStrategy.class.getName(), true, + 1)); } @PerformanceTest @Test public void testLargeTopologiesOnLargeClustersRoundRobin() { Assertions.assertTimeoutPreemptively(Duration.ofSeconds(30), - () -> testLargeTopologiesCommon(RoundRobinResourceAwareStrategy.class.getName(), true, 1)); + () -> testLargeTopologiesCommon(RoundRobinResourceAwareStrategy.class.getName(), + true, 1)); } public static class NeverEndingSchedulingStrategy extends BaseResourceAwareStrategy { @@ -1453,7 +1625,7 @@ public SchedulingResult schedule(Cluster schedulingState, TopologyDetails td) { @Test public void testStrategyTakingTooLong() { - for (Class strategyClass: strategyClasses) { + for (Class strategyClass : strategyClasses) { INimbus iNimbus = new INimbusTest(); Map supMap = genSupervisors(8, 4, 100, 1000); Config config = createClusterConfig(strategyClass, 100, 500, 500, null); @@ -1464,17 +1636,23 @@ public void testStrategyTakingTooLong() { allowedSchedulerStrategies.add(GenericResourceAwareStrategyOld.class.getName()); allowedSchedulerStrategies.add(RoundRobinResourceAwareStrategy.class.getName()); allowedSchedulerStrategies.add(NeverEndingSchedulingStrategy.class.getName()); - config.put(Config.NIMBUS_SCHEDULER_STRATEGY_CLASS_WHITELIST, allowedSchedulerStrategies); + config.put(Config.NIMBUS_SCHEDULER_STRATEGY_CLASS_WHITELIST, + allowedSchedulerStrategies); config.put(DaemonConfig.SCHEDULING_TIMEOUT_SECONDS_PER_TOPOLOGY, 30); - TopologyDetails topo1 = genTopology("topo-1", config, 1, 0, 2, 0, currentTime - 2, 10, "jerry"); - TopologyDetails topo3 = genTopology("topo-3", config, 1, 2, 1, 1, currentTime - 2, 20, "jerry"); + TopologyDetails topo1 = genTopology("topo-1", config, 1, 0, 2, 0, currentTime - 2, 10, + "jerry"); + TopologyDetails topo3 = genTopology("topo-3", config, 1, 2, 1, 1, currentTime - 2, 20, + "jerry"); - config.put(Config.TOPOLOGY_SCHEDULER_STRATEGY, NeverEndingSchedulingStrategy.class.getName()); - TopologyDetails topo2 = genTopology("topo-2", config, 2, 0, 2, 0, currentTime - 2, 20, "jerry"); + config.put(Config.TOPOLOGY_SCHEDULER_STRATEGY, NeverEndingSchedulingStrategy.class + .getName()); + TopologyDetails topo2 = genTopology("topo-2", config, 2, 0, 2, 0, currentTime - 2, 20, + "jerry"); Topologies topologies = new Topologies(topo1, topo2, topo3); - Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config); + Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), + supMap, new HashMap<>(), topologies, config); scheduler = new ResourceAwareScheduler(); scheduler.prepare(config, new StormMetricsRegistry()); @@ -1485,24 +1663,31 @@ public void testStrategyTakingTooLong() { assertFalse(cluster.needsSchedulingRas(topo3)); assertTrue(cluster.getAssignmentById(topo1.getId()) != null, "Topo-1 scheduled?"); - assertEquals(2, cluster.getAssignmentById(topo1.getId()).getExecutorToSlot().size(), "Topo-1 all executors scheduled?"); + assertEquals(2, cluster.getAssignmentById(topo1.getId()).getExecutorToSlot().size(), + "Topo-1 all executors scheduled?"); assertTrue(cluster.getAssignmentById(topo2.getId()) == null, "Topo-2 not scheduled"); assertEquals("Scheduling took too long for " + topo2.getId() + " using strategy " + NeverEndingSchedulingStrategy.class.getName() - + " timeout after 30 seconds using config scheduling.timeout.seconds.per.topology.", cluster.getStatusMap().get(topo2.getId())); + + " timeout after 30 seconds using config " + + "scheduling.timeout.seconds.per.topology.", cluster.getStatusMap().get(topo2 + .getId())); assertTrue(cluster.getAssignmentById(topo3.getId()) != null, "Topo-3 scheduled?"); - assertEquals(3, cluster.getAssignmentById(topo3.getId()).getExecutorToSlot().size(), "Topo-3 all executors scheduled?"); + assertEquals(3, cluster.getAssignmentById(topo3.getId()).getExecutorToSlot().size(), + "Topo-3 all executors scheduled?"); } } - public void testLargeTopologiesCommon(final String strategy, final boolean includeGpu, final int multiplier) { - for (Class strategyClass: strategyClasses) { + public void testLargeTopologiesCommon(final String strategy, final boolean includeGpu, + final int multiplier) { + for (Class strategyClass : strategyClasses) { INimbus iNimbus = new INimbusTest(); - Map supMap = genSupervisorsWithRacks(25 * multiplier, 40, 66, 3 * multiplier, 0, 4700, 226200, new HashMap<>()); + Map supMap = genSupervisorsWithRacks(25 * multiplier, 40, 66, + 3 * multiplier, 0, 4700, 226200, new HashMap<>()); if (includeGpu) { HashMap extraResources = new HashMap<>(); extraResources.put("my.gpu", 1.0); - supMap.putAll(genSupervisorsWithRacks(3 * multiplier, 40, 66, 0, 0, 4700, 226200, extraResources)); + supMap.putAll(genSupervisorsWithRacks(3 * multiplier, 40, 66, 0, 0, 4700, 226200, + extraResources)); } Config config = new Config(); @@ -1523,13 +1708,15 @@ public void testLargeTopologiesCommon(final String strategy, final boolean inclu builder.setBolt("gpu-bolt", new TestBolt(), 40) .addResource("my.gpu", 1.0) .shuffleGrouping("spout-0"); - TopologyDetails td = topoToTopologyDetails(String.format("topology-gpu-%05d", i), config, builder.createTopology(), 0, 0, + TopologyDetails td = topoToTopologyDetails(String.format("topology-gpu-%05d", + i), config, builder.createTopology(), 0, 0, "user", 8192); topologyDetailsMap.put(td.getId(), td); } } Topologies topologies = new Topologies(topologyDetailsMap); - Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config); + Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), + supMap, new HashMap<>(), topologies, config); long startTime = Time.currentTimeMillis(); scheduler.prepare(config, new StormMetricsRegistry()); diff --git a/storm-server/src/test/java/org/apache/storm/scheduler/resource/TestUser.java b/storm-server/src/test/java/org/apache/storm/scheduler/resource/TestUser.java index d3f989dd00a..fba7005ae5b 100644 --- a/storm-server/src/test/java/org/apache/storm/scheduler/resource/TestUser.java +++ b/storm-server/src/test/java/org/apache/storm/scheduler/resource/TestUser.java @@ -1,17 +1,30 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

      - * http://www.apache.org/licenses/LICENSE-2.0 - *

      - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

      http://www.apache.org/licenses/LICENSE-2.0 + * + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.scheduler.resource; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.genSupervisors; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.genTopology; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.toDouble; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.userRes; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.userResourcePool; +import static org.junit.jupiter.api.Assertions.assertEquals; + import java.util.HashMap; import java.util.Map; import org.apache.storm.Config; @@ -22,8 +35,8 @@ import org.apache.storm.scheduler.Topologies; import org.apache.storm.scheduler.TopologyDetails; import org.apache.storm.scheduler.WorkerSlot; -import org.apache.storm.scheduler.resource.normalization.ResourceMetrics; import org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.INimbusTest; +import org.apache.storm.scheduler.resource.normalization.ResourceMetrics; import org.apache.storm.scheduler.resource.strategies.scheduling.DefaultResourceAwareStrategy; import org.apache.storm.scheduler.resource.strategies.scheduling.DefaultResourceAwareStrategyOld; import org.apache.storm.scheduler.resource.strategies.scheduling.GenericResourceAwareStrategy; @@ -32,33 +45,27 @@ import org.apache.storm.utils.Time; import org.junit.jupiter.api.Test; -import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.genSupervisors; -import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.genTopology; -import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.toDouble; -import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.userRes; -import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.userResourcePool; -import static org.junit.jupiter.api.Assertions.assertEquals; - - public class TestUser { private static final Class[] strategyClasses = { - DefaultResourceAwareStrategy.class, - DefaultResourceAwareStrategyOld.class, - RoundRobinResourceAwareStrategy.class, - GenericResourceAwareStrategy.class, - GenericResourceAwareStrategyOld.class, + DefaultResourceAwareStrategy.class, + DefaultResourceAwareStrategyOld.class, + RoundRobinResourceAwareStrategy.class, + GenericResourceAwareStrategy.class, + GenericResourceAwareStrategyOld.class, }; - private Config createClusterConfig(Class strategyClass, double compPcore, double compOnHeap, double compOffHeap, + private Config createClusterConfig(Class strategyClass, double compPcore, double compOnHeap, + double compOffHeap, Map> pools) { - Config config = TestUtilsForResourceAwareScheduler.createClusterConfig(compPcore, compOnHeap, compOffHeap, pools); + Config config = TestUtilsForResourceAwareScheduler.createClusterConfig(compPcore, + compOnHeap, compOffHeap, pools); config.put(Config.TOPOLOGY_SCHEDULER_STRATEGY, strategyClass.getName()); return config; } @Test public void testResourcePoolUtilization() { - for (Class strategyClass: strategyClasses) { + for (Class strategyClass : strategyClasses) { INimbus iNimbus = new INimbusTest(); Map supMap = genSupervisors(4, 4, 100, 1000); double cpuGuarantee = 400.0; @@ -66,20 +73,26 @@ public void testResourcePoolUtilization() { Map> resourceUserPool = userResourcePool( userRes("user1", cpuGuarantee, memoryGuarantee)); Config config = createClusterConfig(strategyClass, 100, 200, 200, resourceUserPool); - TopologyDetails topo1 = genTopology("topo-1", config, 1, 1, 2, 1, Time.currentTimeSecs() - 24, 9, "user1"); + TopologyDetails topo1 = genTopology("topo-1", config, 1, 1, 2, 1, Time + .currentTimeSecs() - 24, 9, "user1"); Topologies topologies = new Topologies(topo1); - Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config); + Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), + supMap, new HashMap<>(), topologies, config); User user1 = new User("user1", toDouble(resourceUserPool.get("user1"))); WorkerSlot slot = cluster.getAvailableSlots().get(0); cluster.assign(slot, topo1.getId(), topo1.getExecutors()); - assertEquals(cpuGuarantee, user1.getCpuResourceGuaranteed(), 0.001, "check cpu resource guarantee"); - assertEquals(memoryGuarantee, user1.getMemoryResourceGuaranteed(), 0.001, "check memory resource guarantee"); + assertEquals(cpuGuarantee, user1.getCpuResourceGuaranteed(), 0.001, + "check cpu resource guarantee"); + assertEquals(memoryGuarantee, user1.getMemoryResourceGuaranteed(), 0.001, + "check memory resource guarantee"); - assertEquals(((100.0 * 3.0) / cpuGuarantee), user1.getCpuResourcePoolUtilization(cluster), 0.001, + assertEquals(((100.0 * 3.0) / cpuGuarantee), user1 + .getCpuResourcePoolUtilization(cluster), 0.001, "check cpu resource pool utilization"); - assertEquals(((200.0 + 200.0) * 3.0) / memoryGuarantee, user1.getMemoryResourcePoolUtilization(cluster), 0.001, + assertEquals(((200.0 + 200.0) * 3.0) / memoryGuarantee, user1 + .getMemoryResourcePoolUtilization(cluster), 0.001, "check memory resource pool utilization"); } } diff --git a/storm-server/src/test/java/org/apache/storm/scheduler/resource/TestUtilsForResourceAwareScheduler.java b/storm-server/src/test/java/org/apache/storm/scheduler/resource/TestUtilsForResourceAwareScheduler.java index 3647aa664f4..14872e2446d 100644 --- a/storm-server/src/test/java/org/apache/storm/scheduler/resource/TestUtilsForResourceAwareScheduler.java +++ b/storm-server/src/test/java/org/apache/storm/scheduler/resource/TestUtilsForResourceAwareScheduler.java @@ -1,25 +1,44 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

      - * http://www.apache.org/licenses/LICENSE-2.0 - *

      - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

      http://www.apache.org/licenses/LICENSE-2.0 + * + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.scheduler.resource; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map.Entry; +import java.util.Map; +import java.util.Random; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import org.apache.storm.networktopography.DNSToSwitchMapping; -import org.apache.storm.scheduler.resource.normalization.NormalizedResources; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.apache.storm.Config; import org.apache.storm.DaemonConfig; import org.apache.storm.generated.Bolt; import org.apache.storm.generated.SpoutSpec; import org.apache.storm.generated.StormTopology; +import org.apache.storm.networktopography.DNSToSwitchMapping; import org.apache.storm.scheduler.Cluster; import org.apache.storm.scheduler.ExecutorDetails; import org.apache.storm.scheduler.INimbus; @@ -30,6 +49,7 @@ import org.apache.storm.scheduler.Topologies; import org.apache.storm.scheduler.TopologyDetails; import org.apache.storm.scheduler.WorkerSlot; +import org.apache.storm.scheduler.resource.normalization.NormalizedResources; import org.apache.storm.scheduler.resource.strategies.priority.DefaultSchedulingPriorityStrategy; import org.apache.storm.scheduler.resource.strategies.scheduling.ConstraintSolverStrategy; import org.apache.storm.scheduler.resource.strategies.scheduling.DefaultResourceAwareStrategy; @@ -52,23 +72,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Random; -import java.util.Set; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import static org.junit.jupiter.api.Assertions.assertEquals; - public class TestUtilsForResourceAwareScheduler { - private static final Logger LOG = LoggerFactory.getLogger(TestUtilsForResourceAwareScheduler.class); + private static final Logger LOG = LoggerFactory + .getLogger(TestUtilsForResourceAwareScheduler.class); public static class TestUserResources { private final String name; @@ -107,11 +113,12 @@ public static Map> userResourcePool(TestUserResource return ret; } - public static Config createCSSClusterConfig(double compPcore, double compOnHeap, double compOffHeap, + public static Config createCSSClusterConfig(double compPcore, double compOnHeap, + double compOffHeap, Map> pools) { Config config = createClusterConfig(compPcore, compOnHeap, compOffHeap, pools); config.put(Config.TOPOLOGY_SCHEDULER_STRATEGY, ConstraintSolverStrategy.class.getName()); - Map> modifiedConstraints = new HashMap<>(); + Map> modifiedConstraints = new HashMap<>(); Map contraints = new HashMap<>(); contraints.put("maxNodeCoLocationCnt", 1); modifiedConstraints.put("testSpout", contraints); @@ -119,29 +126,37 @@ public static Config createCSSClusterConfig(double compPcore, double compOnHeap, return config; } - public static Config createRoundRobinClusterConfig(double compPcore, double compOnHeap, double compOffHeap, + public static Config createRoundRobinClusterConfig(double compPcore, double compOnHeap, + double compOffHeap, Map> pools, Map genericResourceMap) { Config config = createClusterConfig(compPcore, compOnHeap, compOffHeap, pools); config.put(Config.TOPOLOGY_COMPONENT_RESOURCES_MAP, genericResourceMap); - config.put(Config.TOPOLOGY_SCHEDULER_STRATEGY, RoundRobinResourceAwareStrategy.class.getName()); + config.put(Config.TOPOLOGY_SCHEDULER_STRATEGY, RoundRobinResourceAwareStrategy.class + .getName()); return config; } - public static Config createGrasClusterConfig(double compPcore, double compOnHeap, double compOffHeap, + public static Config createGrasClusterConfig(double compPcore, double compOnHeap, + double compOffHeap, Map> pools, Map genericResourceMap) { Config config = createClusterConfig(compPcore, compOnHeap, compOffHeap, pools); config.put(Config.TOPOLOGY_COMPONENT_RESOURCES_MAP, genericResourceMap); - config.put(Config.TOPOLOGY_SCHEDULER_STRATEGY, GenericResourceAwareStrategy.class.getName()); + config.put(Config.TOPOLOGY_SCHEDULER_STRATEGY, GenericResourceAwareStrategy.class + .getName()); return config; } - public static Config createClusterConfig(double compPcore, double compOnHeap, double compOffHeap, + public static Config createClusterConfig(double compPcore, double compOnHeap, + double compOffHeap, Map> pools) { Config config = new Config(); config.putAll(Utils.readDefaultConfig()); - config.put(Config.STORM_NETWORK_TOPOGRAPHY_PLUGIN, GenSupervisorsDnsToSwitchMapping.class.getName()); - config.put(DaemonConfig.RESOURCE_AWARE_SCHEDULER_PRIORITY_STRATEGY, DefaultSchedulingPriorityStrategy.class.getName()); - config.put(Config.TOPOLOGY_SCHEDULER_STRATEGY, DefaultResourceAwareStrategy.class.getName()); + config.put(Config.STORM_NETWORK_TOPOGRAPHY_PLUGIN, GenSupervisorsDnsToSwitchMapping.class + .getName()); + config.put(DaemonConfig.RESOURCE_AWARE_SCHEDULER_PRIORITY_STRATEGY, + DefaultSchedulingPriorityStrategy.class.getName()); + config.put(Config.TOPOLOGY_SCHEDULER_STRATEGY, DefaultResourceAwareStrategy.class + .getName()); config.put(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT, compPcore); config.put(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB, compOffHeap); config.put(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB, compOnHeap); @@ -151,7 +166,8 @@ public static Config createClusterConfig(double compPcore, double compOnHeap, do return config; } - public static Map genSupervisors(int numSup, int numPorts, double cpu, double mem) { + public static Map genSupervisors(int numSup, int numPorts, + double cpu, double mem) { return genSupervisors(numSup, numPorts, 0, cpu, mem); } @@ -196,11 +212,12 @@ public static class GenSupervisorsDnsToSwitchMapping implements DNSToSwitchMappi private Map mappingCache = new ConcurrentHashMap<>(); @Override - public Map resolve(List names) { + public Map resolve(List names) { Map m = new HashMap<>(); for (String name : names) { - m.put(name, mappingCache.computeIfAbsent(name, TestUtilsForResourceAwareScheduler::hostNameToRackName)); + m.put(name, mappingCache.computeIfAbsent(name, + TestUtilsForResourceAwareScheduler::hostNameToRackName)); } return m; } @@ -215,8 +232,10 @@ public static Map genSupervisorsWithRacks( } /** - * Takes one additional parameter numaZonesPerHost. This parameter determines how many supervisors - * will be created on the same host. If numaResourceMultiplier is set to a factor below 1.0, then + * Takes one additional parameter numaZonesPerHost. This parameter determines how many + * supervisors + * will be created on the same host. If numaResourceMultiplier is set to a factor below 1.0, + * then * each subsequent numa zone will have corresponding lower cpu/mem than previous numa zone. * * @param numRacks @@ -228,7 +247,8 @@ public static Map genSupervisorsWithRacks( * @param cpu * @param mem * @param miscResources - * @param numaResourceMultiplier - cpu/mem resource for each numaZone is multiplied by this factor to obtain uneven resources + * @param numaResourceMultiplier - cpu/mem resource for each numaZone is multiplied by this + * factor to obtain uneven resources * @return */ public static Map genSupervisorsWithRacksAndNuma( @@ -236,7 +256,8 @@ public static Map genSupervisorsWithRacksAndNuma( double cpu, double mem, Map miscResources, double numaResourceMultiplier) { Map retList = new HashMap<>(); for (int rack = rackStart; rack < numRacks + rackStart; rack++) { - for (int superInRack = superInRackStart; superInRack < (numSupersPerRack + superInRackStart); superInRack++) { + for (int superInRack = superInRackStart; superInRack < (numSupersPerRack + + superInRackStart); superInRack++) { List ports = new LinkedList<>(); for (int p = 0; p < numPorts; p++) { ports.add(p); @@ -254,11 +275,14 @@ public static Map genSupervisorsWithRacksAndNuma( host = String.format("host-%03d-rack-%03d", superInRack, rack); } Map resourceMap = new HashMap<>(); - resourceMap.put(Config.SUPERVISOR_CPU_CAPACITY, cpu * Math.pow(numaResourceMultiplier, numaZone)); - resourceMap.put(Config.SUPERVISOR_MEMORY_CAPACITY_MB, mem * Math.pow(numaResourceMultiplier, numaZone)); + resourceMap.put(Config.SUPERVISOR_CPU_CAPACITY, cpu * Math + .pow(numaResourceMultiplier, numaZone)); + resourceMap.put(Config.SUPERVISOR_MEMORY_CAPACITY_MB, mem * Math + .pow(numaResourceMultiplier, numaZone)); resourceMap.putAll(miscResources); SupervisorDetails sup = new SupervisorDetails(superId, host, null, ports, - NormalizedResources.RESOURCE_NAME_NORMALIZER.normalizedResourceMap(resourceMap)); + NormalizedResources.RESOURCE_NAME_NORMALIZER + .normalizedResourceMap(resourceMap)); retList.put(sup.getId(), sup); } @@ -301,28 +325,35 @@ public static Topologies addTopologies(Topologies topos, TopologyDetails... deta } for (TopologyDetails td : details) { if (topoMap.put(td.getId(), td) != null) { - throw new IllegalArgumentException("Cannot have multiple topologies with id " + td.getId()); + throw new IllegalArgumentException("Cannot have multiple topologies with id " + td + .getId()); } } return new Topologies(topoMap); } - public static TopologyDetails genTopology(String name, Map config, int numSpout, int numBolt, + public static TopologyDetails genTopology(String name, Map config, int numSpout, + int numBolt, int spoutParallelism, int boltParallelism, int launchTime, int priority, String user) { - return genTopology(name, config, numSpout, numBolt, spoutParallelism, boltParallelism, launchTime, priority, user, + return genTopology(name, config, numSpout, numBolt, spoutParallelism, boltParallelism, + launchTime, priority, user, Double.MAX_VALUE); } - public static TopologyDetails genTopology(String name, Map config, int numSpout, int numBolt, + public static TopologyDetails genTopology(String name, Map config, int numSpout, + int numBolt, int spoutParallelism, int boltParallelism, int launchTime, int priority, String user, double maxHeapSize) { - StormTopology topology = buildTopology(numSpout, numBolt, spoutParallelism, boltParallelism); - return topoToTopologyDetails(name, config, topology, launchTime, priority, user, maxHeapSize); + StormTopology topology = buildTopology(numSpout, numBolt, spoutParallelism, + boltParallelism); + return topoToTopologyDetails(name, config, topology, launchTime, priority, user, + maxHeapSize); } - public static TopologyDetails topoToTopologyDetails(String name, Map config, StormTopology topology, + public static TopologyDetails topoToTopologyDetails(String name, Map config, + StormTopology topology, int launchTime, int priority, String user, double maxHeapSize) { Config conf = new Config(); @@ -338,7 +369,8 @@ public static TopologyDetails topoToTopologyDetails(String name, Map conf, TopologyContext context, SpoutOutputCollector collector) { + public void open(Map conf, TopologyContext context, + SpoutOutputCollector collector) { _collector = collector; } @@ -453,12 +486,14 @@ public Collection allSlotsAvailableForScheduling(Collection> newSlotsByTopologyId) { + public void assignSlots(Topologies topologies, Map> newSlotsByTopologyId) { } @Override - public String getHostName(Map existingSupervisors, String nodeId) { + public String getHostName(Map existingSupervisors, + String nodeId) { if (existingSupervisors.containsKey(nodeId)) { return existingSupervisors.get(nodeId).getHost(); } @@ -478,7 +513,8 @@ private static boolean isContain(String source, String subItem) { return m.find(); } - public static void assertTopologiesNotScheduled(Cluster cluster, Class strategyClass, String... topoNames) { + public static void assertTopologiesNotScheduled(Cluster cluster, Class strategyClass, + String... topoNames) { Topologies topologies = cluster.getTopologies(); for (String topoName : topoNames) { TopologyDetails td = topologies.getByName(topoName); @@ -486,27 +522,34 @@ public static void assertTopologiesNotScheduled(Cluster cluster, Class strategyC String topoId = td.getId(); String status = cluster.getStatus(topoId); Assertions.assertNotNull(status, "Status unknown for topoName " + topoName); - Assertions.assertFalse(isStatusSuccess(status), "Successful status " + status + " for topoName " + topoName); - Assertions.assertNull(cluster.getAssignmentById(topoId), "Found assignment for topoId " + topoId); - Assertions.assertTrue(cluster.needsSchedulingRas(td), "Scheduling not required for topoName " + topoName); + Assertions.assertFalse(isStatusSuccess(status), "Successful status " + status + + " for topoName " + topoName); + Assertions.assertNull(cluster.getAssignmentById(topoId), "Found assignment for topoId " + + topoId); + Assertions.assertTrue(cluster.needsSchedulingRas(td), + "Scheduling not required for topoName " + topoName); } } - public static void assertTopologiesFullyScheduled(Cluster cluster, Class strategyClass, String... topoNames) { + public static void assertTopologiesFullyScheduled(Cluster cluster, Class strategyClass, + String... topoNames) { Topologies topologies = cluster.getTopologies(); for (String topoName : topoNames) { TopologyDetails td = topologies.getByName(topoName); Assertions.assertNotNull(td, "Cannot find topology for topoName " + topoName); String topoId = td.getId(); assertStatusSuccess(cluster, topoId); - Assertions.assertNotNull(cluster.getAssignmentById(topoId), "Cannot find assignment for topoId " + topoId); - Assertions.assertFalse(cluster.needsSchedulingRas(td), "Scheduling required for topoName " + topoName); + Assertions.assertNotNull(cluster.getAssignmentById(topoId), + "Cannot find assignment for topoId " + topoId); + Assertions.assertFalse(cluster.needsSchedulingRas(td), + "Scheduling required for topoName " + topoName); } } - public static void assertTopologiesFullyScheduled(Cluster cluster, Class strategyClass, int expectedScheduledCnt) { + public static void assertTopologiesFullyScheduled(Cluster cluster, Class strategyClass, + int expectedScheduledCnt) { List toposScheduled = new ArrayList<>(); - for (TopologyDetails td: cluster.getTopologies()) { + for (TopologyDetails td : cluster.getTopologies()) { String topoId = td.getId(); if (!isStatusSuccess(cluster.getStatus(topoId)) || cluster.getAssignmentById(topoId) == null @@ -520,7 +563,8 @@ public static void assertTopologiesFullyScheduled(Cluster cluster, Class strateg assertEquals(expectedScheduledCnt, toposScheduled.size(), errMsg); } - public static void assertTopologiesBeenEvicted(Cluster cluster, Class strategyClass, Set evictedTopologies, String... topoNames) { + public static void assertTopologiesBeenEvicted(Cluster cluster, Class strategyClass, + Set evictedTopologies, String... topoNames) { Topologies topologies = cluster.getTopologies(); LOG.info("Evicted topos: {}", evictedTopologies); Assertions.assertNotNull(evictedTopologies, "evictedTopologies is null"); @@ -529,11 +573,13 @@ public static void assertTopologiesBeenEvicted(Cluster cluster, Class strategyCl TopologyDetails td = topologies.getByName(topoName); Assertions.assertNotNull(td, "Cannot find topology for topoName " + topoName); String topoId = td.getId(); - Assertions.assertTrue(evictedTopologies.contains(topoId), "evictedTopologies does not contain topoId " + topoId); + Assertions.assertTrue(evictedTopologies.contains(topoId), + "evictedTopologies does not contain topoId " + topoId); } } - public static void assertTopologiesNotBeenEvicted(Cluster cluster, Class strategyClass, Set evictedTopologies, String... topoNames) { + public static void assertTopologiesNotBeenEvicted(Cluster cluster, Class strategyClass, + Set evictedTopologies, String... topoNames) { Topologies topologies = cluster.getTopologies(); LOG.info("Evicted topos: {}", evictedTopologies); Assertions.assertNotNull(evictedTopologies, "evictedTopologies is null"); @@ -542,16 +588,19 @@ public static void assertTopologiesNotBeenEvicted(Cluster cluster, Class strateg TopologyDetails td = topologies.getByName(topoName); Assertions.assertNotNull(td, "Cannot find topology for topoName " + topoName); String topoId = td.getId(); - Assertions.assertFalse(evictedTopologies.contains(topoId), "evictedTopologies contains topoId " + topoId); + Assertions.assertFalse(evictedTopologies.contains(topoId), + "evictedTopologies contains topoId " + topoId); } } public static void assertStatusSuccess(Cluster cluster, String topoId) { - Assertions.assertTrue(isStatusSuccess(cluster.getStatus(topoId)), "topology " + topoId + " in unsuccessful status: " + cluster.getStatus(topoId)); + Assertions.assertTrue(isStatusSuccess(cluster.getStatus(topoId)), "topology " + topoId + + " in unsuccessful status: " + cluster.getStatus(topoId)); } public static boolean isStatusSuccess(String status) { - return isContain(status, "fully") && isContain(status, "scheduled") && !isContain(status, "unsuccessful"); + return isContain(status, "fully") && isContain(status, "scheduled") && !isContain(status, + "unsuccessful"); } public static Map getSupervisorToMemoryUsage(ISchedulingState cluster, Topologies topologies) { @@ -566,29 +615,36 @@ public static Map getSupervisorToMemoryUsage(ISchedul Map executorToSupervisor = new HashMap<>(); Map> supervisorToExecutors = new HashMap<>(); TopologyDetails topology = topologies.getById(assignment.getTopologyId()); - for (Map.Entry entry : assignment.getExecutorToSlot().entrySet()) { - executorToSupervisor.put(entry.getKey(), cluster.getSupervisorById(entry.getValue().getNodeId())); + for (Map.Entry entry : assignment.getExecutorToSlot() + .entrySet()) { + executorToSupervisor.put(entry.getKey(), cluster.getSupervisorById(entry.getValue() + .getNodeId())); } - for (Map.Entry entry : executorToSupervisor.entrySet()) { - List executorsOnSupervisor = supervisorToExecutors.get(entry.getValue()); + for (Map.Entry entry : executorToSupervisor + .entrySet()) { + List executorsOnSupervisor = supervisorToExecutors.get(entry + .getValue()); if (executorsOnSupervisor == null) { executorsOnSupervisor = new ArrayList<>(); supervisorToExecutors.put(entry.getValue(), executorsOnSupervisor); } executorsOnSupervisor.add(entry.getKey()); } - for (Map.Entry> entry : supervisorToExecutors.entrySet()) { + for (Map.Entry> entry : supervisorToExecutors + .entrySet()) { Double supervisorUsedMemory = 0.0; for (ExecutorDetails executor : entry.getValue()) { supervisorUsedMemory += topology.getTotalMemReqTask(executor); } - superToMem.put(entry.getKey(), superToMem.get(entry.getKey()) + supervisorUsedMemory); + superToMem.put(entry.getKey(), superToMem.get(entry.getKey()) + + supervisorUsedMemory); } } return superToMem; } - public static Map getSupervisorToCpuUsage(ISchedulingState cluster, Topologies topologies) { + public static Map getSupervisorToCpuUsage(ISchedulingState cluster, + Topologies topologies) { Map superToCpu = new HashMap<>(); Collection assignments = cluster.getAssignments().values(); Collection supervisors = cluster.getSupervisors().values(); @@ -600,18 +656,23 @@ public static Map getSupervisorToCpuUsage(IScheduling Map executorToSupervisor = new HashMap<>(); Map> supervisorToExecutors = new HashMap<>(); TopologyDetails topology = topologies.getById(assignment.getTopologyId()); - for (Map.Entry entry : assignment.getExecutorToSlot().entrySet()) { - executorToSupervisor.put(entry.getKey(), cluster.getSupervisorById(entry.getValue().getNodeId())); + for (Map.Entry entry : assignment.getExecutorToSlot() + .entrySet()) { + executorToSupervisor.put(entry.getKey(), cluster.getSupervisorById(entry.getValue() + .getNodeId())); } - for (Map.Entry entry : executorToSupervisor.entrySet()) { - List executorsOnSupervisor = supervisorToExecutors.get(entry.getValue()); + for (Map.Entry entry : executorToSupervisor + .entrySet()) { + List executorsOnSupervisor = supervisorToExecutors.get(entry + .getValue()); if (executorsOnSupervisor == null) { executorsOnSupervisor = new ArrayList<>(); supervisorToExecutors.put(entry.getValue(), executorsOnSupervisor); } executorsOnSupervisor.add(entry.getKey()); } - for (Map.Entry> entry : supervisorToExecutors.entrySet()) { + for (Map.Entry> entry : supervisorToExecutors + .entrySet()) { Double supervisorUsedCpu = 0.0; for (ExecutorDetails executor : entry.getValue()) { supervisorUsedCpu += topology.getTotalCpuReqTask(executor); diff --git a/storm-server/src/test/java/org/apache/storm/scheduler/resource/normalization/NormalizedResourceOfferTest.java b/storm-server/src/test/java/org/apache/storm/scheduler/resource/normalization/NormalizedResourceOfferTest.java index d193e09df3f..8ee9889851b 100644 --- a/storm-server/src/test/java/org/apache/storm/scheduler/resource/normalization/NormalizedResourceOfferTest.java +++ b/storm-server/src/test/java/org/apache/storm/scheduler/resource/normalization/NormalizedResourceOfferTest.java @@ -18,20 +18,21 @@ package org.apache.storm.scheduler.resource.normalization; +import static org.junit.jupiter.api.Assertions.assertEquals; + import java.util.HashMap; import java.util.Map; import org.apache.storm.Constants; import org.apache.storm.metric.StormMetricsRegistry; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; - public class NormalizedResourceOfferTest { @Test public void testNodeOverExtendedCpu() { NormalizedResourceOffer availableResources = createOffer(100.0, 0.0); NormalizedResourceOffer scheduledResources = createOffer(110.0, 0.0); - availableResources.remove(scheduledResources, new ResourceMetrics(new StormMetricsRegistry())); + availableResources.remove(scheduledResources, + new ResourceMetrics(new StormMetricsRegistry())); assertEquals(0.0, availableResources.getTotalCpu(), 0.001); } @@ -39,7 +40,8 @@ public void testNodeOverExtendedCpu() { public void testNodeOverExtendedMemory() { NormalizedResourceOffer availableResources = createOffer(0.0, 5.0); NormalizedResourceOffer scheduledResources = createOffer(0.0, 10.0); - availableResources.remove(scheduledResources, new ResourceMetrics(new StormMetricsRegistry())); + availableResources.remove(scheduledResources, + new ResourceMetrics(new StormMetricsRegistry())); assertEquals(0.0, availableResources.getTotalMemoryMb(), 0.001); } diff --git a/storm-server/src/test/java/org/apache/storm/scheduler/resource/normalization/NormalizedResourceRequestTest.java b/storm-server/src/test/java/org/apache/storm/scheduler/resource/normalization/NormalizedResourceRequestTest.java index 9d590f166c0..129a1808f9b 100644 --- a/storm-server/src/test/java/org/apache/storm/scheduler/resource/normalization/NormalizedResourceRequestTest.java +++ b/storm-server/src/test/java/org/apache/storm/scheduler/resource/normalization/NormalizedResourceRequestTest.java @@ -18,6 +18,9 @@ package org.apache.storm.scheduler.resource.normalization; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + import java.util.HashMap; import java.util.Map; import org.apache.storm.Config; @@ -25,9 +28,6 @@ import org.apache.storm.daemon.Acker; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; - public class NormalizedResourceRequestTest { @Test @@ -35,7 +35,8 @@ public void testAckerCPUSetting() { Map topoConf = new HashMap<>(); topoConf.put(Config.TOPOLOGY_ACKER_CPU_PCORE_PERCENT, 40); topoConf.put(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT, 50); - NormalizedResourceRequest request = new NormalizedResourceRequest(topoConf, Acker.ACKER_COMPONENT_ID); + NormalizedResourceRequest request = new NormalizedResourceRequest(topoConf, + Acker.ACKER_COMPONENT_ID); Map normalizedMap = request.toNormalizedMap(); Double cpu = normalizedMap.get(Constants.COMMON_CPU_RESOURCE_NAME); assertNotNull(cpu); @@ -47,7 +48,8 @@ public void testNonAckerCPUSetting() { Map topoConf = new HashMap<>(); topoConf.put(Config.TOPOLOGY_ACKER_CPU_PCORE_PERCENT, 40); topoConf.put(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT, 50); - NormalizedResourceRequest request = new NormalizedResourceRequest(topoConf, "notAnAckerComponent"); + NormalizedResourceRequest request = new NormalizedResourceRequest(topoConf, + "notAnAckerComponent"); Map normalizedMap = request.toNormalizedMap(); Double cpu = normalizedMap.get(Constants.COMMON_CPU_RESOURCE_NAME); assertNotNull(cpu); diff --git a/storm-server/src/test/java/org/apache/storm/scheduler/resource/normalization/NormalizedResourcesExtension.java b/storm-server/src/test/java/org/apache/storm/scheduler/resource/normalization/NormalizedResourcesExtension.java index b3c8091a7ce..7b428b61b38 100644 --- a/storm-server/src/test/java/org/apache/storm/scheduler/resource/normalization/NormalizedResourcesExtension.java +++ b/storm-server/src/test/java/org/apache/storm/scheduler/resource/normalization/NormalizedResourcesExtension.java @@ -6,10 +6,10 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - *

      - * http://www.apache.org/licenses/LICENSE-2.0 - *

      - * Unless required by applicable law or agreed to in writing, software + * + *

      http://www.apache.org/licenses/LICENSE-2.0 + * + *

      Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and diff --git a/storm-server/src/test/java/org/apache/storm/scheduler/resource/normalization/NormalizedResourcesTest.java b/storm-server/src/test/java/org/apache/storm/scheduler/resource/normalization/NormalizedResourcesTest.java index 892d71f0d7e..853c5a15969 100644 --- a/storm-server/src/test/java/org/apache/storm/scheduler/resource/normalization/NormalizedResourcesTest.java +++ b/storm-server/src/test/java/org/apache/storm/scheduler/resource/normalization/NormalizedResourcesTest.java @@ -16,7 +16,6 @@ package org.apache.storm.scheduler.resource.normalization; - import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -44,8 +43,10 @@ public void reset() { @Test public void testAddCpu() { - NormalizedResources resources = new NormalizedResources(normalize(Collections.singletonMap(Constants.COMMON_CPU_RESOURCE_NAME, 1))); - NormalizedResources addedResources = new NormalizedResources(normalize(Collections.singletonMap(Constants.COMMON_CPU_RESOURCE_NAME, 1))); + NormalizedResources resources = new NormalizedResources(normalize(Collections + .singletonMap(Constants.COMMON_CPU_RESOURCE_NAME, 1))); + NormalizedResources addedResources = new NormalizedResources(normalize(Collections + .singletonMap(Constants.COMMON_CPU_RESOURCE_NAME, 1))); resources.add(addedResources); @@ -56,8 +57,10 @@ public void testAddCpu() { @Test public void testAddToExistingResource() { - NormalizedResources resources = new NormalizedResources(normalize(Collections.singletonMap(gpuResourceName, 1))); - NormalizedResources addedResources = new NormalizedResources(normalize(Collections.singletonMap(gpuResourceName, 1))); + NormalizedResources resources = new NormalizedResources(normalize(Collections + .singletonMap(gpuResourceName, 1))); + NormalizedResources addedResources = new NormalizedResources(normalize(Collections + .singletonMap(gpuResourceName, 1))); resources.add(addedResources); @@ -68,7 +71,8 @@ public void testAddToExistingResource() { @Test public void testAddWhenOtherHasMoreResourcesThanThis() { NormalizedResources resources = new NormalizedResources(normalize(Collections.emptyMap())); - NormalizedResources addedResources = new NormalizedResources(normalize(Collections.singletonMap(gpuResourceName, 1))); + NormalizedResources addedResources = new NormalizedResources(normalize(Collections + .singletonMap(gpuResourceName, 1))); resources.add(addedResources); @@ -79,8 +83,10 @@ public void testAddWhenOtherHasMoreResourcesThanThis() { @Test public void testAddWhenOtherHasDifferentResourceThanThis() { String disks = "disks"; - NormalizedResources resources = new NormalizedResources(normalize(Collections.singletonMap(disks, 23))); - NormalizedResources addedResources = new NormalizedResources(normalize(Collections.singletonMap(gpuResourceName, 1))); + NormalizedResources resources = new NormalizedResources(normalize(Collections + .singletonMap(disks, 23))); + NormalizedResources addedResources = new NormalizedResources(normalize(Collections + .singletonMap(gpuResourceName, 1))); resources.add(addedResources); @@ -91,8 +97,10 @@ public void testAddWhenOtherHasDifferentResourceThanThis() { @Test public void testRemoveZeroesWhenResourcesBecomeNegative() { - NormalizedResources resources = new NormalizedResources(normalize(Collections.singletonMap(gpuResourceName, 1))); - NormalizedResources removedResources = new NormalizedResources(normalize(Collections.singletonMap(gpuResourceName, 2))); + NormalizedResources resources = new NormalizedResources(normalize(Collections + .singletonMap(gpuResourceName, 1))); + NormalizedResources removedResources = new NormalizedResources(normalize(Collections + .singletonMap(gpuResourceName, 2))); resources.remove(removedResources, new ResourceMetrics(new StormMetricsRegistry())); Map normalizedMap = resources.toNormalizedMap(); @@ -101,8 +109,10 @@ public void testRemoveZeroesWhenResourcesBecomeNegative() { @Test public void testRemoveZeroesWhenCpuBecomesNegative() { - NormalizedResources resources = new NormalizedResources(normalize(Collections.singletonMap(Constants.COMMON_CPU_RESOURCE_NAME, 1))); - NormalizedResources removedResources = new NormalizedResources(normalize(Collections.singletonMap(Constants.COMMON_CPU_RESOURCE_NAME, 2))); + NormalizedResources resources = new NormalizedResources(normalize(Collections + .singletonMap(Constants.COMMON_CPU_RESOURCE_NAME, 1))); + NormalizedResources removedResources = new NormalizedResources(normalize(Collections + .singletonMap(Constants.COMMON_CPU_RESOURCE_NAME, 2))); resources.remove(removedResources, new ResourceMetrics(new StormMetricsRegistry())); assertThat(resources.getTotalCpu(), is(0.0)); @@ -110,8 +120,10 @@ public void testRemoveZeroesWhenCpuBecomesNegative() { @Test public void testRemoveFromCpu() { - NormalizedResources resources = new NormalizedResources(normalize(Collections.singletonMap(Constants.COMMON_CPU_RESOURCE_NAME, 2))); - NormalizedResources removedResources = new NormalizedResources(normalize(Collections.singletonMap(Constants.COMMON_CPU_RESOURCE_NAME, 1))); + NormalizedResources resources = new NormalizedResources(normalize(Collections + .singletonMap(Constants.COMMON_CPU_RESOURCE_NAME, 2))); + NormalizedResources removedResources = new NormalizedResources(normalize(Collections + .singletonMap(Constants.COMMON_CPU_RESOURCE_NAME, 1))); resources.remove(removedResources, new ResourceMetrics(new StormMetricsRegistry())); @@ -122,8 +134,10 @@ public void testRemoveFromCpu() { @Test public void testRemoveFromExistingResources() { - NormalizedResources resources = new NormalizedResources(normalize(Collections.singletonMap(gpuResourceName, 15))); - NormalizedResources removedResources = new NormalizedResources(normalize(Collections.singletonMap(gpuResourceName, 1))); + NormalizedResources resources = new NormalizedResources(normalize(Collections + .singletonMap(gpuResourceName, 15))); + NormalizedResources removedResources = new NormalizedResources(normalize(Collections + .singletonMap(gpuResourceName, 1))); resources.remove(removedResources, new ResourceMetrics(new StormMetricsRegistry())); @@ -133,8 +147,10 @@ public void testRemoveFromExistingResources() { @Test public void testCouldHoldWithTooFewCpus() { - NormalizedResources resources = new NormalizedResources(normalize(Collections.singletonMap(Constants.COMMON_CPU_RESOURCE_NAME, 1))); - NormalizedResources resourcesToCheck = new NormalizedResources(normalize(Collections.singletonMap(Constants.COMMON_CPU_RESOURCE_NAME, 2))); + NormalizedResources resources = new NormalizedResources(normalize(Collections + .singletonMap(Constants.COMMON_CPU_RESOURCE_NAME, 1))); + NormalizedResources resourcesToCheck = new NormalizedResources(normalize(Collections + .singletonMap(Constants.COMMON_CPU_RESOURCE_NAME, 2))); boolean couldHold = resources.couldHoldIgnoringSharedMemory(resourcesToCheck, 100, 1); @@ -143,8 +159,10 @@ public void testCouldHoldWithTooFewCpus() { @Test public void testCouldHoldWithTooFewResource() { - NormalizedResources resources = new NormalizedResources(normalize(Collections.singletonMap(gpuResourceName, 1))); - NormalizedResources resourcesToCheck = new NormalizedResources(normalize(Collections.singletonMap(gpuResourceName, 2))); + NormalizedResources resources = new NormalizedResources(normalize(Collections + .singletonMap(gpuResourceName, 1))); + NormalizedResources resourcesToCheck = new NormalizedResources(normalize(Collections + .singletonMap(gpuResourceName, 2))); boolean couldHold = resources.couldHoldIgnoringSharedMemory(resourcesToCheck, 100, 1); @@ -153,8 +171,10 @@ public void testCouldHoldWithTooFewResource() { @Test public void testCouldHoldWithTooLittleMemory() { - NormalizedResources resources = new NormalizedResources(normalize(Collections.singletonMap(gpuResourceName, 1))); - NormalizedResources resourcesToCheck = new NormalizedResources(normalize(Collections.singletonMap(gpuResourceName, 1))); + NormalizedResources resources = new NormalizedResources(normalize(Collections + .singletonMap(gpuResourceName, 1))); + NormalizedResources resourcesToCheck = new NormalizedResources(normalize(Collections + .singletonMap(gpuResourceName, 1))); boolean couldHold = resources.couldHoldIgnoringSharedMemory(resourcesToCheck, 100, 200); @@ -164,7 +184,8 @@ public void testCouldHoldWithTooLittleMemory() { @Test public void testCouldHoldWithMissingResource() { NormalizedResources resources = new NormalizedResources(normalize(Collections.emptyMap())); - NormalizedResources resourcesToCheck = new NormalizedResources(normalize(Collections.singletonMap(gpuResourceName, 1))); + NormalizedResources resourcesToCheck = new NormalizedResources(normalize(Collections + .singletonMap(gpuResourceName, 1))); boolean couldHold = resources.couldHoldIgnoringSharedMemory(resourcesToCheck, 100, 1); @@ -187,7 +208,8 @@ public void testCouldHoldWithEnoughResources() { @Test public void testCalculateAvgUsageWithNoResourcesInTotal() { NormalizedResources resources = new NormalizedResources(normalize(Collections.emptyMap())); - NormalizedResources usedResources = new NormalizedResources(normalize(Collections.emptyMap())); + NormalizedResources usedResources = new NormalizedResources(normalize(Collections + .emptyMap())); double avg = resources.calculateAveragePercentageUsedBy(usedResources, 0, 0); @@ -196,8 +218,10 @@ public void testCalculateAvgUsageWithNoResourcesInTotal() { @Test public void testCalculateAvgWithOnlyCpu() { - NormalizedResources resources = new NormalizedResources(normalize(Collections.singletonMap(Constants.COMMON_CPU_RESOURCE_NAME, 2))); - NormalizedResources usedResources = new NormalizedResources(normalize(Collections.singletonMap(Constants.COMMON_CPU_RESOURCE_NAME, 1))); + NormalizedResources resources = new NormalizedResources(normalize(Collections + .singletonMap(Constants.COMMON_CPU_RESOURCE_NAME, 2))); + NormalizedResources usedResources = new NormalizedResources(normalize(Collections + .singletonMap(Constants.COMMON_CPU_RESOURCE_NAME, 1))); double avg = resources.calculateAveragePercentageUsedBy(usedResources, 0, 0); @@ -206,12 +230,14 @@ public void testCalculateAvgWithOnlyCpu() { @Test public void testCalculateAvgWithCpuAndMem() { - NormalizedResources resources = new NormalizedResources(normalize(Collections.singletonMap(Constants.COMMON_CPU_RESOURCE_NAME, 2))); - NormalizedResources usedResources = new NormalizedResources(normalize(Collections.singletonMap(Constants.COMMON_CPU_RESOURCE_NAME, 1))); + NormalizedResources resources = new NormalizedResources(normalize(Collections + .singletonMap(Constants.COMMON_CPU_RESOURCE_NAME, 2))); + NormalizedResources usedResources = new NormalizedResources(normalize(Collections + .singletonMap(Constants.COMMON_CPU_RESOURCE_NAME, 1))); double avg = resources.calculateAveragePercentageUsedBy(usedResources, 4, 1); - assertThat(avg, is((50.0 + 25.0)/2)); + assertThat(avg, is((50.0 + 25.0) / 2)); } @Test @@ -227,7 +253,7 @@ public void testCalculateAvgWithCpuMemAndGenericResource() { double avg = resources.calculateAveragePercentageUsedBy(usedResources, 4, 1); - assertThat(avg, is((50.0 + 25.0 + 10.0)/3)); + assertThat(avg, is((50.0 + 25.0 + 10.0) / 3)); } @Test @@ -242,14 +268,16 @@ public void testCalculateAvgWithUnusedResource() { double avg = resources.calculateAveragePercentageUsedBy(usedResources, 4, 1); - //The resource that is not used should count as if it is being used 0% - assertThat(avg, is((50.0 + 25.0)/3)); + // The resource that is not used should count as if it is being used 0% + assertThat(avg, is((50.0 + 25.0) / 3)); } @Test public void testCalculateAvgThrowsIfTotalIsMissingCpu() { - NormalizedResources resources = new NormalizedResources(normalize(Collections.singletonMap(Constants.COMMON_CPU_RESOURCE_NAME, 2))); - NormalizedResources usedResources = new NormalizedResources(normalize(Collections.singletonMap(Constants.COMMON_CPU_RESOURCE_NAME, 5))); + NormalizedResources resources = new NormalizedResources(normalize(Collections + .singletonMap(Constants.COMMON_CPU_RESOURCE_NAME, 2))); + NormalizedResources usedResources = new NormalizedResources(normalize(Collections + .singletonMap(Constants.COMMON_CPU_RESOURCE_NAME, 5))); assertThrows(IllegalArgumentException.class, () -> resources.calculateAveragePercentageUsedBy(usedResources, 0, 0)); @@ -257,8 +285,10 @@ public void testCalculateAvgThrowsIfTotalIsMissingCpu() { @Test public void testCalculateAvgThrowsIfTotalIsMissingMemory() { - NormalizedResources resources = new NormalizedResources(normalize(Collections.singletonMap(Constants.COMMON_CPU_RESOURCE_NAME, 2))); - NormalizedResources usedResources = new NormalizedResources(normalize(Collections.singletonMap(Constants.COMMON_CPU_RESOURCE_NAME, 1))); + NormalizedResources resources = new NormalizedResources(normalize(Collections + .singletonMap(Constants.COMMON_CPU_RESOURCE_NAME, 2))); + NormalizedResources usedResources = new NormalizedResources(normalize(Collections + .singletonMap(Constants.COMMON_CPU_RESOURCE_NAME, 1))); assertThrows(IllegalArgumentException.class, () -> resources.calculateAveragePercentageUsedBy(usedResources, 100, 500)); @@ -296,7 +326,8 @@ public void testCalculateAvgWithTooLittleResourceInTotal() { @Test public void testCalculateMinUsageWithNoResourcesInTotal() { NormalizedResources resources = new NormalizedResources(normalize(Collections.emptyMap())); - NormalizedResources usedResources = new NormalizedResources(normalize(Collections.emptyMap())); + NormalizedResources usedResources = new NormalizedResources(normalize(Collections + .emptyMap())); double min = resources.calculateMinPercentageUsedBy(usedResources, 0, 0); @@ -305,8 +336,10 @@ public void testCalculateMinUsageWithNoResourcesInTotal() { @Test public void testCalculateMinWithOnlyCpu() { - NormalizedResources resources = new NormalizedResources(normalize(Collections.singletonMap(Constants.COMMON_CPU_RESOURCE_NAME, 2))); - NormalizedResources usedResources = new NormalizedResources(normalize(Collections.singletonMap(Constants.COMMON_CPU_RESOURCE_NAME, 1))); + NormalizedResources resources = new NormalizedResources(normalize(Collections + .singletonMap(Constants.COMMON_CPU_RESOURCE_NAME, 2))); + NormalizedResources usedResources = new NormalizedResources(normalize(Collections + .singletonMap(Constants.COMMON_CPU_RESOURCE_NAME, 1))); double min = resources.calculateMinPercentageUsedBy(usedResources, 0, 0); @@ -315,8 +348,10 @@ public void testCalculateMinWithOnlyCpu() { @Test public void testCalculateMinWithCpuAndMem() { - NormalizedResources resources = new NormalizedResources(normalize(Collections.singletonMap(Constants.COMMON_CPU_RESOURCE_NAME, 2))); - NormalizedResources usedResources = new NormalizedResources(normalize(Collections.singletonMap(Constants.COMMON_CPU_RESOURCE_NAME, 1))); + NormalizedResources resources = new NormalizedResources(normalize(Collections + .singletonMap(Constants.COMMON_CPU_RESOURCE_NAME, 2))); + NormalizedResources usedResources = new NormalizedResources(normalize(Collections + .singletonMap(Constants.COMMON_CPU_RESOURCE_NAME, 1))); double min = resources.calculateMinPercentageUsedBy(usedResources, 4, 1); @@ -351,14 +386,16 @@ public void testCalculateMinWithUnusedResource() { double min = resources.calculateMinPercentageUsedBy(usedResources, 4, 1); - //The resource that is not used should count as if it is being used 0% + // The resource that is not used should count as if it is being used 0% assertThat(min, is(0.0)); } @Test public void testCalculateMinThrowsIfTotalIsMissingCpu() { - NormalizedResources resources = new NormalizedResources(normalize(Collections.singletonMap(Constants.COMMON_CPU_RESOURCE_NAME, 2))); - NormalizedResources usedResources = new NormalizedResources(normalize(Collections.singletonMap(Constants.COMMON_CPU_RESOURCE_NAME, 5))); + NormalizedResources resources = new NormalizedResources(normalize(Collections + .singletonMap(Constants.COMMON_CPU_RESOURCE_NAME, 2))); + NormalizedResources usedResources = new NormalizedResources(normalize(Collections + .singletonMap(Constants.COMMON_CPU_RESOURCE_NAME, 5))); assertThrows(IllegalArgumentException.class, () -> resources.calculateMinPercentageUsedBy(usedResources, 0, 0)); @@ -366,8 +403,10 @@ public void testCalculateMinThrowsIfTotalIsMissingCpu() { @Test public void testCalculateMinThrowsIfTotalIsMissingMemory() { - NormalizedResources resources = new NormalizedResources(normalize(Collections.singletonMap(Constants.COMMON_CPU_RESOURCE_NAME, 2))); - NormalizedResources usedResources = new NormalizedResources(normalize(Collections.singletonMap(Constants.COMMON_CPU_RESOURCE_NAME, 1))); + NormalizedResources resources = new NormalizedResources(normalize(Collections + .singletonMap(Constants.COMMON_CPU_RESOURCE_NAME, 2))); + NormalizedResources usedResources = new NormalizedResources(normalize(Collections + .singletonMap(Constants.COMMON_CPU_RESOURCE_NAME, 1))); assertThrows(IllegalArgumentException.class, () -> resources.calculateMinPercentageUsedBy(usedResources, 100, 500)); @@ -398,6 +437,7 @@ public void testCalculateMinWithTooLittleResourceInTotal() { usedResourcesMap.put(gpuResourceName, 5.0); NormalizedResources usedResources = new NormalizedResources(normalize(usedResourcesMap)); - assertThrows(IllegalArgumentException.class, () -> resources.calculateMinPercentageUsedBy(usedResources, 4, 1)); + assertThrows(IllegalArgumentException.class, () -> resources + .calculateMinPercentageUsedBy(usedResources, 4, 1)); } } diff --git a/storm-server/src/test/java/org/apache/storm/scheduler/resource/normalization/ResourceMapArrayBridgeTest.java b/storm-server/src/test/java/org/apache/storm/scheduler/resource/normalization/ResourceMapArrayBridgeTest.java index 7bb3194c802..99379b1591b 100644 --- a/storm-server/src/test/java/org/apache/storm/scheduler/resource/normalization/ResourceMapArrayBridgeTest.java +++ b/storm-server/src/test/java/org/apache/storm/scheduler/resource/normalization/ResourceMapArrayBridgeTest.java @@ -16,7 +16,6 @@ package org.apache.storm.scheduler.resource.normalization; - import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; diff --git a/storm-server/src/test/java/org/apache/storm/scheduler/resource/strategies/eviction/TestDefaultEvictionStrategy.java b/storm-server/src/test/java/org/apache/storm/scheduler/resource/strategies/eviction/TestDefaultEvictionStrategy.java index 1f0302d6475..9544455f723 100644 --- a/storm-server/src/test/java/org/apache/storm/scheduler/resource/strategies/eviction/TestDefaultEvictionStrategy.java +++ b/storm-server/src/test/java/org/apache/storm/scheduler/resource/strategies/eviction/TestDefaultEvictionStrategy.java @@ -6,10 +6,10 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - *

      - * http://www.apache.org/licenses/LICENSE-2.0 - *

      - * Unless required by applicable law or agreed to in writing, software + * + *

      http://www.apache.org/licenses/LICENSE-2.0 + * + *

      Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -18,7 +18,20 @@ package org.apache.storm.scheduler.resource.strategies.eviction; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.INimbusTest; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.addTopologies; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.assertTopologiesFullyScheduled; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.assertTopologiesNotScheduled; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.createClusterConfig; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.genSupervisors; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.genTopology; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.userRes; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.userResourcePool; + +import java.util.HashMap; +import java.util.Map; import org.apache.storm.Config; +import org.apache.storm.metric.StormMetricsRegistry; import org.apache.storm.scheduler.Cluster; import org.apache.storm.scheduler.INimbus; import org.apache.storm.scheduler.IScheduler; @@ -26,6 +39,7 @@ import org.apache.storm.scheduler.Topologies; import org.apache.storm.scheduler.resource.ResourceAwareScheduler; import org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler; +import org.apache.storm.scheduler.resource.normalization.ResourceMetrics; import org.apache.storm.scheduler.resource.strategies.scheduling.DefaultResourceAwareStrategy; import org.apache.storm.scheduler.resource.strategies.scheduling.DefaultResourceAwareStrategyOld; import org.apache.storm.scheduler.resource.strategies.scheduling.GenericResourceAwareStrategy; @@ -36,29 +50,23 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.HashMap; -import java.util.Map; - -import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.*; - -import org.apache.storm.metric.StormMetricsRegistry; -import org.apache.storm.scheduler.resource.normalization.ResourceMetrics; - public class TestDefaultEvictionStrategy { private static final Logger LOG = LoggerFactory.getLogger(TestDefaultEvictionStrategy.class); private static final Class[] strategyClasses = { - DefaultResourceAwareStrategy.class, - DefaultResourceAwareStrategyOld.class, - RoundRobinResourceAwareStrategy.class, - GenericResourceAwareStrategy.class, - GenericResourceAwareStrategyOld.class, + DefaultResourceAwareStrategy.class, + DefaultResourceAwareStrategyOld.class, + RoundRobinResourceAwareStrategy.class, + GenericResourceAwareStrategy.class, + GenericResourceAwareStrategyOld.class, }; private int currentTime = 1450418597; private IScheduler scheduler = null; - private Config createClusterConfig(Class strategyClass, double compPcore, double compOnHeap, double compOffHeap, + private Config createClusterConfig(Class strategyClass, double compPcore, double compOnHeap, + double compOffHeap, Map> pools) { - Config config = TestUtilsForResourceAwareScheduler.createClusterConfig(compPcore, compOnHeap, compOffHeap, pools); + Config config = TestUtilsForResourceAwareScheduler.createClusterConfig(compPcore, + compOnHeap, compOffHeap, pools); config.put(Config.TOPOLOGY_SCHEDULER_STRATEGY, strategyClass.getName()); return config; } @@ -72,13 +80,15 @@ public void cleanup() { } /** - * The resources in the cluster are limited. In the first round of scheduling, all resources in the cluster is used. - * User jerry submits another topology. Since user jerry has his resource guarantees satisfied, and user bobby + * The resources in the cluster are limited. In the first round of scheduling, all resources in + * the cluster is used. + * User jerry submits another topology. Since user jerry has his resource guarantees satisfied, + * and user bobby * has exceeded his resource guarantee, topo-3 from user bobby should be evicted. */ @Test public void testEviction() { - for (Class strategyClass: strategyClasses) { + for (Class strategyClass : strategyClasses) { INimbus iNimbus = new INimbusTest(); Map supMap = genSupervisors(4, 4, 100, 1000); Map> resourceUserPool = userResourcePool( @@ -92,28 +102,31 @@ public void testEviction() { genTopology("topo-3", config, 1, 0, 1, 0, currentTime - 2, 20, "bobby"), genTopology("topo-4", config, 1, 0, 1, 0, currentTime - 2, 29, "derek")); - Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config); + Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), + supMap, new HashMap<>(), topologies, config); scheduler = new ResourceAwareScheduler(); scheduler.prepare(config, new StormMetricsRegistry()); scheduler.schedule(topologies, cluster); - assertTopologiesFullyScheduled(cluster, strategyClass, "topo-1", "topo-2", "topo-3", "topo-4"); + assertTopologiesFullyScheduled(cluster, strategyClass, "topo-1", "topo-2", "topo-3", + "topo-4"); - //user jerry submits another topology + // user jerry submits another topology topologies = addTopologies(topologies, genTopology("topo-6", config, 1, 0, 1, 0, currentTime - 2, 20, "jerry")); cluster = new Cluster(cluster, topologies); scheduler.schedule(topologies, cluster); - //topo-3 evicted (lowest priority) - assertTopologiesFullyScheduled(cluster, strategyClass, "topo-1", "topo-2", "topo-4", "topo-6"); + // topo-3 evicted (lowest priority) + assertTopologiesFullyScheduled(cluster, strategyClass, "topo-1", "topo-2", "topo-4", + "topo-6"); assertTopologiesNotScheduled(cluster, strategyClass, "topo-3"); } } @Test public void testEvictMultipleTopologies() { - for (Class strategyClass: strategyClasses) { + for (Class strategyClass : strategyClasses) { INimbus iNimbus = new INimbusTest(); Map supMap = genSupervisors(4, 4, 100, 1000); Map> resourceUserPool = userResourcePool( @@ -126,30 +139,32 @@ public void testEvictMultipleTopologies() { genTopology("topo-3", config, 1, 0, 1, 0, currentTime - 2, 20, "bobby"), genTopology("topo-4", config, 1, 0, 1, 0, currentTime - 2, 29, "derek"), genTopology("topo-5", config, 1, 0, 1, 0, currentTime - 2, 29, "derek")); - Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config); + Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), + supMap, new HashMap<>(), topologies, config); scheduler = new ResourceAwareScheduler(); scheduler.prepare(config, new StormMetricsRegistry()); LOG.info("\n\n\t\tScheduling topos 2 to 5..."); scheduler.schedule(topologies, cluster); LOG.info("\n\n\t\tDone scheduling..."); - assertTopologiesFullyScheduled(cluster, strategyClass,"topo-2", "topo-3", "topo-4", "topo-5"); + assertTopologiesFullyScheduled(cluster, strategyClass, "topo-2", "topo-3", "topo-4", + "topo-5"); - //user jerry submits another topology + // user jerry submits another topology topologies = addTopologies(topologies, genTopology("topo-1", config, 2, 0, 1, 0, currentTime - 2, 10, "jerry")); cluster = new Cluster(cluster, topologies); LOG.info("\n\n\t\tScheduling topos 1 to 5"); scheduler.schedule(topologies, cluster); LOG.info("\n\n\t\tDone scheduling..."); - //bobby has no guarantee so topo-2 and topo-3 evicted + // bobby has no guarantee so topo-2 and topo-3 evicted assertTopologiesFullyScheduled(cluster, strategyClass, "topo-1", "topo-4", "topo-5"); - assertTopologiesNotScheduled(cluster, strategyClass,"topo-2", "topo-3"); + assertTopologiesNotScheduled(cluster, strategyClass, "topo-2", "topo-3"); } } @Test public void testEvictMultipleTopologiesFromMultipleUsersInCorrectOrder() { - for (Class strategyClass: strategyClasses) { + for (Class strategyClass : strategyClasses) { INimbus iNimbus = new INimbusTest(); Map supMap = genSupervisors(4, 4, 100, 1000); Map> resourceUserPool = userResourcePool( @@ -162,29 +177,34 @@ public void testEvictMultipleTopologiesFromMultipleUsersInCorrectOrder() { genTopology("topo-3", config, 1, 0, 1, 0, currentTime - 2, 20, "bobby"), genTopology("topo-4", config, 1, 0, 1, 0, currentTime - 2, 29, "derek"), genTopology("topo-5", config, 1, 0, 1, 0, currentTime - 15, 29, "derek")); - Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config); + Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), + supMap, new HashMap<>(), topologies, config); scheduler = new ResourceAwareScheduler(); scheduler.prepare(config, new StormMetricsRegistry()); scheduler.schedule(topologies, cluster); - assertTopologiesFullyScheduled(cluster, strategyClass, "topo-2", "topo-3", "topo-4", "topo-5"); + assertTopologiesFullyScheduled(cluster, strategyClass, "topo-2", "topo-3", "topo-4", + "topo-5"); - //user jerry submits another topology + // user jerry submits another topology topologies = addTopologies(topologies, genTopology("topo-1", config, 1, 0, 1, 0, currentTime - 2, 10, "jerry")); cluster = new Cluster(cluster, topologies); scheduler.schedule(topologies, cluster); - //topo-3 evicted since user bobby don't have any resource guarantees and topo-3 is the lowest priority for user bobby - assertTopologiesFullyScheduled(cluster, strategyClass, "topo-1", "topo-2", "topo-4", "topo-5"); - assertTopologiesNotScheduled(cluster, strategyClass,"topo-3"); + // topo-3 evicted since user bobby don't have any resource guarantees and topo-3 is the + // lowest priority for user bobby + assertTopologiesFullyScheduled(cluster, strategyClass, "topo-1", "topo-2", "topo-4", + "topo-5"); + assertTopologiesNotScheduled(cluster, strategyClass, "topo-3"); topologies = addTopologies(topologies, genTopology("topo-6", config, 1, 0, 1, 0, currentTime - 2, 10, "jerry")); cluster = new Cluster(cluster, topologies); scheduler.schedule(topologies, cluster); - //topo-2 evicted since user bobby don't have any resource guarantees and topo-2 is the next lowest priority for user bobby + // topo-2 evicted since user bobby don't have any resource guarantees and topo-2 is the + // next lowest priority for user bobby assertTopologiesFullyScheduled(cluster, strategyClass, "topo-1", "topo-4", "topo-5"); assertTopologiesNotScheduled(cluster, strategyClass, "topo-2", "topo-3"); @@ -193,20 +213,22 @@ public void testEvictMultipleTopologiesFromMultipleUsersInCorrectOrder() { cluster = new Cluster(cluster, topologies); scheduler.schedule(topologies, cluster); - // since user derek has exceeded his resource guarantee while user jerry has not topo-5 or topo-4 could be evicted because they have the same priority - // but topo-4 was submitted earlier thus we choose that one to evict (somewhat arbitrary) - assertTopologiesFullyScheduled(cluster, strategyClass,"topo-1", "topo-5", "topo-7"); - assertTopologiesNotScheduled(cluster, strategyClass,"topo-2", "topo-3", "topo-4"); + // since user derek has exceeded his resource guarantee while user jerry has not topo-5 + // or topo-4 could be evicted because they have the same priority + // but topo-4 was submitted earlier thus we choose that one to evict (somewhat + // arbitrary) + assertTopologiesFullyScheduled(cluster, strategyClass, "topo-1", "topo-5", "topo-7"); + assertTopologiesNotScheduled(cluster, strategyClass, "topo-2", "topo-3", "topo-4"); } } /** * If topologies from other users cannot be evicted to make space - * check if there is a topology with lower priority that can be evicted from the current user + * check if there is a topology with lower priority that can be evicted from the current user. */ @Test public void testEvictTopologyFromItself() { - for (Class strategyClass: strategyClasses) { + for (Class strategyClass : strategyClasses) { INimbus iNimbus = new INimbusTest(); Map supMap = genSupervisors(4, 4, 100, 1000); Map> resourceUserPool = userResourcePool( @@ -220,16 +242,18 @@ public void testEvictTopologyFromItself() { genTopology("topo-2", config, 1, 0, 1, 0, currentTime - 2, 20, "jerry"), genTopology("topo-5", config, 1, 0, 1, 0, currentTime - 2, 10, "bobby"), genTopology("topo-6", config, 1, 0, 1, 0, currentTime - 2, 29, "derek")); - Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config); + Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), + supMap, new HashMap<>(), topologies, config); scheduler = new ResourceAwareScheduler(); scheduler.prepare(config, new StormMetricsRegistry()); LOG.info("\n\n\t\tScheduling topos 1,2,5,6"); scheduler.schedule(topologies, cluster); LOG.info("\n\n\t\tDone Scheduling..."); - assertTopologiesFullyScheduled(cluster, strategyClass, "topo-1", "topo-2", "topo-5", "topo-6"); + assertTopologiesFullyScheduled(cluster, strategyClass, "topo-1", "topo-2", "topo-5", + "topo-6"); - //user jerry submits another topology into a full cluster + // user jerry submits another topology into a full cluster // topo3 should not be able to scheduled topologies = addTopologies(topologies, genTopology("topo-3", config, 1, 0, 1, 0, currentTime - 2, 29, "jerry")); @@ -238,11 +262,13 @@ public void testEvictTopologyFromItself() { scheduler.schedule(topologies, cluster); LOG.info("\n\n\t\tDone Scheduling..."); - assertTopologiesFullyScheduled(cluster, strategyClass, "topo-1", "topo-2", "topo-5", "topo-6"); + assertTopologiesFullyScheduled(cluster, strategyClass, "topo-1", "topo-2", "topo-5", + "topo-6"); assertTopologiesNotScheduled(cluster, strategyClass, "topo-3"); - //user jerry submits another topology but this one should be scheduled since it has higher priority than the - //rest of jerry's running topologies + // user jerry submits another topology but this one should be scheduled since it has + // higher priority than the + // rest of jerry's running topologies topologies = addTopologies(topologies, genTopology("topo-4", config, 1, 0, 1, 0, currentTime - 2, 10, "jerry")); cluster = new Cluster(cluster, topologies); @@ -250,17 +276,18 @@ public void testEvictTopologyFromItself() { scheduler.schedule(topologies, cluster); LOG.info("\n\n\t\tDone Scheduling..."); - assertTopologiesFullyScheduled(cluster, strategyClass, "topo-1", "topo-4", "topo-5", "topo-6"); + assertTopologiesFullyScheduled(cluster, strategyClass, "topo-1", "topo-4", "topo-5", + "topo-6"); assertTopologiesNotScheduled(cluster, strategyClass, "topo-2", "topo-3"); } } /** - * If users are above his or her guarantee, check if topology eviction works correctly + * If users are above his or her guarantee, check if topology eviction works correctly. */ @Test public void testOverGuaranteeEviction() { - for (Class strategyClass: strategyClasses) { + for (Class strategyClass : strategyClasses) { INimbus iNimbus = new INimbusTest(); Map supMap = genSupervisors(4, 4, 100, 1000); Map> resourceUserPool = userResourcePool( @@ -274,18 +301,21 @@ public void testOverGuaranteeEviction() { genTopology("topo-3", config, 1, 0, 1, 0, currentTime - 2, 10, "bobby"), genTopology("topo-4", config, 1, 0, 1, 0, currentTime - 2, 10, "bobby"), genTopology("topo-5", config, 1, 0, 1, 0, currentTime - 2, 29, "derek")); - Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config); + Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), + supMap, new HashMap<>(), topologies, config); scheduler = new ResourceAwareScheduler(); scheduler.prepare(config, new StormMetricsRegistry()); LOG.info("\n\n\t\tScheduling topos 1,3,4,5"); scheduler.schedule(topologies, cluster); LOG.info("\n\n\t\tDone scheduling..."); - assertTopologiesFullyScheduled(cluster, strategyClass, "topo-1", "topo-3", "topo-4", "topo-5"); + assertTopologiesFullyScheduled(cluster, strategyClass, "topo-1", "topo-3", "topo-4", + "topo-5"); - //user derek submits another topology into a full cluster - //topo6 should not be able to scheduled initially, but since topo6 has higher priority than topo5 - //topo5 will be evicted so that topo6 can be scheduled + // user derek submits another topology into a full cluster + // topo6 should not be able to scheduled initially, but since topo6 has higher priority + // than topo5 + // topo5 will be evicted so that topo6 can be scheduled topologies = addTopologies(topologies, genTopology("topo-6", config, 1, 0, 1, 0, currentTime - 2, 10, "derek")); cluster = new Cluster(cluster, topologies); @@ -293,10 +323,11 @@ public void testOverGuaranteeEviction() { scheduler.schedule(topologies, cluster); LOG.info("\n\n\t\tDone scheduling..."); - assertTopologiesFullyScheduled(cluster, strategyClass, "topo-1", "topo-3", "topo-4", "topo-6"); + assertTopologiesFullyScheduled(cluster, strategyClass, "topo-1", "topo-3", "topo-4", + "topo-6"); assertTopologiesNotScheduled(cluster, strategyClass, "topo-5"); - //user jerry submits topo2 + // user jerry submits topo2 topologies = addTopologies(topologies, genTopology("topo-2", config, 1, 0, 1, 0, currentTime - 2, 20, "jerry")); cluster = new Cluster(cluster, topologies); @@ -304,8 +335,9 @@ public void testOverGuaranteeEviction() { scheduler.schedule(topologies, cluster); LOG.info("\n\n\t\tDone scheduling..."); - assertTopologiesFullyScheduled(cluster, strategyClass, "topo-1", "topo-3", "topo-4", "topo-6"); - assertTopologiesNotScheduled(cluster, strategyClass,"topo-2", "topo-5"); + assertTopologiesFullyScheduled(cluster, strategyClass, "topo-1", "topo-3", "topo-4", + "topo-6"); + assertTopologiesNotScheduled(cluster, strategyClass, "topo-2", "topo-5"); } } } diff --git a/storm-server/src/test/java/org/apache/storm/scheduler/resource/strategies/priority/TestFIFOSchedulingPriorityStrategy.java b/storm-server/src/test/java/org/apache/storm/scheduler/resource/strategies/priority/TestFIFOSchedulingPriorityStrategy.java index 7977cfaff41..a9034e153e7 100644 --- a/storm-server/src/test/java/org/apache/storm/scheduler/resource/strategies/priority/TestFIFOSchedulingPriorityStrategy.java +++ b/storm-server/src/test/java/org/apache/storm/scheduler/resource/strategies/priority/TestFIFOSchedulingPriorityStrategy.java @@ -18,14 +18,28 @@ package org.apache.storm.scheduler.resource.strategies.priority; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.INimbusTest; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.addTopologies; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.assertTopologiesFullyScheduled; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.assertTopologiesNotScheduled; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.createClusterConfig; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.genSupervisors; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.genTopology; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.userRes; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.userResourcePool; + +import java.util.HashMap; +import java.util.Map; import org.apache.storm.Config; import org.apache.storm.DaemonConfig; +import org.apache.storm.metric.StormMetricsRegistry; import org.apache.storm.scheduler.Cluster; import org.apache.storm.scheduler.INimbus; import org.apache.storm.scheduler.SupervisorDetails; import org.apache.storm.scheduler.Topologies; import org.apache.storm.scheduler.resource.ResourceAwareScheduler; import org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler; +import org.apache.storm.scheduler.resource.normalization.ResourceMetrics; import org.apache.storm.scheduler.resource.strategies.scheduling.DefaultResourceAwareStrategy; import org.apache.storm.scheduler.resource.strategies.scheduling.DefaultResourceAwareStrategyOld; import org.apache.storm.scheduler.resource.strategies.scheduling.GenericResourceAwareStrategy; @@ -33,82 +47,92 @@ import org.apache.storm.scheduler.resource.strategies.scheduling.RoundRobinResourceAwareStrategy; import org.apache.storm.utils.Time; import org.junit.jupiter.api.Test; - -import java.util.HashMap; -import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.*; - -import org.apache.storm.metric.StormMetricsRegistry; -import org.apache.storm.scheduler.resource.normalization.ResourceMetrics; - public class TestFIFOSchedulingPriorityStrategy { - private static final Logger LOG = LoggerFactory.getLogger(TestFIFOSchedulingPriorityStrategy.class); + private static final Logger LOG = LoggerFactory + .getLogger(TestFIFOSchedulingPriorityStrategy.class); private static final Class[] strategyClasses = { - DefaultResourceAwareStrategy.class, - DefaultResourceAwareStrategyOld.class, - RoundRobinResourceAwareStrategy.class, - GenericResourceAwareStrategy.class, - GenericResourceAwareStrategyOld.class, + DefaultResourceAwareStrategy.class, + DefaultResourceAwareStrategyOld.class, + RoundRobinResourceAwareStrategy.class, + GenericResourceAwareStrategy.class, + GenericResourceAwareStrategyOld.class, }; - private Config createClusterConfig(Class strategyClass, double compPcore, double compOnHeap, double compOffHeap, + private Config createClusterConfig(Class strategyClass, double compPcore, double compOnHeap, + double compOffHeap, Map> pools) { - Config config = TestUtilsForResourceAwareScheduler.createClusterConfig(compPcore, compOnHeap, compOffHeap, pools); + Config config = TestUtilsForResourceAwareScheduler.createClusterConfig(compPcore, + compOnHeap, compOffHeap, pools); config.put(Config.TOPOLOGY_SCHEDULER_STRATEGY, strategyClass.getName()); return config; } @Test public void testFIFOEvictionStrategy() { - for (Class strategyClass: strategyClasses) { + for (Class strategyClass : strategyClasses) { try (Time.SimulatedTime sim = new Time.SimulatedTime()) { INimbus iNimbus = new INimbusTest(); Map supMap = genSupervisors(4, 4, 100.0, 1000.0); Map> resourceUserPool = userResourcePool( userRes("jerry", 200.0, 2000.0)); Config config = createClusterConfig(strategyClass, 100, 500, 500, resourceUserPool); - config.put(DaemonConfig.RESOURCE_AWARE_SCHEDULER_PRIORITY_STRATEGY, FIFOSchedulingPriorityStrategy.class.getName()); + config.put(DaemonConfig.RESOURCE_AWARE_SCHEDULER_PRIORITY_STRATEGY, + FIFOSchedulingPriorityStrategy.class.getName()); Topologies topologies = new Topologies( - genTopology("topo-1-jerry", config, 1, 0, 1, 0, Time.currentTimeSecs() - 250, 20, "jerry"), - genTopology("topo-2-bobby", config, 1, 0, 1, 0, Time.currentTimeSecs() - 200, 10, "bobby"), - genTopology("topo-3-bobby", config, 1, 0, 1, 0, Time.currentTimeSecs() - 300, 20, "bobby"), - genTopology("topo-4-derek", config, 1, 0, 1, 0, Time.currentTimeSecs() - 201, 29, "derek")); - Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config); + genTopology("topo-1-jerry", config, 1, 0, 1, 0, Time + .currentTimeSecs() - 250, 20, "jerry"), + genTopology("topo-2-bobby", config, 1, 0, 1, 0, Time + .currentTimeSecs() - 200, 10, "bobby"), + genTopology("topo-3-bobby", config, 1, 0, 1, 0, Time + .currentTimeSecs() - 300, 20, "bobby"), + genTopology("topo-4-derek", config, 1, 0, 1, 0, Time + .currentTimeSecs() - 201, 29, "derek")); + Cluster cluster = new Cluster(iNimbus, + new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config); ResourceAwareScheduler rs = new ResourceAwareScheduler(); rs.prepare(config, new StormMetricsRegistry()); try { rs.schedule(topologies, cluster); - assertTopologiesFullyScheduled(cluster, strategyClass, "topo-1-jerry", "topo-2-bobby", "topo-3-bobby", "topo-4-derek"); + assertTopologiesFullyScheduled(cluster, strategyClass, "topo-1-jerry", + "topo-2-bobby", "topo-3-bobby", "topo-4-derek"); LOG.info("\n\n\t\tINSERTING topo-5"); - //new topology needs to be scheduled - //topo-3 should be evicted since it's been up the longest + // new topology needs to be scheduled + // topo-3 should be evicted since it's been up the longest topologies = addTopologies(topologies, - genTopology("topo-5-derek", config, 1, 0, 1, 0, Time.currentTimeSecs() - 15, 29, "derek")); + genTopology("topo-5-derek", config, 1, 0, 1, 0, Time + .currentTimeSecs() - 15, 29, "derek")); - cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config); + cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), + supMap, new HashMap<>(), topologies, config); rs.schedule(topologies, cluster); - assertTopologiesFullyScheduled(cluster, strategyClass, "topo-1-jerry", "topo-2-bobby", "topo-4-derek", "topo-5-derek"); + assertTopologiesFullyScheduled(cluster, strategyClass, "topo-1-jerry", + "topo-2-bobby", "topo-4-derek", "topo-5-derek"); assertTopologiesNotScheduled(cluster, strategyClass, "topo-3-bobby"); LOG.info("\n\n\t\tINSERTING topo-6"); - //new topology needs to be scheduled. topo-4 should be evicted. Even though topo-1 from user jerry is older, topo-1 will not be evicted - //since user jerry has enough resource guarantee + // new topology needs to be scheduled. topo-4 should be evicted. Even though + // topo-1 from user jerry is older, topo-1 will not be evicted + // since user jerry has enough resource guarantee topologies = addTopologies(topologies, - genTopology("topo-6-bobby", config, 1, 0, 1, 0, Time.currentTimeSecs() - 10, 29, "bobby")); + genTopology("topo-6-bobby", config, 1, 0, 1, 0, Time + .currentTimeSecs() - 10, 29, "bobby")); - cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config); + cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), + supMap, new HashMap<>(), topologies, config); rs.schedule(topologies, cluster); - assertTopologiesFullyScheduled(cluster, strategyClass, "topo-1-jerry", "topo-2-bobby", "topo-5-derek", "topo-6-bobby"); - assertTopologiesNotScheduled(cluster, strategyClass, "topo-3-bobby", "topo-4-derek"); + assertTopologiesFullyScheduled(cluster, strategyClass, "topo-1-jerry", + "topo-2-bobby", "topo-5-derek", "topo-6-bobby"); + assertTopologiesNotScheduled(cluster, strategyClass, "topo-3-bobby", + "topo-4-derek"); } finally { rs.cleanup(); } diff --git a/storm-server/src/test/java/org/apache/storm/scheduler/resource/strategies/priority/TestGenericResourceAwareSchedulingPriorityStrategy.java b/storm-server/src/test/java/org/apache/storm/scheduler/resource/strategies/priority/TestGenericResourceAwareSchedulingPriorityStrategy.java index 74cc5247dcb..1a572da5ec2 100644 --- a/storm-server/src/test/java/org/apache/storm/scheduler/resource/strategies/priority/TestGenericResourceAwareSchedulingPriorityStrategy.java +++ b/storm-server/src/test/java/org/apache/storm/scheduler/resource/strategies/priority/TestGenericResourceAwareSchedulingPriorityStrategy.java @@ -18,6 +18,21 @@ package org.apache.storm.scheduler.resource.strategies.priority; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.addTopologies; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.assertTopologiesBeenEvicted; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.assertTopologiesFullyScheduled; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.assertTopologiesNotBeenEvicted; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.assertTopologiesNotScheduled; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.genSupervisors; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.genTopology; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.userRes; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.userResourcePool; + +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; import org.apache.storm.Config; import org.apache.storm.DaemonConfig; import org.apache.storm.metric.StormMetricsRegistry; @@ -38,26 +53,10 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; - -import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.addTopologies; -import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.assertTopologiesBeenEvicted; -import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.assertTopologiesFullyScheduled; -import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.assertTopologiesNotBeenEvicted; -import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.assertTopologiesNotScheduled; -import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.genSupervisors; -import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.genTopology; -import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.userRes; -import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.userResourcePool; - public class TestGenericResourceAwareSchedulingPriorityStrategy { private static final Class[] strategyClasses = { - GenericResourceAwareStrategy.class, - GenericResourceAwareStrategyOld.class, + GenericResourceAwareStrategy.class, + GenericResourceAwareStrategyOld.class, }; private final int currentTime = Time.currentTimeSecs(); private IScheduler scheduler = null; @@ -70,141 +69,184 @@ public void cleanup() { } } - private Config createGrasClusterConfig(Class strategyClass, double compPcore, double compOnHeap, double compOffHeap, + private Config createGrasClusterConfig(Class strategyClass, double compPcore, double compOnHeap, + double compOffHeap, Map> pools, Map genericResourceMap) { - Config config = TestUtilsForResourceAwareScheduler.createGrasClusterConfig(compPcore, compOnHeap, compOffHeap, pools, genericResourceMap); + Config config = TestUtilsForResourceAwareScheduler.createGrasClusterConfig(compPcore, + compOnHeap, compOffHeap, pools, genericResourceMap); config.put(Config.TOPOLOGY_SCHEDULER_STRATEGY, strategyClass.getName()); return config; } /* - * DefaultSchedulingPriorityStrategy will not evict topo as long as the resources request can be met + * DefaultSchedulingPriorityStrategy will not evict topo as long as the resources request can be + * met * - * Ethan asks for heavy cpu and memory while Rui asks for little cpu and memory but heavy generic resource + * Ethan asks for heavy cpu and memory while Rui asks for little cpu and memory but heavy + * generic resource * Since Rui's all types of resources request can be met, no eviction will happen. */ @Test public void testDefaultSchedulingPriorityStrategyNotEvicting() { - for (Class strategyClass: strategyClasses) { + for (Class strategyClass : strategyClasses) { Map requestedgenericResourcesMap = new HashMap<>(); requestedgenericResourcesMap.put("generic.resource.1", 40.0); // Use full memory and cpu of the cluster capacity - Config ruiConf = createGrasClusterConfig(strategyClass, 20, 50, 50, null, requestedgenericResourcesMap); - Config ethanConf = createGrasClusterConfig(strategyClass, 80, 400, 500, null, Collections.emptyMap()); + Config ruiConf = createGrasClusterConfig(strategyClass, 20, 50, 50, null, + requestedgenericResourcesMap); + Config ethanConf = createGrasClusterConfig(strategyClass, 80, 400, 500, null, + Collections.emptyMap()); Topologies topologies = new Topologies( - genTopology("ethan-topo-1", ethanConf, 1, 0, 1, 0, currentTime - 2, 10, "ethan"), - genTopology("ethan-topo-2", ethanConf, 1, 0, 1, 0, currentTime - 2, 20, "ethan"), - genTopology("ethan-topo-3", ethanConf, 1, 0, 1, 0, currentTime - 2, 28, "ethan"), - genTopology("ethan-topo-4", ethanConf, 1, 0, 1, 0, currentTime - 2, 29, "ethan")); + genTopology("ethan-topo-1", ethanConf, 1, 0, 1, 0, currentTime - 2, 10, + "ethan"), + genTopology("ethan-topo-2", ethanConf, 1, 0, 1, 0, currentTime - 2, 20, + "ethan"), + genTopology("ethan-topo-3", ethanConf, 1, 0, 1, 0, currentTime - 2, 28, + "ethan"), + genTopology("ethan-topo-4", ethanConf, 1, 0, 1, 0, currentTime - 2, 29, + "ethan")); Topologies withNewTopo = addTopologies(topologies, genTopology("rui-topo-1", ruiConf, 1, 0, 4, 0, currentTime - 2, 10, "rui")); - Config config = mkClusterConfig(strategyClass, DefaultSchedulingPriorityStrategy.class.getName()); + Config config = mkClusterConfig(strategyClass, DefaultSchedulingPriorityStrategy.class + .getName()); Cluster cluster = mkTestCluster(topologies, config); scheduler = new ResourceAwareScheduler(); scheduler.prepare(config, new StormMetricsRegistry()); scheduler.schedule(topologies, cluster); - assertTopologiesFullyScheduled(cluster, strategyClass, "ethan-topo-1", "ethan-topo-2", "ethan-topo-3", "ethan-topo-4"); + assertTopologiesFullyScheduled(cluster, strategyClass, "ethan-topo-1", "ethan-topo-2", + "ethan-topo-3", "ethan-topo-4"); cluster = new Cluster(cluster, withNewTopo); scheduler.schedule(withNewTopo, cluster); - Map> evictedTopos = ((ResourceAwareScheduler) scheduler).getEvictedTopologiesMap(); + Map> evictedTopos = ((ResourceAwareScheduler) scheduler) + .getEvictedTopologiesMap(); - assertTopologiesFullyScheduled(cluster, strategyClass, "ethan-topo-1", "ethan-topo-2", "ethan-topo-3", "ethan-topo-4"); - assertTopologiesNotBeenEvicted(cluster, strategyClass, collectMapValues(evictedTopos), "ethan-topo-1", "ethan-topo-2", "ethan-topo-3", "ethan-topo-4"); + assertTopologiesFullyScheduled(cluster, strategyClass, "ethan-topo-1", "ethan-topo-2", + "ethan-topo-3", "ethan-topo-4"); + assertTopologiesNotBeenEvicted(cluster, strategyClass, collectMapValues(evictedTopos), + "ethan-topo-1", "ethan-topo-2", "ethan-topo-3", "ethan-topo-4"); assertTopologiesFullyScheduled(cluster, strategyClass, "rui-topo-1"); } } /* - * DefaultSchedulingPriorityStrategy does not take generic resources into account when calculating score - * So even if a user is requesting a lot of generic resources other than CPU and memory, scheduler will still score it very low and kick out other topologies + * DefaultSchedulingPriorityStrategy does not take generic resources into account when + * calculating score + * So even if a user is requesting a lot of generic resources other than CPU and memory, + * scheduler will still score it very low and kick out other topologies * - * Ethan asks for medium cpu and memory while Rui asks for little cpu and memory but heavy generic resource - * However, Rui's generic request can not be met and default scoring system is not taking generic resources into account, + * Ethan asks for medium cpu and memory while Rui asks for little cpu and memory but heavy + * generic resource + * However, Rui's generic request can not be met and default scoring system is not taking + * generic resources into account, * so the score of Rui's new topo will be much lower than all Ethan's topos'. * Then all Ethan's topo will be evicted in trying to make rooms for Rui. */ @Test public void testDefaultSchedulingPriorityStrategyEvicting() { - for (Class strategyClass: strategyClasses) { + for (Class strategyClass : strategyClasses) { Map requestedgenericResourcesMap = new HashMap<>(); requestedgenericResourcesMap.put("generic.resource.1", 40.0); - Config ruiConf = createGrasClusterConfig(strategyClass, 10, 10, 10, null, requestedgenericResourcesMap); - Config ethanConf = createGrasClusterConfig(strategyClass, 60, 200, 300, null, Collections.emptyMap()); + Config ruiConf = createGrasClusterConfig(strategyClass, 10, 10, 10, null, + requestedgenericResourcesMap); + Config ethanConf = createGrasClusterConfig(strategyClass, 60, 200, 300, null, + Collections.emptyMap()); Topologies topologies = new Topologies( - genTopology("ethan-topo-1", ethanConf, 1, 0, 1, 0, currentTime - 2, 10, "ethan"), - genTopology("ethan-topo-2", ethanConf, 1, 0, 1, 0, currentTime - 2, 20, "ethan"), - genTopology("ethan-topo-3", ethanConf, 1, 0, 1, 0, currentTime - 2, 28, "ethan"), - genTopology("ethan-topo-4", ethanConf, 1, 0, 1, 0, currentTime - 2, 29, "ethan")); + genTopology("ethan-topo-1", ethanConf, 1, 0, 1, 0, currentTime - 2, 10, + "ethan"), + genTopology("ethan-topo-2", ethanConf, 1, 0, 1, 0, currentTime - 2, 20, + "ethan"), + genTopology("ethan-topo-3", ethanConf, 1, 0, 1, 0, currentTime - 2, 28, + "ethan"), + genTopology("ethan-topo-4", ethanConf, 1, 0, 1, 0, currentTime - 2, 29, + "ethan")); Topologies withNewTopo = addTopologies(topologies, genTopology("rui-topo-1", ruiConf, 1, 0, 5, 0, currentTime - 2, 10, "rui")); - Config config = mkClusterConfig(strategyClass, DefaultSchedulingPriorityStrategy.class.getName()); + Config config = mkClusterConfig(strategyClass, DefaultSchedulingPriorityStrategy.class + .getName()); Cluster cluster = mkTestCluster(topologies, config); scheduler = new ResourceAwareScheduler(); scheduler.prepare(config, new StormMetricsRegistry()); scheduler.schedule(topologies, cluster); - assertTopologiesFullyScheduled(cluster, strategyClass, "ethan-topo-1", "ethan-topo-2", "ethan-topo-3", "ethan-topo-4"); + assertTopologiesFullyScheduled(cluster, strategyClass, "ethan-topo-1", "ethan-topo-2", + "ethan-topo-3", "ethan-topo-4"); cluster = new Cluster(cluster, withNewTopo); scheduler.schedule(withNewTopo, cluster); - Map> evictedTopos = ((ResourceAwareScheduler) scheduler).getEvictedTopologiesMap(); + Map> evictedTopos = ((ResourceAwareScheduler) scheduler) + .getEvictedTopologiesMap(); - assertTopologiesFullyScheduled(cluster, strategyClass, "ethan-topo-1", "ethan-topo-2", "ethan-topo-3", "ethan-topo-4"); - assertTopologiesBeenEvicted(cluster, strategyClass, collectMapValues(evictedTopos), "ethan-topo-1", "ethan-topo-2", "ethan-topo-3", "ethan-topo-4"); + assertTopologiesFullyScheduled(cluster, strategyClass, "ethan-topo-1", "ethan-topo-2", + "ethan-topo-3", "ethan-topo-4"); + assertTopologiesBeenEvicted(cluster, strategyClass, collectMapValues(evictedTopos), + "ethan-topo-1", "ethan-topo-2", "ethan-topo-3", "ethan-topo-4"); assertTopologiesNotScheduled(cluster, strategyClass, "rui-topo-1"); } } /* - * GenericResourceAwareSchedulingPriorityStrategy extend scoring formula to accommodate generic resources + * GenericResourceAwareSchedulingPriorityStrategy extend scoring formula to accommodate generic + * resources * - * Same setting as testDefaultSchedulingPriorityStrategyEvicting, but this time, new scoring system is taking generic resources into account, - * the score of rui's new topo will be higher than all Ethan's topos' due to its crazy generic request. + * Same setting as testDefaultSchedulingPriorityStrategyEvicting, but this time, new scoring + * system is taking generic resources into account, + * the score of rui's new topo will be higher than all Ethan's topos' due to its crazy generic + * request. * At the end, all Ethan's topo will not be evicted as expected. */ @Test public void testGenericSchedulingPriorityStrategyEvicting() { - for (Class strategyClass: strategyClasses) { + for (Class strategyClass : strategyClasses) { Map requestedgenericResourcesMap = new HashMap<>(); requestedgenericResourcesMap.put("generic.resource.1", 40.0); - Config ruiConf = createGrasClusterConfig(strategyClass, 10, 10, 10, null, requestedgenericResourcesMap); - Config ethanConf = createGrasClusterConfig(strategyClass, 60, 200, 300, null, Collections.emptyMap()); + Config ruiConf = createGrasClusterConfig(strategyClass, 10, 10, 10, null, + requestedgenericResourcesMap); + Config ethanConf = createGrasClusterConfig(strategyClass, 60, 200, 300, null, + Collections.emptyMap()); Topologies topologies = new Topologies( - genTopology("ethan-topo-1", ethanConf, 1, 0, 1, 0, currentTime - 2, 10, "ethan"), - genTopology("ethan-topo-2", ethanConf, 1, 0, 1, 0, currentTime - 2, 20, "ethan"), - genTopology("ethan-topo-3", ethanConf, 1, 0, 1, 0, currentTime - 2, 28, "ethan"), - genTopology("ethan-topo-4", ethanConf, 1, 0, 1, 0, currentTime - 2, 29, "ethan")); + genTopology("ethan-topo-1", ethanConf, 1, 0, 1, 0, currentTime - 2, 10, + "ethan"), + genTopology("ethan-topo-2", ethanConf, 1, 0, 1, 0, currentTime - 2, 20, + "ethan"), + genTopology("ethan-topo-3", ethanConf, 1, 0, 1, 0, currentTime - 2, 28, + "ethan"), + genTopology("ethan-topo-4", ethanConf, 1, 0, 1, 0, currentTime - 2, 29, + "ethan")); Topologies withNewTopo = addTopologies(topologies, genTopology("rui-topo-1", ruiConf, 1, 0, 5, 0, currentTime - 2, 10, "rui")); - Config config = mkClusterConfig(strategyClass, GenericResourceAwareSchedulingPriorityStrategy.class.getName()); + Config config = mkClusterConfig(strategyClass, + GenericResourceAwareSchedulingPriorityStrategy.class.getName()); Cluster cluster = mkTestCluster(topologies, config); scheduler = new ResourceAwareScheduler(); scheduler.prepare(config, new StormMetricsRegistry()); scheduler.schedule(topologies, cluster); - assertTopologiesFullyScheduled(cluster, strategyClass, "ethan-topo-1", "ethan-topo-2", "ethan-topo-3", "ethan-topo-4"); + assertTopologiesFullyScheduled(cluster, strategyClass, "ethan-topo-1", "ethan-topo-2", + "ethan-topo-3", "ethan-topo-4"); cluster = new Cluster(cluster, withNewTopo); scheduler.schedule(withNewTopo, cluster); - Map> evictedTopos = ((ResourceAwareScheduler) scheduler).getEvictedTopologiesMap(); + Map> evictedTopos = ((ResourceAwareScheduler) scheduler) + .getEvictedTopologiesMap(); - assertTopologiesFullyScheduled(cluster, strategyClass, "ethan-topo-1", "ethan-topo-2", "ethan-topo-3", "ethan-topo-4"); - assertTopologiesNotBeenEvicted(cluster, strategyClass, collectMapValues(evictedTopos), "ethan-topo-1", "ethan-topo-2", "ethan-topo-3", "ethan-topo-4"); + assertTopologiesFullyScheduled(cluster, strategyClass, "ethan-topo-1", "ethan-topo-2", + "ethan-topo-3", "ethan-topo-4"); + assertTopologiesNotBeenEvicted(cluster, strategyClass, collectMapValues(evictedTopos), + "ethan-topo-1", "ethan-topo-2", "ethan-topo-3", "ethan-topo-4"); assertTopologiesNotScheduled(cluster, strategyClass, "rui-topo-1"); } } - private Config mkClusterConfig(Class strategyClass, String SchedulingPriorityStrategy) { Map> resourceUserPool = userResourcePool( userRes("rui", 200, 2000), @@ -213,9 +255,12 @@ private Config mkClusterConfig(Class strategyClass, String SchedulingPriorityStr Map genericResourcesOfferedMap = new HashMap<>(); genericResourcesOfferedMap.put("generic.resource.1", 50.0); - Config config = createGrasClusterConfig(strategyClass, 100, 500, 500, resourceUserPool, genericResourcesOfferedMap); - config.put(DaemonConfig.RESOURCE_AWARE_SCHEDULER_PRIORITY_STRATEGY, SchedulingPriorityStrategy); - config.put(DaemonConfig.RESOURCE_AWARE_SCHEDULER_MAX_TOPOLOGY_SCHEDULING_ATTEMPTS, 2); // allow 1 round of evictions + Config config = createGrasClusterConfig(strategyClass, 100, 500, 500, resourceUserPool, + genericResourcesOfferedMap); + config.put(DaemonConfig.RESOURCE_AWARE_SCHEDULER_PRIORITY_STRATEGY, + SchedulingPriorityStrategy); + config.put(DaemonConfig.RESOURCE_AWARE_SCHEDULER_MAX_TOPOLOGY_SCHEDULING_ATTEMPTS, + 2); // allow 1 round of evictions return config; } @@ -223,14 +268,18 @@ private Config mkClusterConfig(Class strategyClass, String SchedulingPriorityStr private Cluster mkTestCluster(Topologies topologies, Config config) { INimbus iNimbus = new TestUtilsForResourceAwareScheduler.INimbusTest(); - Map genericResourcesOfferedMap = (Map) config.get(Config.TOPOLOGY_COMPONENT_RESOURCES_MAP); + Map genericResourcesOfferedMap = (Map) config + .get(Config.TOPOLOGY_COMPONENT_RESOURCES_MAP); if (genericResourcesOfferedMap == null || genericResourcesOfferedMap.isEmpty()) { - throw new IllegalArgumentException("Generic resources map must contain something in this test: " + throw new IllegalArgumentException("Generic resources map must contain something in " + + "this test: " + TestGenericResourceAwareSchedulingPriorityStrategy.class.getName()); } - Map supMap = genSupervisors(4, 4, 100, 1000, genericResourcesOfferedMap); + Map supMap = genSupervisors(4, 4, 100, 1000, + genericResourcesOfferedMap); - return new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config); + return new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, + new HashMap<>(), topologies, config); } private Set collectMapValues(Map> map) { diff --git a/storm-server/src/test/java/org/apache/storm/scheduler/resource/strategies/scheduling/TestConstraintSolverStrategy.java b/storm-server/src/test/java/org/apache/storm/scheduler/resource/strategies/scheduling/TestConstraintSolverStrategy.java index e408f99d8ef..581ab1fd150 100644 --- a/storm-server/src/test/java/org/apache/storm/scheduler/resource/strategies/scheduling/TestConstraintSolverStrategy.java +++ b/storm-server/src/test/java/org/apache/storm/scheduler/resource/strategies/scheduling/TestConstraintSolverStrategy.java @@ -18,16 +18,35 @@ package org.apache.storm.scheduler.resource.strategies.scheduling; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.INimbusTest; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.assertStatusSuccess; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.createCSSClusterConfig; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.genSupervisors; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.genTopology; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.isStatusSuccess; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; import java.util.Set; - +import java.util.stream.Stream; +import net.minidev.json.JSONValue; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.core.config.Configurator; import org.apache.storm.Config; import org.apache.storm.DaemonConfig; +import org.apache.storm.metric.StormMetricsRegistry; import org.apache.storm.scheduler.Cluster; import org.apache.storm.scheduler.ExecutorDetails; import org.apache.storm.scheduler.SchedulerAssignment; @@ -38,35 +57,18 @@ import org.apache.storm.scheduler.WorkerSlot; import org.apache.storm.scheduler.resource.ResourceAwareScheduler; import org.apache.storm.scheduler.resource.SchedulingResult; +import org.apache.storm.scheduler.resource.normalization.ResourceMetrics; import org.apache.storm.scheduler.resource.strategies.scheduling.sorter.ExecSorterByConstraintSeverity; import org.apache.storm.scheduler.resource.strategies.scheduling.sorter.IExecSorter; import org.apache.storm.scheduler.resource.strategies.scheduling.sorter.NodeSorterHostProximity; import org.apache.storm.utils.Time; import org.apache.storm.utils.Utils; -import net.minidev.json.JSONValue; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.HashMap; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.stream.Stream; - -import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.*; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; - -import org.apache.storm.metric.StormMetricsRegistry; -import org.apache.storm.scheduler.resource.normalization.ResourceMetrics; - public class TestConstraintSolverStrategy { public static Stream data() { return Stream.of( @@ -76,7 +78,8 @@ public static Stream data() { private static final Logger LOG = LoggerFactory.getLogger(TestConstraintSolverStrategy.class); private static final int MAX_TRAVERSAL_DEPTH = 2000; private static final int NORMAL_BOLT_PARALLEL = 11; - //Dropping the parallelism of the bolts to 3 instead of 11 so we can find a solution in a reasonable amount of work when backtracking. + // Dropping the parallelism of the bolts to 3 instead of 11 so we can find a solution in a + // reasonable amount of work when backtracking. private static final int BACKTRACK_BOLT_PARALLEL = 3; private static final int CO_LOCATION_CNT = 2; @@ -85,7 +88,7 @@ public TestConstraintSolverStrategy() { BaseResourceAwareStrategy.class, ResourceAwareScheduler.class, NodeSorterHostProximity.class, Cluster.class ); - Level logLevel = Level.INFO ; // switch to Level.DEBUG for verbose otherwise Level.INFO + Level logLevel = Level.INFO; // switch to Level.DEBUG for verbose otherwise Level.INFO classesToDebug.forEach(x -> Configurator.setLevel(x.getName(), logLevel)); } @@ -105,15 +108,18 @@ public static void addConstraints(String comp1, String comp2, List> } /** - * Make test Topology configuration, but with the newer spread constraints that allow associating a number - * with the spread. This number represents the maximum co-located component count. Default under the old + * Make test Topology configuration, but with the newer spread constraints that allow + * associating a number + * with the spread. This number represents the maximum co-located component count. Default under + * the old * configuration is assumed to be 1. * * @param maxCoLocationCnt Maximum co-located component (spout-0), minimum value is 1. * @param consolidatedConfigFlag The consolidated config flag * @return topology configuration map */ - public Map makeTestTopoConf(int maxCoLocationCnt, boolean consolidatedConfigFlag) { + public Map makeTestTopoConf(int maxCoLocationCnt, + boolean consolidatedConfigFlag) { if (maxCoLocationCnt < 1) { maxCoLocationCnt = 1; } @@ -142,42 +148,56 @@ public Map makeTestTopoConf(int maxCoLocationCnt, boolean consol return config; } + public Map makeTestTopoConf(boolean consolidatedConfigFlag) { + return makeTestTopoConf(1, consolidatedConfigFlag); + } + /** * Set Config.TOPOLOGY_RAS_CONSTRAINTS (when consolidatedConfigFlag is true) or both - * Config.TOPOLOGY_RAS_CONSTRAINTS/Config.TOPOLOGY_SPREAD_COMPONENTS (when consolidatedConfigFlag is false). + * Config.TOPOLOGY_RAS_CONSTRAINTS/Config.TOPOLOGY_SPREAD_COMPONENTS (when + * consolidatedConfigFlag is false). * - * When consolidatedConfigFlag is true, use the new more consolidated format to set Config.TOPOLOGY_RAS_CONSTRAINTS. - * When false, use the old configuration format for Config.TOPOLOGY_RAS_CONSTRAINTS/TOPOLOGY_SPREAD_COMPONENTS. + *

      When consolidatedConfigFlag is true, use the new more consolidated format to set + * Config.TOPOLOGY_RAS_CONSTRAINTS. + * When false, use the old configuration format for + * Config.TOPOLOGY_RAS_CONSTRAINTS/TOPOLOGY_SPREAD_COMPONENTS. * - * @param constraints List of components, where the first one cannot co-exist with the others in the list + * @param constraints List of components, where the first one cannot co-exist with the others in + * the list * @param spreads Map of component and its maxCoLocationCnt * @param config Configuration to be updated * @param consolidatedConfigFlag The consolidated config flag */ - private void setConstraintConfig(List> constraints, Map spreads, Map config, boolean consolidatedConfigFlag) { + private void setConstraintConfig(List> constraints, Map spreads, + Map config, boolean consolidatedConfigFlag) { if (consolidatedConfigFlag) { // single configuration for each component - Map> modifiedConstraints = new HashMap<>(); - for (List constraint: constraints) { + Map> modifiedConstraints = new HashMap<>(); + for (List constraint : constraints) { if (constraint.size() < 2) { continue; } String comp = constraint.get(0); List others = constraint.subList(1, constraint.size()); - List incompatibleComponents = (List) modifiedConstraints.computeIfAbsent(comp, k -> new HashMap<>()) + List incompatibleComponents = (List) modifiedConstraints + .computeIfAbsent(comp, k -> new HashMap<>()) .computeIfAbsent(ConstraintSolverConfig.CONSTRAINT_TYPE_INCOMPATIBLE_COMPONENTS, k -> new ArrayList<>()); incompatibleComponents.addAll(others); } - for (String comp: spreads.keySet()) { - modifiedConstraints.computeIfAbsent(comp, k -> new HashMap<>()).put(ConstraintSolverConfig.CONSTRAINT_TYPE_MAX_NODE_CO_LOCATION_CNT, "" + spreads.get(comp)); + for (String comp : spreads.keySet()) { + modifiedConstraints.computeIfAbsent(comp, k -> new HashMap<>()) + .put(ConstraintSolverConfig.CONSTRAINT_TYPE_MAX_NODE_CO_LOCATION_CNT, "" + + spreads.get(comp)); } config.put(Config.TOPOLOGY_RAS_CONSTRAINTS, modifiedConstraints); } else { // constraint and MaxCoLocationCnts are separate - no maxCoLocationCnt implied as 1 config.put(Config.TOPOLOGY_RAS_CONSTRAINTS, constraints); - for (Map.Entry e: spreads.entrySet()) { + for (Map.Entry e : spreads.entrySet()) { if (e.getValue() > 1) { - fail(String.format("Invalid %s=%d for component=%s, expecting 1 for old-style configuration", + fail(String + .format("Invalid %s=%d for component=%s, expecting 1 for old-style " + + "configuration", ConstraintSolverConfig.CONSTRAINT_TYPE_MAX_NODE_CO_LOCATION_CNT, e.getValue(), e.getKey())); @@ -187,10 +207,6 @@ private void setConstraintConfig(List> constraints, Map makeTestTopoConf(boolean consolidatedConfigFlag) { - return makeTestTopoConf(1, consolidatedConfigFlag); - } - public static TopologyDetails makeTopology(Map config, int boltParallel) { return genTopology("testTopo", config, 1, 4, 4, boltParallel, 0, 0, "user"); } @@ -199,15 +215,18 @@ public static Cluster makeCluster(Topologies topologies) { return makeCluster(topologies, null); } - public static Cluster makeCluster(Topologies topologies, Map supMap) { + public static Cluster makeCluster(Topologies topologies, Map supMap) { if (supMap == null) { supMap = genSupervisors(4, 2, 120, 1200); } Map config = Utils.readDefaultConfig(); - return new Cluster(new INimbusTest(), new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config); + return new Cluster(new INimbusTest(), new ResourceMetrics(new StormMetricsRegistry()), + supMap, new HashMap<>(), topologies, config); } - public void basicUnitTestWithKillAndRecover(ConstraintSolverStrategy cs, int boltParallel, int coLocationCnt, boolean consolidatedConfigFlag) { + public void basicUnitTestWithKillAndRecover(ConstraintSolverStrategy cs, int boltParallel, + int coLocationCnt, boolean consolidatedConfigFlag) { Map config = makeTestTopoConf(coLocationCnt, consolidatedConfigFlag); cs.prepare(config); @@ -221,18 +240,19 @@ public void basicUnitTestWithKillAndRecover(ConstraintSolverStrategy cs, int bol assertTrue(result.isSuccess(), "Assert scheduling topology success " + result); assertEquals(0, cluster.getUnassignedExecutors(topo).size(), - "Assert no unassigned executors, found unassigned: " + cluster.getUnassignedExecutors(topo)); + "Assert no unassigned executors, found unassigned: " + cluster + .getUnassignedExecutors(topo)); assertTrue(ConstraintSolverStrategy.validateSolution(cluster, topo), "Valid Scheduling?"); LOG.info("Slots Used {}", cluster.getAssignmentById(topo.getId()).getSlots()); LOG.info("Assignment {}", cluster.getAssignmentById(topo.getId()).getSlotToExecutors()); - //simulate worker loss + // simulate worker loss SchedulerAssignment assignment = cluster.getAssignmentById(topo.getId()); Set slotsToDelete = new HashSet<>(); Set slots = assignment.getSlots(); int i = 0; - for (WorkerSlot slot: slots) { + for (WorkerSlot slot : slots) { if (i % 2 == 0) { slotsToDelete.add(slot); } @@ -240,7 +260,7 @@ public void basicUnitTestWithKillAndRecover(ConstraintSolverStrategy cs, int bol } LOG.info("KILL WORKER(s) {}", slotsToDelete); - for (WorkerSlot slot: slotsToDelete) { + for (WorkerSlot slot : slotsToDelete) { cluster.freeSlot(slot); } @@ -251,7 +271,8 @@ public void basicUnitTestWithKillAndRecover(ConstraintSolverStrategy cs, int bol LOG.info("Done scheduling {}...", result); assertTrue(result.isSuccess(), "Assert scheduling topology success " + result); - assertEquals(0, cluster.getUnassignedExecutors(topo).size(), "topo all executors scheduled?"); + assertEquals(0, cluster.getUnassignedExecutors(topo).size(), + "topo all executors scheduled?"); assertTrue(ConstraintSolverStrategy.validateSolution(cluster, topo), "Valid Scheduling?"); } @@ -266,8 +287,8 @@ public void testMissingConfig(boolean consolidatedConfigFlag) { // with one or more undefined components with partial constraints { - String s = consolidatedConfigFlag ? - String.format( + String s = consolidatedConfigFlag + ? String.format( "{ \"comp-1\": " + " { \"%s\": 2, " + " \"%s\": [\"comp-2\", \"comp-3\" ] }, " @@ -298,9 +319,12 @@ public void testMissingConfig(boolean consolidatedConfigFlag) { Map conf = new HashMap<>(); conf.put(Config.TOPOLOGY_RAS_CONSTRAINTS, jsonValue); new ConstraintSolverConfig("test-topoid-2", conf, new HashSet<>()); - new ConstraintSolverConfig("test-topoid-3", conf, new HashSet<>(Collections.singletonList("comp-x"))); - new ConstraintSolverConfig("test-topoid-4", conf, new HashSet<>(Collections.singletonList("comp-1"))); - new ConstraintSolverConfig("test-topoid-5", conf, new HashSet<>(Collections.singletonList("comp-1, comp-x"))); + new ConstraintSolverConfig("test-topoid-3", conf, new HashSet<>(Collections + .singletonList("comp-x"))); + new ConstraintSolverConfig("test-topoid-4", conf, new HashSet<>(Collections + .singletonList("comp-1"))); + new ConstraintSolverConfig("test-topoid-5", conf, new HashSet<>(Collections + .singletonList("comp-1, comp-x"))); } } @@ -326,7 +350,8 @@ public void testNewConstraintFormat(boolean consolidatedConfigFlag) { config.put(Config.TOPOLOGY_RAS_CONSTRAINTS, jsonValue); Set allComps = new HashSet<>(); allComps.addAll(Arrays.asList("comp-1", "comp-2", "comp-3", "comp-4", "comp-5")); - ConstraintSolverConfig constraintSolverConfig = new ConstraintSolverConfig("test-topoid-1", config, allComps); + ConstraintSolverConfig constraintSolverConfig = new ConstraintSolverConfig("test-topoid-1", + config, allComps); Set expectedSetComp1 = new HashSet<>(); expectedSetComp1.addAll(Arrays.asList("comp-2", "comp-3")); @@ -334,20 +359,27 @@ public void testNewConstraintFormat(boolean consolidatedConfigFlag) { expectedSetComp2.addAll(Arrays.asList("comp-1", "comp-4")); Set expectedSetComp3 = new HashSet<>(); expectedSetComp3.addAll(Arrays.asList("comp-1", "comp-5")); - assertEquals(expectedSetComp1, constraintSolverConfig.getIncompatibleComponentSets().get("comp-1"), "comp-1 incompatible components"); - assertEquals(expectedSetComp2, constraintSolverConfig.getIncompatibleComponentSets().get("comp-2"), "comp-2 incompatible components"); - assertEquals(expectedSetComp3, constraintSolverConfig.getIncompatibleComponentSets().get("comp-3"), "comp-3 incompatible components"); - assertEquals(2, (int) constraintSolverConfig.getMaxNodeCoLocationCnts().getOrDefault("comp-1", -1), "comp-1 maxNodeCoLocationCnt"); - assertNull(constraintSolverConfig.getMaxNodeCoLocationCnts().get("comp-2"), "comp-2 maxNodeCoLocationCnt"); + assertEquals(expectedSetComp1, constraintSolverConfig.getIncompatibleComponentSets() + .get("comp-1"), "comp-1 incompatible components"); + assertEquals(expectedSetComp2, constraintSolverConfig.getIncompatibleComponentSets() + .get("comp-2"), "comp-2 incompatible components"); + assertEquals(expectedSetComp3, constraintSolverConfig.getIncompatibleComponentSets() + .get("comp-3"), "comp-3 incompatible components"); + assertEquals(2, (int) constraintSolverConfig.getMaxNodeCoLocationCnts() + .getOrDefault("comp-1", -1), "comp-1 maxNodeCoLocationCnt"); + assertNull(constraintSolverConfig.getMaxNodeCoLocationCnts().get("comp-2"), + "comp-2 maxNodeCoLocationCnt"); } @ParameterizedTest @MethodSource("data") public void testConstraintSolverForceBacktrackWithSpreadCoLocation(boolean consolidatedConfigFlag) { - //The best way to force backtracking is to change the heuristic so the components are reversed, so it is hard + // The best way to force backtracking is to change the heuristic so the components are + // reversed, so it is hard // to find an answer. if (CO_LOCATION_CNT > 1 && !consolidatedConfigFlag) { - LOG.info("INFO: Skipping Test {} with {}={} (required 1), and consolidatedConfigFlag={} (required false)", + LOG.info("INFO: Skipping Test {} with {}={} (required 1), and " + + "consolidatedConfigFlag={} (required false)", "testConstraintSolverForceBacktrackWithSpreadCoLocation", ConstraintSolverConfig.CONSTRAINT_TYPE_MAX_NODE_CO_LOCATION_CNT, CO_LOCATION_CNT, @@ -361,7 +393,8 @@ protected void prepareForScheduling(Cluster cluster, TopologyDetails topologyDet super.prepareForScheduling(cluster, topologyDetails); // set a reversing execSorter instance - IExecSorter execSorter = new ExecSorterByConstraintSeverity(cluster, topologyDetails) { + IExecSorter execSorter = new ExecSorterByConstraintSeverity(cluster, + topologyDetails) { @Override public List sortExecutors(Set unassignedExecutors) { List tmp = super.sortExecutors(unassignedExecutors); @@ -375,20 +408,23 @@ public List sortExecutors(Set unassignedExecut setExecSorter(execSorter); } }; - basicUnitTestWithKillAndRecover(cs, BACKTRACK_BOLT_PARALLEL, CO_LOCATION_CNT, consolidatedConfigFlag); + basicUnitTestWithKillAndRecover(cs, BACKTRACK_BOLT_PARALLEL, CO_LOCATION_CNT, + consolidatedConfigFlag); } @ParameterizedTest @MethodSource("data") public void testConstraintSolver(boolean consolidatedConfigFlag) { - basicUnitTestWithKillAndRecover(new ConstraintSolverStrategy(), NORMAL_BOLT_PARALLEL, 1, consolidatedConfigFlag); + basicUnitTestWithKillAndRecover(new ConstraintSolverStrategy(), NORMAL_BOLT_PARALLEL, 1, + consolidatedConfigFlag); } @ParameterizedTest @MethodSource("data") public void testConstraintSolverWithSpreadCoLocation(boolean consolidatedConfigFlag) { if (CO_LOCATION_CNT > 1 && !consolidatedConfigFlag) { - LOG.info("INFO: Skipping Test {} with {}={} (required 1), and consolidatedConfigFlag={} (required false)", + LOG.info("INFO: Skipping Test {} with {}={} (required 1), and " + + "consolidatedConfigFlag={} (required false)", "testConstraintSolverWithSpreadCoLocation", ConstraintSolverConfig.CONSTRAINT_TYPE_MAX_NODE_CO_LOCATION_CNT, CO_LOCATION_CNT, @@ -396,7 +432,8 @@ public void testConstraintSolverWithSpreadCoLocation(boolean consolidatedConfigF return; } - basicUnitTestWithKillAndRecover(new ConstraintSolverStrategy(), NORMAL_BOLT_PARALLEL, CO_LOCATION_CNT, consolidatedConfigFlag); + basicUnitTestWithKillAndRecover(new ConstraintSolverStrategy(), NORMAL_BOLT_PARALLEL, + CO_LOCATION_CNT, consolidatedConfigFlag); } public void basicFailureTest(String confKey, Object confValue, @@ -419,7 +456,8 @@ public void basicFailureTest(String confKey, Object confValue, @ParameterizedTest @MethodSource("data") public void testTooManyStateTransitions(boolean consolidatedConfigFlag) { - basicFailureTest(Config.TOPOLOGY_RAS_CONSTRAINT_MAX_STATE_SEARCH, 10, new ConstraintSolverStrategy(), consolidatedConfigFlag); + basicFailureTest(Config.TOPOLOGY_RAS_CONSTRAINT_MAX_STATE_SEARCH, 10, + new ConstraintSolverStrategy(), consolidatedConfigFlag); } @ParameterizedTest @@ -429,12 +467,13 @@ public void testTimeout(boolean consolidatedConfigFlag) { ConstraintSolverStrategy cs = new ConstraintSolverStrategy() { @Override protected SchedulingResult scheduleExecutorsOnNodes(List orderedExecutors, Iterable sortedNodes) { - //Each time we try to schedule a new component simulate taking 1 second longer + // Each time we try to schedule a new component simulate taking 1 second longer Time.advanceTime(1_001); return super.scheduleExecutorsOnNodes(orderedExecutors, sortedNodes); } }; - basicFailureTest(Config.TOPOLOGY_RAS_CONSTRAINT_MAX_TIME_SECS, 1, cs, consolidatedConfigFlag); + basicFailureTest(Config.TOPOLOGY_RAS_CONSTRAINT_MAX_TIME_SECS, 1, cs, + consolidatedConfigFlag); } } @@ -446,11 +485,14 @@ public void testScheduleLargeExecutorConstraintCountSmall(boolean consolidatedCo /* * Test scheduling large number of executors and constraints. - * This test can succeed only with new style config that allows maxCoLocationCnt = parallelismMultiplier. - * In prior code, this test would succeed because effectively the old code did not properly enforce the + * This test can succeed only with new style config that allows maxCoLocationCnt = + * parallelismMultiplier. + * In prior code, this test would succeed because effectively the old code did not properly + * enforce the * SPREAD constraint. * - * Cluster has sufficient resources for scheduling to succeed but can fail due to StackOverflowError. + * Cluster has sufficient resources for scheduling to succeed but can fail due to + * StackOverflowError. */ @ParameterizedTest @MethodSource("data") @@ -458,13 +500,17 @@ public void testScheduleLargeExecutorConstraintCountLarge(boolean consolidatedCo testScheduleLargeExecutorConstraintCount(20, consolidatedConfigFlag); } - private void testScheduleLargeExecutorConstraintCount(int parallelismMultiplier, boolean consolidatedConfigFlag) { + private void testScheduleLargeExecutorConstraintCount(int parallelismMultiplier, + boolean consolidatedConfigFlag) { if (parallelismMultiplier > 1 && !consolidatedConfigFlag) { - assertFalse(consolidatedConfigFlag, "Large parallelism test requires new consolidated constraint format with maxCoLocationCnt=" + parallelismMultiplier); + assertFalse(consolidatedConfigFlag, + "Large parallelism test requires new consolidated constraint format with " + + "maxCoLocationCnt=" + parallelismMultiplier); return; } - // Add 1 topology with large number of executors and constraints. Too many can cause a java.lang.StackOverflowError + // Add 1 topology with large number of executors and constraints. Too many can cause a + // java.lang.StackOverflowError Config config = createCSSClusterConfig(10, 10, 0, null); config.put(Config.TOPOLOGY_RAS_CONSTRAINT_MAX_STATE_SEARCH, 50000); config.put(Config.TOPOLOGY_RAS_CONSTRAINT_MAX_TIME_SECS, 120); @@ -485,10 +531,12 @@ private void testScheduleLargeExecutorConstraintCount(int parallelismMultiplier, setConstraintConfig(constraints, spreads, config, consolidatedConfigFlag); - TopologyDetails topo = genTopology("testTopo-" + parallelismMultiplier, config, 10, 10, 30 * parallelismMultiplier, 30 * parallelismMultiplier, 31414, 0, "user"); + TopologyDetails topo = genTopology("testTopo-" + parallelismMultiplier, config, 10, 10, + 30 * parallelismMultiplier, 30 * parallelismMultiplier, 31414, 0, "user"); Topologies topologies = new Topologies(topo); - Map supMap = genSupervisors(30 * parallelismMultiplier, 30, 3500, 35000); + Map supMap = genSupervisors(30 * parallelismMultiplier, 30, 3500, + 35000); Cluster cluster = makeCluster(topologies, supMap); ResourceAwareScheduler scheduler = new ResourceAwareScheduler(); @@ -496,8 +544,10 @@ private void testScheduleLargeExecutorConstraintCount(int parallelismMultiplier, scheduler.schedule(topologies, cluster); boolean scheduleSuccess = isStatusSuccess(cluster.getStatus(topo.getId())); - LOG.info("testScheduleLargeExecutorCount scheduling {} with {}x executor multiplier, consolidatedConfigFlag={}", - scheduleSuccess ? "succeeds" : "fails", parallelismMultiplier, consolidatedConfigFlag); + LOG.info("testScheduleLargeExecutorCount scheduling {} with {}x executor multiplier, " + + "consolidatedConfigFlag={}", + scheduleSuccess + ? "succeeds" : "fails", parallelismMultiplier, consolidatedConfigFlag); assertTrue(scheduleSuccess); } @@ -505,7 +555,8 @@ private void testScheduleLargeExecutorConstraintCount(int parallelismMultiplier, @MethodSource("data") public void testIntegrationWithRAS(boolean consolidatedConfigFlag) { if (!consolidatedConfigFlag) { - LOG.info("Skipping test since bolt-1 maxCoLocationCnt=10 requires consolidatedConfigFlag=true, current={}", consolidatedConfigFlag); + LOG.info("Skipping test since bolt-1 maxCoLocationCnt=10 requires " + + "consolidatedConfigFlag=true, current={}", consolidatedConfigFlag); return; } @@ -541,29 +592,33 @@ public void testIntegrationWithRAS(boolean consolidatedConfigFlag) { try { rs.schedule(topologies, cluster); assertStatusSuccess(cluster, topo.getId()); - assertEquals(0, cluster.getUnassignedExecutors(topo).size(), "topo all executors scheduled?"); + assertEquals(0, cluster.getUnassignedExecutors(topo).size(), + "topo all executors scheduled?"); } finally { rs.cleanup(); } - //simulate worker loss + // simulate worker loss Map newExecToSlot = new HashMap<>(); - Map execToSlot = cluster.getAssignmentById(topo.getId()).getExecutorToSlot(); - Iterator> it =execToSlot.entrySet().iterator(); - for (int i = 0; i execToSlot = cluster.getAssignmentById(topo.getId()) + .getExecutorToSlot(); + Iterator> it = execToSlot.entrySet().iterator(); + for (int i = 0; i < execToSlot.size() / 2; i++) { ExecutorDetails exec = it.next().getKey(); WorkerSlot ws = it.next().getValue(); newExecToSlot.put(exec, ws); } Map newAssignments = new HashMap<>(); - newAssignments.put(topo.getId(), new SchedulerAssignmentImpl(topo.getId(), newExecToSlot, null, null)); + newAssignments.put(topo.getId(), new SchedulerAssignmentImpl(topo.getId(), newExecToSlot, + null, null)); cluster.setAssignments(newAssignments, false); rs.prepare(config, new StormMetricsRegistry()); try { rs.schedule(topologies, cluster); assertStatusSuccess(cluster, topo.getId()); - assertEquals(0, cluster.getUnassignedExecutors(topo).size(), "topo all executors scheduled?"); + assertEquals(0, cluster.getUnassignedExecutors(topo).size(), + "topo all executors scheduled?"); } finally { rs.cleanup(); } @@ -584,7 +639,8 @@ public void testZeroExecutorScheduling(boolean consolidatedConfigFlag) { cs.schedule(cluster, topo); LOG.info("********************* Scheduling Zero Unassigned Executors *********************"); cs.schedule(cluster, topo); // reschedule a fully schedule topology - LOG.info("********************* End of Scheduling Zero Unassigned Executors *********************"); + LOG.info("********************* End of Scheduling Zero Unassigned Executors " + + "*********************"); } @ParameterizedTest diff --git a/storm-server/src/test/java/org/apache/storm/scheduler/resource/strategies/scheduling/TestDefaultResourceAwareStrategy.java b/storm-server/src/test/java/org/apache/storm/scheduler/resource/strategies/scheduling/TestDefaultResourceAwareStrategy.java index 0fa0f86fb8b..e260ac9d87f 100644 --- a/storm-server/src/test/java/org/apache/storm/scheduler/resource/strategies/scheduling/TestDefaultResourceAwareStrategy.java +++ b/storm-server/src/test/java/org/apache/storm/scheduler/resource/strategies/scheduling/TestDefaultResourceAwareStrategy.java @@ -18,6 +18,33 @@ package org.apache.storm.scheduler.resource.strategies.scheduling; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.INimbusTest; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.TestBolt; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.TestSpout; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.assertTopologiesFullyScheduled; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.assertTopologiesNotScheduled; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.genExecsAndComps; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.genSupervisors; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.genTopology; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.closeTo; +import static org.hamcrest.Matchers.is; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map.Entry; +import java.util.Map; +import java.util.stream.Collectors; import org.apache.storm.Config; import org.apache.storm.daemon.StormCommon; import org.apache.storm.daemon.nimbus.Nimbus; @@ -57,56 +84,32 @@ import org.junit.jupiter.params.provider.EnumSource; import org.junit.jupiter.params.provider.ValueSource; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.stream.Collectors; - -import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.INimbusTest; -import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.TestBolt; -import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.TestSpout; -import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.assertTopologiesFullyScheduled; -import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.assertTopologiesNotScheduled; -import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.genExecsAndComps; -import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.genSupervisors; -import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.genTopology; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.closeTo; -import static org.hamcrest.Matchers.is; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - @ExtendWith({NormalizedResourcesExtension.class}) public class TestDefaultResourceAwareStrategy { private static final Class[] strategyClasses = { - DefaultResourceAwareStrategy.class, - DefaultResourceAwareStrategyOld.class, + DefaultResourceAwareStrategy.class, + DefaultResourceAwareStrategyOld.class, }; private static final int CURRENT_TIME = 1450418597; private static IScheduler scheduler = null; + private enum SharedMemoryType { SHARED_OFF_HEAP_NODE, SHARED_OFF_HEAP_WORKER, SHARED_ON_HEAP_WORKER } + protected enum WorkerRestrictionType { WORKER_RESTRICTION_ONE_EXECUTOR, WORKER_RESTRICTION_ONE_COMPONENT, WORKER_RESTRICTION_NONE } - private Config createClusterConfig(Class strategyClass, double compPcore, double compOnHeap, double compOffHeap, + private Config createClusterConfig(Class strategyClass, double compPcore, double compOnHeap, + double compOffHeap, Map> pools) { - Config config = TestUtilsForResourceAwareScheduler.createClusterConfig(compPcore, compOnHeap, compOffHeap, pools); + Config config = TestUtilsForResourceAwareScheduler.createClusterConfig(compPcore, + compOnHeap, compOffHeap, pools); config.put(Config.TOPOLOGY_SCHEDULER_STRATEGY, strategyClass.getName()); return config; } @@ -114,7 +117,7 @@ private Config createClusterConfig(Class strategyClass, double compPcore, double private static class TestDNSToSwitchMapping implements DNSToSwitchMapping { private final Map result; - public TestDNSToSwitchMapping(Map ... racks) { + public TestDNSToSwitchMapping(Map... racks) { Map ret = new HashMap<>(); for (int rackNum = 0; rackNum < racks.length; rackNum++) { String rack = "rack-" + rackNum; @@ -145,7 +148,7 @@ public void cleanup() { @ParameterizedTest @EnumSource(SharedMemoryType.class) public void testMultipleSharedMemoryWithOneExecutorPerWorker(SharedMemoryType memoryType) { - for (Class strategyClass: strategyClasses) { + for (Class strategyClass : strategyClasses) { int spoutParallelism = 4; double cpuPercent = 10; double memoryOnHeap = 10; @@ -158,7 +161,8 @@ public void testMultipleSharedMemoryWithOneExecutorPerWorker(SharedMemoryType me switch (memoryType) { case SHARED_OFF_HEAP_NODE: builder.setSpout("spout", new TestSpout(), spoutParallelism) - .addSharedMemory(new SharedOffHeapWithinNode(sharedOffHeapWithinNode, "spout shared off heap within node")); + .addSharedMemory(new SharedOffHeapWithinNode(sharedOffHeapWithinNode, + "spout shared off heap within node")); break; case SHARED_OFF_HEAP_WORKER: builder.setSpout("spout", new TestSpout(), spoutParallelism) @@ -166,13 +170,17 @@ public void testMultipleSharedMemoryWithOneExecutorPerWorker(SharedMemoryType me break; case SHARED_ON_HEAP_WORKER: builder.setSpout("spout", new TestSpout(), spoutParallelism) - .addSharedMemory(new SharedOnHeap(sharedOnHeapWithinWorker, "spout shared on heap within worker")); + .addSharedMemory(new SharedOnHeap(sharedOnHeapWithinWorker, + "spout shared on heap within worker")); + break; + default: break; } StormTopology stormToplogy = builder.createTopology(); INimbus iNimbus = new INimbusTest(); Map supMap = genSupervisors(4, 4, 500, 1000); - Config conf = createClusterConfig(strategyClass, cpuPercent, memoryOnHeap, memoryOffHeap, null); + Config conf = createClusterConfig(strategyClass, cpuPercent, memoryOnHeap, + memoryOffHeap, null); conf.put(Config.TOPOLOGY_PRIORITY, 0); conf.put(Config.TOPOLOGY_NAME, "testTopology"); @@ -182,50 +190,75 @@ public void testMultipleSharedMemoryWithOneExecutorPerWorker(SharedMemoryType me genExecsAndComps(stormToplogy), CURRENT_TIME, "user"); Topologies topologies = new Topologies(topo); - Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, conf); + Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), + supMap, new HashMap<>(), topologies, conf); scheduler = new ResourceAwareScheduler(); scheduler.prepare(conf, new StormMetricsRegistry()); scheduler.schedule(topologies, cluster); - TopologyResources topologyResources = cluster.getTopologyResourcesMap().get(topo.getId()); + TopologyResources topologyResources = cluster.getTopologyResourcesMap().get(topo + .getId()); SchedulerAssignment assignment = cluster.getAssignmentById(topo.getId()); - long numNodes = assignment.getSlotToExecutors().keySet().stream().map(WorkerSlot::getNodeId).distinct().count(); + long numNodes = assignment.getSlotToExecutors().keySet().stream() + .map(WorkerSlot::getNodeId).distinct().count(); switch (memoryType) { case SHARED_OFF_HEAP_NODE: // 4 workers on single node. OffHeapNode memory is shared - assertThat(topologyResources.getAssignedMemOnHeap(), closeTo(spoutParallelism * memoryOnHeap, 0.01)); - assertThat(topologyResources.getAssignedMemOffHeap(), closeTo(spoutParallelism * memoryOffHeap + sharedOffHeapWithinNode, 0.01)); + assertThat(topologyResources.getAssignedMemOnHeap(), + closeTo(spoutParallelism * memoryOnHeap, 0.01)); + assertThat(topologyResources.getAssignedMemOffHeap(), + closeTo(spoutParallelism * memoryOffHeap + sharedOffHeapWithinNode, + 0.01)); assertThat(topologyResources.getAssignedSharedMemOnHeap(), closeTo(0, 0.01)); - assertThat(topologyResources.getAssignedSharedMemOffHeap(), closeTo(sharedOffHeapWithinNode, 0.01)); - assertThat(topologyResources.getAssignedNonSharedMemOnHeap(), closeTo(spoutParallelism * memoryOnHeap, 0.01)); - assertThat(topologyResources.getAssignedNonSharedMemOffHeap(), closeTo(spoutParallelism * memoryOffHeap, 0.01)); + assertThat(topologyResources.getAssignedSharedMemOffHeap(), + closeTo(sharedOffHeapWithinNode, 0.01)); + assertThat(topologyResources.getAssignedNonSharedMemOnHeap(), + closeTo(spoutParallelism * memoryOnHeap, 0.01)); + assertThat(topologyResources.getAssignedNonSharedMemOffHeap(), + closeTo(spoutParallelism * memoryOffHeap, 0.01)); assertThat(numNodes, is(1L)); assertThat(cluster.getAssignedNumWorkers(topo), is(spoutParallelism)); break; case SHARED_OFF_HEAP_WORKER: - // 4 workers on 2 nodes. OffHeapWorker memory not shared -- consumed 4x, once for each worker - assertThat(topologyResources.getAssignedMemOnHeap(), closeTo(spoutParallelism * memoryOnHeap, 0.01)); - assertThat(topologyResources.getAssignedMemOffHeap(), closeTo(spoutParallelism * (memoryOffHeap + sharedOffHeapWithinWorker), 0.01)); + // 4 workers on 2 nodes. OffHeapWorker memory not shared -- consumed 4x, once + // for each worker + assertThat(topologyResources.getAssignedMemOnHeap(), + closeTo(spoutParallelism * memoryOnHeap, 0.01)); + assertThat(topologyResources.getAssignedMemOffHeap(), + closeTo(spoutParallelism * (memoryOffHeap + sharedOffHeapWithinWorker), + 0.01)); assertThat(topologyResources.getAssignedSharedMemOnHeap(), closeTo(0, 0.01)); - assertThat(topologyResources.getAssignedSharedMemOffHeap(), closeTo(spoutParallelism * sharedOffHeapWithinWorker, 0.01)); - assertThat(topologyResources.getAssignedNonSharedMemOnHeap(), closeTo(spoutParallelism * memoryOnHeap, 0.01)); - assertThat(topologyResources.getAssignedNonSharedMemOffHeap(), closeTo(spoutParallelism * memoryOffHeap, 0.01)); + assertThat(topologyResources.getAssignedSharedMemOffHeap(), + closeTo(spoutParallelism * sharedOffHeapWithinWorker, 0.01)); + assertThat(topologyResources.getAssignedNonSharedMemOnHeap(), + closeTo(spoutParallelism * memoryOnHeap, 0.01)); + assertThat(topologyResources.getAssignedNonSharedMemOffHeap(), + closeTo(spoutParallelism * memoryOffHeap, 0.01)); assertThat(numNodes, is(2L)); assertThat(cluster.getAssignedNumWorkers(topo), is(spoutParallelism)); break; case SHARED_ON_HEAP_WORKER: - // 4 workers on 2 nodes. onHeap memory not shared -- consumed 4x, once for each worker - assertThat(topologyResources.getAssignedMemOnHeap(), closeTo(spoutParallelism * (memoryOnHeap + sharedOnHeapWithinWorker), 0.01)); - assertThat(topologyResources.getAssignedMemOffHeap(), closeTo(spoutParallelism * memoryOffHeap, 0.01)); - assertThat(topologyResources.getAssignedSharedMemOnHeap(), closeTo(spoutParallelism * sharedOnHeapWithinWorker, 0.01)); + // 4 workers on 2 nodes. onHeap memory not shared -- consumed 4x, once for each + // worker + assertThat(topologyResources.getAssignedMemOnHeap(), + closeTo(spoutParallelism * (memoryOnHeap + sharedOnHeapWithinWorker), + 0.01)); + assertThat(topologyResources.getAssignedMemOffHeap(), + closeTo(spoutParallelism * memoryOffHeap, 0.01)); + assertThat(topologyResources.getAssignedSharedMemOnHeap(), + closeTo(spoutParallelism * sharedOnHeapWithinWorker, 0.01)); assertThat(topologyResources.getAssignedSharedMemOffHeap(), closeTo(0, 0.01)); - assertThat(topologyResources.getAssignedNonSharedMemOnHeap(), closeTo(spoutParallelism * memoryOnHeap, 0.01)); - assertThat(topologyResources.getAssignedNonSharedMemOffHeap(), closeTo(spoutParallelism * memoryOffHeap, 0.01)); + assertThat(topologyResources.getAssignedNonSharedMemOnHeap(), + closeTo(spoutParallelism * memoryOnHeap, 0.01)); + assertThat(topologyResources.getAssignedNonSharedMemOffHeap(), + closeTo(spoutParallelism * memoryOffHeap, 0.01)); assertThat(numNodes, is(2L)); assertThat(cluster.getAssignedNumWorkers(topo), is(spoutParallelism)); break; + default: + break; } } } @@ -235,7 +268,7 @@ public void testMultipleSharedMemoryWithOneExecutorPerWorker(SharedMemoryType me */ @Test public void testSchedulingNegativeResources() { - for (Class strategyClass: strategyClasses) { + for (Class strategyClass : strategyClasses) { int spoutParallelism = 2; int boltParallelism = 2; double cpuPercent = 10; @@ -245,7 +278,8 @@ public void testSchedulingNegativeResources() { double sharedOffHeapWithinNode = 700; double sharedOffHeapWithinWorker = 500; - Config conf = createClusterConfig(strategyClass, cpuPercent, memoryOnHeap, memoryOffHeap, null); + Config conf = createClusterConfig(strategyClass, cpuPercent, memoryOnHeap, + memoryOffHeap, null); TopologyDetails[] topo = new TopologyDetails[2]; // 1st topology @@ -253,11 +287,16 @@ public void testSchedulingNegativeResources() { builder.setSpout("spout", new TestSpout(), spoutParallelism); builder.setBolt("bolt-1", new TestBolt(), - boltParallelism).addSharedMemory(new SharedOffHeapWithinWorker(sharedOffHeapWithinWorker, "bolt-1 shared off heap within worker")).shuffleGrouping("spout"); + boltParallelism) + .addSharedMemory(new SharedOffHeapWithinWorker(sharedOffHeapWithinWorker, + "bolt-1 shared off heap within worker")).shuffleGrouping("spout"); builder.setBolt("bolt-2", new TestBolt(), - boltParallelism).addSharedMemory(new SharedOffHeapWithinNode(sharedOffHeapWithinNode, "bolt-2 shared off heap within node")).shuffleGrouping("bolt-1"); + boltParallelism) + .addSharedMemory(new SharedOffHeapWithinNode(sharedOffHeapWithinNode, + "bolt-2 shared off heap within node")).shuffleGrouping("bolt-1"); builder.setBolt("bolt-3", new TestBolt(), - boltParallelism).addSharedMemory(new SharedOnHeap(sharedOnHeapWithinWorker, "bolt-3 shared on heap within worker")).shuffleGrouping("bolt-2"); + boltParallelism).addSharedMemory(new SharedOnHeap(sharedOnHeapWithinWorker, + "bolt-3 shared on heap within worker")).shuffleGrouping("bolt-2"); StormTopology stormTopology = builder.createTopology(); conf.put(Config.TOPOLOGY_PRIORITY, 1); @@ -269,7 +308,9 @@ public void testSchedulingNegativeResources() { // 2nd topology builder = new TopologyBuilder(); builder.setSpout("spout", new TestSpout(), - spoutParallelism).addSharedMemory(new SharedOffHeapWithinNode(sharedOffHeapWithinNode, "spout shared off heap within node")); + spoutParallelism) + .addSharedMemory(new SharedOffHeapWithinNode(sharedOffHeapWithinNode, + "spout shared off heap within node")); stormTopology = builder.createTopology(); conf.put(Config.TOPOLOGY_PRIORITY, 0); @@ -279,7 +320,8 @@ public void testSchedulingNegativeResources() { Map supMap = genSupervisors(1, 4, 500, 2000); Topologies topologies = new Topologies(topo[0]); - Cluster cluster = new Cluster(new INimbusTest(), new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, conf); + Cluster cluster = new Cluster(new INimbusTest(), + new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, conf); // schedule 1st topology scheduler = new ResourceAwareScheduler(); @@ -288,8 +330,10 @@ public void testSchedulingNegativeResources() { assertTopologiesFullyScheduled(cluster, strategyClass, topo[0].getName()); // attempt scheduling both topologies. - // this triggered negative resource event as the second topology incorrectly scheduled with the first in place - // first topology should get evicted for higher priority (lower value) second topology to successfully schedule + // this triggered negative resource event as the second topology incorrectly scheduled + // with the first in place + // first topology should get evicted for higher priority (lower value) second topology + // to successfully schedule topologies = new Topologies(topo[0], topo[1]); cluster = new Cluster(cluster, topologies); scheduler.schedule(topologies, cluster); @@ -297,17 +341,18 @@ public void testSchedulingNegativeResources() { assertTopologiesFullyScheduled(cluster, strategyClass, topo[1].getName()); // check negative resource count - assertThat(cluster.getResourceMetrics().getNegativeResourceEventsMeter().getCount(), is(0L)); + assertThat(cluster.getResourceMetrics().getNegativeResourceEventsMeter().getCount(), + is(0L)); } } /** - * test if the scheduling shared memory is correct with/without oneExecutorPerWorker enabled + * Test if the scheduling shared memory is correct with/without oneExecutorPerWorker enabled. */ @ParameterizedTest @EnumSource(WorkerRestrictionType.class) public void testDefaultResourceAwareStrategySharedMemory(WorkerRestrictionType schedulingLimitation) { - for (Class strategyClass: strategyClasses) { + for (Class strategyClass : strategyClasses) { int spoutParallelism = 2; int boltParallelism = 2; int numBolts = 3; @@ -322,17 +367,23 @@ public void testDefaultResourceAwareStrategySharedMemory(WorkerRestrictionType s builder.setSpout("spout", new TestSpout(), spoutParallelism); builder.setBolt("bolt-1", new TestBolt(), - boltParallelism).addSharedMemory(new SharedOffHeapWithinWorker(sharedOffHeapWithinWorker, "bolt-1 shared off heap within worker")).shuffleGrouping("spout"); + boltParallelism) + .addSharedMemory(new SharedOffHeapWithinWorker(sharedOffHeapWithinWorker, + "bolt-1 shared off heap within worker")).shuffleGrouping("spout"); builder.setBolt("bolt-2", new TestBolt(), - boltParallelism).addSharedMemory(new SharedOffHeapWithinNode(sharedOffHeapWithinNode, "bolt-2 shared off heap within node")).shuffleGrouping("bolt-1"); + boltParallelism) + .addSharedMemory(new SharedOffHeapWithinNode(sharedOffHeapWithinNode, + "bolt-2 shared off heap within node")).shuffleGrouping("bolt-1"); builder.setBolt("bolt-3", new TestBolt(), - boltParallelism).addSharedMemory(new SharedOnHeap(sharedOnHeapWithinWorker, "bolt-3 shared on heap within worker")).shuffleGrouping("bolt-2"); + boltParallelism).addSharedMemory(new SharedOnHeap(sharedOnHeapWithinWorker, + "bolt-3 shared on heap within worker")).shuffleGrouping("bolt-2"); StormTopology stormTopology = builder.createTopology(); INimbus iNimbus = new INimbusTest(); Map supMap = genSupervisors(4, 4, 500, 2000); - Config conf = createClusterConfig(strategyClass, cpuPercent, memoryOnHeap, memoryOffHeap, null); + Config conf = createClusterConfig(strategyClass, cpuPercent, memoryOnHeap, + memoryOffHeap, null); conf.put(Config.TOPOLOGY_PRIORITY, 0); conf.put(Config.TOPOLOGY_NAME, "testTopology"); @@ -344,12 +395,15 @@ public void testDefaultResourceAwareStrategySharedMemory(WorkerRestrictionType s case WORKER_RESTRICTION_ONE_COMPONENT: conf.put(Config.TOPOLOGY_RAS_ONE_COMPONENT_PER_WORKER, true); break; + default: + break; } TopologyDetails topo = new TopologyDetails("testTopology-id", conf, stormTopology, 0, genExecsAndComps(stormTopology), CURRENT_TIME, "user"); Topologies topologies = new Topologies(topo); - Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, conf); + Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), + supMap, new HashMap<>(), topologies, conf); scheduler = new ResourceAwareScheduler(); scheduler.prepare(conf, new StormMetricsRegistry()); @@ -372,7 +426,8 @@ public void testDefaultResourceAwareStrategySharedMemory(WorkerRestrictionType s // WorkerRestrictionType.WORKER_RESTRICTION_ONE_COMPONENT // expect 4 workers, 1 node - for (Entry entry : cluster.getSupervisorsResourcesMap().entrySet()) { + for (Entry entry : cluster.getSupervisorsResourcesMap() + .entrySet()) { String supervisorId = entry.getKey(); SupervisorResources resources = entry.getValue(); assertTrue(resources.getTotalCpu() >= resources.getUsedCpu(), supervisorId); @@ -381,9 +436,12 @@ public void testDefaultResourceAwareStrategySharedMemory(WorkerRestrictionType s int totalNumberOfTasks = spoutParallelism + boltParallelism * numBolts; SchedulerAssignment assignment = cluster.getAssignmentById(topo.getId()); - TopologyResources topologyResources = cluster.getTopologyResourcesMap().get(topo.getId()); - long numNodes = assignment.getSlotToExecutors().keySet().stream().map(WorkerSlot::getNodeId).distinct().count(); - String assignmentString = "Assignments:\n\t" + assignment.getSlotToExecutors().entrySet().stream() + TopologyResources topologyResources = cluster.getTopologyResourcesMap().get(topo + .getId()); + long numNodes = assignment.getSlotToExecutors().keySet().stream() + .map(WorkerSlot::getNodeId).distinct().count(); + String assignmentString = "Assignments:\n\t" + assignment.getSlotToExecutors() + .entrySet().stream() .map(x -> String.format("Node=%s, components=%s", x.getKey().getNodeId(), x.getValue().stream() @@ -396,34 +454,48 @@ public void testDefaultResourceAwareStrategySharedMemory(WorkerRestrictionType s if (schedulingLimitation == WorkerRestrictionType.WORKER_RESTRICTION_NONE) { // Everything should fit in a single slot double totalExpectedCPU = totalNumberOfTasks * cpuPercent; - double totalExpectedOnHeap = (totalNumberOfTasks * memoryOnHeap) + sharedOnHeapWithinWorker; - double totalExpectedWorkerOffHeap = (totalNumberOfTasks * memoryOffHeap) + sharedOffHeapWithinWorker; + double totalExpectedOnHeap = (totalNumberOfTasks * memoryOnHeap) + + sharedOnHeapWithinWorker; + double totalExpectedWorkerOffHeap = (totalNumberOfTasks * memoryOffHeap) + + sharedOffHeapWithinWorker; assertThat(assignment.getSlots().size(), is(1)); WorkerSlot ws = assignment.getSlots().iterator().next(); String nodeId = ws.getNodeId(); assertThat(assignment.getNodeIdToTotalSharedOffHeapNodeMemory().size(), is(1)); - assertThat(assignment.getNodeIdToTotalSharedOffHeapNodeMemory().get(nodeId), closeTo(sharedOffHeapWithinNode, 0.01)); + assertThat(assignment.getNodeIdToTotalSharedOffHeapNodeMemory().get(nodeId), + closeTo(sharedOffHeapWithinNode, 0.01)); assertThat(assignment.getScheduledResources().size(), is(1)); WorkerResources resources = assignment.getScheduledResources().get(ws); assertThat(resources.get_cpu(), closeTo(totalExpectedCPU, 0.01)); assertThat(resources.get_mem_on_heap(), closeTo(totalExpectedOnHeap, 0.01)); assertThat(resources.get_mem_off_heap(), closeTo(totalExpectedWorkerOffHeap, 0.01)); - assertThat(resources.get_shared_mem_on_heap(), closeTo(sharedOnHeapWithinWorker, 0.01)); - assertThat(resources.get_shared_mem_off_heap(), closeTo(sharedOffHeapWithinWorker, 0.01)); + assertThat(resources.get_shared_mem_on_heap(), closeTo(sharedOnHeapWithinWorker, + 0.01)); + assertThat(resources.get_shared_mem_off_heap(), closeTo(sharedOffHeapWithinWorker, + 0.01)); } else if (schedulingLimitation == WorkerRestrictionType.WORKER_RESTRICTION_ONE_EXECUTOR) { - double expectedMemOnHeap = (totalNumberOfTasks * memoryOnHeap) + 2 * sharedOnHeapWithinWorker; - double expectedMemOffHeap = (totalNumberOfTasks * memoryOffHeap) + 2 * sharedOffHeapWithinWorker + 2 * sharedOffHeapWithinNode; + double expectedMemOnHeap = (totalNumberOfTasks * memoryOnHeap) + + 2 * sharedOnHeapWithinWorker; + double expectedMemOffHeap = (totalNumberOfTasks * memoryOffHeap) + + 2 * sharedOffHeapWithinWorker + 2 * sharedOffHeapWithinNode; double expectedMemSharedOnHeap = 2 * sharedOnHeapWithinWorker; - double expectedMemSharedOffHeap = 2 * sharedOffHeapWithinWorker + 2 * sharedOffHeapWithinNode; + double expectedMemSharedOffHeap = 2 * sharedOffHeapWithinWorker + + 2 * sharedOffHeapWithinNode; double expectedMemNonSharedOnHeap = totalNumberOfTasks * memoryOnHeap; double expectedMemNonSharedOffHeap = totalNumberOfTasks * memoryOffHeap; - assertThat(topologyResources.getAssignedMemOnHeap(), closeTo(expectedMemOnHeap, 0.01)); - assertThat(topologyResources.getAssignedMemOffHeap(), closeTo(expectedMemOffHeap, 0.01)); - assertThat(topologyResources.getAssignedSharedMemOnHeap(), closeTo(expectedMemSharedOnHeap, 0.01)); - assertThat(topologyResources.getAssignedSharedMemOffHeap(), closeTo(expectedMemSharedOffHeap, 0.01)); - assertThat(topologyResources.getAssignedNonSharedMemOnHeap(), closeTo(expectedMemNonSharedOnHeap, 0.01)); - assertThat(topologyResources.getAssignedNonSharedMemOffHeap(), closeTo(expectedMemNonSharedOffHeap, 0.01)); + assertThat(topologyResources.getAssignedMemOnHeap(), closeTo(expectedMemOnHeap, + 0.01)); + assertThat(topologyResources.getAssignedMemOffHeap(), closeTo(expectedMemOffHeap, + 0.01)); + assertThat(topologyResources.getAssignedSharedMemOnHeap(), + closeTo(expectedMemSharedOnHeap, 0.01)); + assertThat(topologyResources.getAssignedSharedMemOffHeap(), + closeTo(expectedMemSharedOffHeap, 0.01)); + assertThat(topologyResources.getAssignedNonSharedMemOnHeap(), + closeTo(expectedMemNonSharedOnHeap, 0.01)); + assertThat(topologyResources.getAssignedNonSharedMemOffHeap(), + closeTo(expectedMemNonSharedOffHeap, 0.01)); double totalExpectedCPU = totalNumberOfTasks * cpuPercent; assertThat(topologyResources.getAssignedCpu(), closeTo(totalExpectedCPU, 0.01)); @@ -432,18 +504,27 @@ public void testDefaultResourceAwareStrategySharedMemory(WorkerRestrictionType s assertThat(assignment.getSlots().size(), is(8)); assertThat(assignmentString, numNodes, is(2L)); } else if (schedulingLimitation == WorkerRestrictionType.WORKER_RESTRICTION_ONE_COMPONENT) { - double expectedMemOnHeap = (totalNumberOfTasks * memoryOnHeap) + sharedOnHeapWithinWorker; - double expectedMemOffHeap = (totalNumberOfTasks * memoryOffHeap) + sharedOffHeapWithinWorker + sharedOffHeapWithinNode; + double expectedMemOnHeap = (totalNumberOfTasks * memoryOnHeap) + + sharedOnHeapWithinWorker; + double expectedMemOffHeap = (totalNumberOfTasks * memoryOffHeap) + + sharedOffHeapWithinWorker + sharedOffHeapWithinNode; double expectedMemSharedOnHeap = sharedOnHeapWithinWorker; - double expectedMemSharedOffHeap = sharedOffHeapWithinWorker + sharedOffHeapWithinNode; + double expectedMemSharedOffHeap = sharedOffHeapWithinWorker + + sharedOffHeapWithinNode; double expectedMemNonSharedOnHeap = totalNumberOfTasks * memoryOnHeap; double expectedMemNonSharedOffHeap = totalNumberOfTasks * memoryOffHeap; - assertThat(topologyResources.getAssignedMemOnHeap(), closeTo(expectedMemOnHeap, 0.01)); - assertThat(topologyResources.getAssignedMemOffHeap(), closeTo(expectedMemOffHeap, 0.01)); - assertThat(topologyResources.getAssignedSharedMemOnHeap(), closeTo(expectedMemSharedOnHeap, 0.01)); - assertThat(topologyResources.getAssignedSharedMemOffHeap(), closeTo(expectedMemSharedOffHeap, 0.01)); - assertThat(topologyResources.getAssignedNonSharedMemOnHeap(), closeTo(expectedMemNonSharedOnHeap, 0.01)); - assertThat(topologyResources.getAssignedNonSharedMemOffHeap(), closeTo(expectedMemNonSharedOffHeap, 0.01)); + assertThat(topologyResources.getAssignedMemOnHeap(), closeTo(expectedMemOnHeap, + 0.01)); + assertThat(topologyResources.getAssignedMemOffHeap(), closeTo(expectedMemOffHeap, + 0.01)); + assertThat(topologyResources.getAssignedSharedMemOnHeap(), + closeTo(expectedMemSharedOnHeap, 0.01)); + assertThat(topologyResources.getAssignedSharedMemOffHeap(), + closeTo(expectedMemSharedOffHeap, 0.01)); + assertThat(topologyResources.getAssignedNonSharedMemOnHeap(), + closeTo(expectedMemNonSharedOnHeap, 0.01)); + assertThat(topologyResources.getAssignedNonSharedMemOffHeap(), + closeTo(expectedMemNonSharedOffHeap, 0.01)); double totalExpectedCPU = totalNumberOfTasks * cpuPercent; assertThat(topologyResources.getAssignedCpu(), closeTo(totalExpectedCPU, 0.01)); @@ -456,18 +537,22 @@ public void testDefaultResourceAwareStrategySharedMemory(WorkerRestrictionType s } /** - * test if the scheduling logic for the DefaultResourceAwareStrategy is correct + * Test if the scheduling logic for the DefaultResourceAwareStrategy is correct * when topology.acker.executors.per.worker is set to different values. * - * If {@link Config#TOPOLOGY_ACKER_EXECUTORS} is not set, - * it will be calculated by Nimbus as (num of estimated worker * topology.acker.executors.per.worker). - * In this test, {@link Config#TOPOLOGY_ACKER_EXECUTORS} is set to 2 (num of estimated workers based on topo resources usage) + *

      If {@link Config#TOPOLOGY_ACKER_EXECUTORS} is not set, + * it will be calculated by Nimbus as (num of estimated worker * + * topology.acker.executors.per.worker). + * In this test, {@link Config#TOPOLOGY_ACKER_EXECUTORS} is set to 2 (num of estimated workers + * based on topo resources usage) * - * For different value for {@link Config#TOPOLOGY_RAS_ACKER_EXECUTORS_PER_WORKER}: + *

      For different value for {@link Config#TOPOLOGY_RAS_ACKER_EXECUTORS_PER_WORKER}: * -1: Note we don't really set it to be -1. - * It is just a special case in this test that topology.acker.executors.per.worker is unset, nimbus will set to 1 by default. + * It is just a special case in this test that topology.acker.executors.per.worker is unset, + * nimbus will set to 1 by default. * 0: Since {@link Config#TOPOLOGY_ACKER_EXECUTORS} is not set either, acking is disabled. - * 1: 2 ackers in total. Distribute 1 acker per worker. With ackers being added, this topology will now need 3 workers. + * 1: 2 ackers in total. Distribute 1 acker per worker. With ackers being added, this topology + * will now need 3 workers. * Then first two worker will get 1 acker and last worker get 0. * 2: 4 ackers in total. First two workers will get 2 acker per worker respectively. */ @@ -475,7 +560,7 @@ public void testDefaultResourceAwareStrategySharedMemory(WorkerRestrictionType s @ValueSource(ints = {-1, 0, 1, 2}) public void testDefaultResourceAwareStrategyWithoutSettingAckerExecutors(int numOfAckersPerWorker) throws InvalidTopologyException { - for (Class strategyClass: strategyClasses) { + for (Class strategyClass : strategyClasses) { int spoutParallelism = 1; int boltParallelism = 2; TopologyBuilder builder = new TopologyBuilder(); @@ -504,72 +589,79 @@ public void testDefaultResourceAwareStrategyWithoutSettingAckerExecutors(int num // but with ackers added, probably more worker will be launched. // Parameterized test on different numOfAckersPerWorker if (numOfAckersPerWorker == -1) { - // Both Config.TOPOLOGY_ACKER_EXECUTORS and Config.TOPOLOGY_RAS_ACKER_EXECUTORS_PER_WORKER are not set + // Both Config.TOPOLOGY_ACKER_EXECUTORS and + // Config.TOPOLOGY_RAS_ACKER_EXECUTORS_PER_WORKER are not set // Default will be 2 (estimate num of workers) and 1 respectively } else { conf.put(Config.TOPOLOGY_RAS_ACKER_EXECUTORS_PER_WORKER, numOfAckersPerWorker); } - int estimatedNumWorker = ServerUtils.getEstimatedWorkerCountForRasTopo(conf, stormTopology); + int estimatedNumWorker = ServerUtils.getEstimatedWorkerCountForRasTopo(conf, + stormTopology); Nimbus.setUpAckerExecutorConfigs(topoName, conf, conf, estimatedNumWorker); conf.put(Config.TOPOLOGY_ACKER_RESOURCES_ONHEAP_MEMORY_MB, 250); conf.put(Config.TOPOLOGY_ACKER_CPU_PCORE_PERCENT, 50); TopologyDetails topo = new TopologyDetails("testTopology-id", conf, stormTopology, 0, - genExecsAndComps(StormCommon.systemTopology(conf, stormTopology)), CURRENT_TIME, "user"); + genExecsAndComps(StormCommon.systemTopology(conf, + stormTopology)), CURRENT_TIME, "user"); Topologies topologies = new Topologies(topo); - Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, conf); + Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), + supMap, new HashMap<>(), topologies, conf); scheduler = new ResourceAwareScheduler(); scheduler.prepare(conf, new StormMetricsRegistry()); scheduler.schedule(topologies, cluster); - // Ordered execs: [[6, 6], [2, 2], [4, 4], [5, 5], [1, 1], [3, 3], [0, 0], [8, 8], [7, 7]] + // Ordered execs: [[6, 6], [2, 2], [4, 4], [5, 5], [1, 1], [3, 3], [0, 0], [8, 8], [7, + // 7]] // Ackers: [[8, 8], [7, 7]] (+ [[9, 9], [10, 10]] when numOfAckersPerWorker=2) HashSet> expectedScheduling = new HashSet<>(); if (numOfAckersPerWorker == -1 || numOfAckersPerWorker == 1) { - // Setting topology.acker.executors = null and topology.acker.executors.per.worker = null - // are equivalent to topology.acker.executors = null and topology.acker.executors.per.worker = 1 + // Setting topology.acker.executors = null and topology.acker.executors.per.worker = + // null + // are equivalent to topology.acker.executors = null and + // topology.acker.executors.per.worker = 1 expectedScheduling.add(new HashSet<>(Arrays.asList( - new ExecutorDetails(6, 6), //bolt-3 - new ExecutorDetails(2, 2), //bolt-1 - new ExecutorDetails(4, 4), //bolt-2 - new ExecutorDetails(8, 8)))); //acker + new ExecutorDetails(6, 6), // bolt-3 + new ExecutorDetails(2, 2), // bolt-1 + new ExecutorDetails(4, 4), // bolt-2 + new ExecutorDetails(8, 8)))); // acker expectedScheduling.add(new HashSet<>(Arrays.asList( - new ExecutorDetails(5, 5), //bolt-3 - new ExecutorDetails(1, 1), //bolt-1 - new ExecutorDetails(3, 3), //bolt-2 - new ExecutorDetails(7, 7)))); //acker + new ExecutorDetails(5, 5), // bolt-3 + new ExecutorDetails(1, 1), // bolt-1 + new ExecutorDetails(3, 3), // bolt-2 + new ExecutorDetails(7, 7)))); // acker expectedScheduling.add(new HashSet<>(Arrays.asList( - new ExecutorDetails(0, 0)))); //spout + new ExecutorDetails(0, 0)))); // spout } else if (numOfAckersPerWorker == 0) { expectedScheduling.add(new HashSet<>(Arrays.asList( - new ExecutorDetails(6, 6), //bolt-3 - new ExecutorDetails(2, 2), //bolt-1 - new ExecutorDetails(4, 4), //bolt-2 - new ExecutorDetails(5, 5)))); //bolt-3 + new ExecutorDetails(6, 6), // bolt-3 + new ExecutorDetails(2, 2), // bolt-1 + new ExecutorDetails(4, 4), // bolt-2 + new ExecutorDetails(5, 5)))); // bolt-3 expectedScheduling.add(new HashSet<>(Arrays.asList( - new ExecutorDetails(0, 0), //spout - new ExecutorDetails(3, 3), //bolt-2 - new ExecutorDetails(1, 1)))); //bolt-1 + new ExecutorDetails(0, 0), // spout + new ExecutorDetails(3, 3), // bolt-2 + new ExecutorDetails(1, 1)))); // bolt-1 } else if (numOfAckersPerWorker == 2) { expectedScheduling.add(new HashSet<>(Arrays.asList( - new ExecutorDetails(6, 6), //bolt-3 - new ExecutorDetails(2, 2), //bolt-1 - new ExecutorDetails(7, 7), //acker - new ExecutorDetails(8, 8)))); //acker + new ExecutorDetails(6, 6), // bolt-3 + new ExecutorDetails(2, 2), // bolt-1 + new ExecutorDetails(7, 7), // acker + new ExecutorDetails(8, 8)))); // acker expectedScheduling.add(new HashSet<>(Arrays.asList( - new ExecutorDetails(4, 4), //bolt-2 - new ExecutorDetails(5, 5), //bolt-3 - new ExecutorDetails(9, 9), //acker - new ExecutorDetails(10, 10)))); //acker + new ExecutorDetails(4, 4), // bolt-2 + new ExecutorDetails(5, 5), // bolt-3 + new ExecutorDetails(9, 9), // acker + new ExecutorDetails(10, 10)))); // acker expectedScheduling.add(new HashSet<>(Arrays.asList( - new ExecutorDetails(1, 1), //bolt-1 - new ExecutorDetails(3, 3), //bolt-2 - new ExecutorDetails(0, 0)))); //spout + new ExecutorDetails(1, 1), // bolt-1 + new ExecutorDetails(3, 3), // bolt-2 + new ExecutorDetails(0, 0)))); // spout } HashSet> foundScheduling = new HashSet<>(); SchedulerAssignment assignment = cluster.getAssignmentById("testTopology-id"); @@ -582,17 +674,17 @@ public void testDefaultResourceAwareStrategyWithoutSettingAckerExecutors(int num } /** - * test if the scheduling logic for the DefaultResourceAwareStrategy is correct + * Test if the scheduling logic for the DefaultResourceAwareStrategy is correct * when topology.acker.executors is set. * - * If yes, topology.acker.executors.per.worker setting will be ignored and calculated as + *

      If yes, topology.acker.executors.per.worker setting will be ignored and calculated as * Math.ceil(topology.acker.executors / estimate num of workers) by Nimbus */ @ParameterizedTest @ValueSource(ints = {-1, 0, 2, 300}) public void testDefaultResourceAwareStrategyWithSettingAckerExecutors(int numOfAckersPerWorker) throws InvalidTopologyException { - for (Class strategyClass: strategyClasses) { + for (Class strategyClass : strategyClasses) { int spoutParallelism = 1; int boltParallelism = 2; TopologyBuilder builder = new TopologyBuilder(); @@ -627,41 +719,45 @@ public void testDefaultResourceAwareStrategyWithSettingAckerExecutors(int numOfA conf.put(Config.TOPOLOGY_RAS_ACKER_EXECUTORS_PER_WORKER, numOfAckersPerWorker); } - int estimatedNumWorker = ServerUtils.getEstimatedWorkerCountForRasTopo(conf, stormTopology); + int estimatedNumWorker = ServerUtils.getEstimatedWorkerCountForRasTopo(conf, + stormTopology); Nimbus.setUpAckerExecutorConfigs(topoName, conf, conf, estimatedNumWorker); conf.put(Config.TOPOLOGY_ACKER_RESOURCES_ONHEAP_MEMORY_MB, 250); conf.put(Config.TOPOLOGY_ACKER_CPU_PCORE_PERCENT, 50); TopologyDetails topo = new TopologyDetails("testTopology-id", conf, stormTopology, 0, - genExecsAndComps(StormCommon.systemTopology(conf, stormTopology)), CURRENT_TIME, "user"); + genExecsAndComps(StormCommon.systemTopology(conf, + stormTopology)), CURRENT_TIME, "user"); Topologies topologies = new Topologies(topo); - Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, conf); + Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), + supMap, new HashMap<>(), topologies, conf); scheduler = new ResourceAwareScheduler(); scheduler.prepare(conf, new StormMetricsRegistry()); scheduler.schedule(topologies, cluster); - // Sorted execs: [[6, 6], [2, 2], [4, 4], [5, 5], [1, 1], [3, 3], [0, 0], [8, 8], [7, 7], [10, 10], [9, 9]] + // Sorted execs: [[6, 6], [2, 2], [4, 4], [5, 5], [1, 1], [3, 3], [0, 0], [8, 8], [7, + // 7], [10, 10], [9, 9]] // Ackers: [[8, 8], [7, 7], [10, 10], [9, 9]] HashSet> expectedScheduling = new HashSet<>(); expectedScheduling.add(new HashSet<>(Arrays.asList( - new ExecutorDetails(6, 6), //bolt-3 - new ExecutorDetails(2, 2), //bolt-1 - new ExecutorDetails(7, 7), //acker - new ExecutorDetails(8, 8)))); //acker + new ExecutorDetails(6, 6), // bolt-3 + new ExecutorDetails(2, 2), // bolt-1 + new ExecutorDetails(7, 7), // acker + new ExecutorDetails(8, 8)))); // acker expectedScheduling.add(new HashSet<>(Arrays.asList( - new ExecutorDetails(5, 5), //bolt-3 - new ExecutorDetails(4, 4), //bolt-2 - new ExecutorDetails(9, 9), //acker - new ExecutorDetails(10, 10)))); //acker + new ExecutorDetails(5, 5), // bolt-3 + new ExecutorDetails(4, 4), // bolt-2 + new ExecutorDetails(9, 9), // acker + new ExecutorDetails(10, 10)))); // acker expectedScheduling.add(new HashSet<>(Arrays.asList( - new ExecutorDetails(0, 0), //spout - new ExecutorDetails(3, 3), //bolt-2 - new ExecutorDetails(1, 1)))); //bolt-1 + new ExecutorDetails(0, 0), // spout + new ExecutorDetails(3, 3), // bolt-2 + new ExecutorDetails(1, 1)))); // bolt-1 HashSet> foundScheduling = new HashSet<>(); SchedulerAssignment assignment = cluster.getAssignmentById("testTopology-id"); @@ -674,12 +770,13 @@ public void testDefaultResourceAwareStrategyWithSettingAckerExecutors(int numOfA } /** - * test if the scheduling logic for the DefaultResourceAwareStrategy (when made by network proximity needs.) is correct + * Test if the scheduling logic for the DefaultResourceAwareStrategy (when made by network + * proximity needs.) is correct. */ @Test public void testDefaultResourceAwareStrategyInFavorOfShuffle() throws InvalidTopologyException { - for (Class strategyClass: strategyClasses) { + for (Class strategyClass : strategyClasses) { int spoutParallelism = 1; int boltParallelism = 2; TopologyBuilder builder = new TopologyBuilder(); @@ -704,10 +801,12 @@ public void testDefaultResourceAwareStrategyInFavorOfShuffle() conf.put(Config.TOPOLOGY_RAS_ORDER_EXECUTORS_BY_PROXIMITY_NEEDS, true); TopologyDetails topo = new TopologyDetails("testTopology-id", conf, stormToplogy, 0, - genExecsAndComps(StormCommon.systemTopology(conf, stormToplogy)), CURRENT_TIME, "user"); + genExecsAndComps(StormCommon.systemTopology(conf, + stormToplogy)), CURRENT_TIME, "user"); Topologies topologies = new Topologies(topo); - Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, conf); + Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), + supMap, new HashMap<>(), topologies, conf); ResourceAwareScheduler rs = new ResourceAwareScheduler(); @@ -718,15 +817,15 @@ public void testDefaultResourceAwareStrategyInFavorOfShuffle() HashSet> expectedScheduling = new HashSet<>(); expectedScheduling.add(new HashSet<>(Arrays.asList( - new ExecutorDetails(0, 0), //spout - new ExecutorDetails(6, 6), //bolt-2 - new ExecutorDetails(2, 2), //bolt-1 - new ExecutorDetails(7, 7)))); //acker + new ExecutorDetails(0, 0), // spout + new ExecutorDetails(6, 6), // bolt-2 + new ExecutorDetails(2, 2), // bolt-1 + new ExecutorDetails(7, 7)))); // acker expectedScheduling.add(new HashSet<>(Arrays.asList( - new ExecutorDetails(3, 3), //bolt-3 - new ExecutorDetails(5, 5), //bolt-2 - new ExecutorDetails(4, 4), //bolt-3 - new ExecutorDetails(1, 1)))); //bolt-1 + new ExecutorDetails(3, 3), // bolt-3 + new ExecutorDetails(5, 5), // bolt-2 + new ExecutorDetails(4, 4), // bolt-3 + new ExecutorDetails(1, 1)))); // bolt-1 HashSet> foundScheduling = new HashSet<>(); SchedulerAssignment assignment = cluster.getAssignmentById("testTopology-id"); for (Collection execs : assignment.getSlotToExecutors().values()) { @@ -738,28 +837,33 @@ public void testDefaultResourceAwareStrategyInFavorOfShuffle() } /** - * Test whether strategy will choose correct rack + * Test whether strategy will choose correct rack. */ @Test public void testMultipleRacks() { - for (Class strategyClass: strategyClasses) { + for (Class strategyClass : strategyClasses) { final Map supMap = new HashMap<>(); final Map supMapRack0 = genSupervisors(10, 4, 0, 400, 8000); - //generate another rack of supervisors with less resources + // generate another rack of supervisors with less resources final Map supMapRack1 = genSupervisors(10, 4, 10, 200, 4000); - //generate some supervisors that are depleted of one resource + // generate some supervisors that are depleted of one resource final Map supMapRack2 = genSupervisors(10, 4, 20, 0, 8000); - //generate some that has alot of memory but little of cpu - final Map supMapRack3 = genSupervisors(10, 4, 30, 10, 8000 * 2 + 4000); + // generate some that has alot of memory but little of cpu + final Map supMapRack3 = genSupervisors(10, 4, 30, 10, + 8000 * 2 + 4000); - //generate some that has alot of cpu but little of memory - final Map supMapRack4 = genSupervisors(10, 4, 40, 400 + 200 + 10, 1000); + // generate some that has alot of cpu but little of memory + final Map supMapRack4 = genSupervisors(10, 4, 40, 400 + 200 + + 10, 1000); - //Generate some that have neither resource, to verify that the strategy will prioritize this last - //Also put a generic resource with 0 value in the resources list, to verify that it doesn't affect the sorting - final Map supMapRack5 = genSupervisors(10, 4, 50, 0.0, 0.0, Collections.singletonMap("gpu.count", 0.0)); + // Generate some that have neither resource, to verify that the strategy will prioritize + // this last + // Also put a generic resource with 0 value in the resources list, to verify that it + // doesn't affect the sorting + final Map supMapRack5 = genSupervisors(10, 4, 50, 0.0, 0.0, + Collections.singletonMap("gpu.count", 0.0)); supMap.putAll(supMapRack0); supMap.putAll(supMapRack1); @@ -772,23 +876,28 @@ public void testMultipleRacks() { config.put(Config.TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB, Double.MAX_VALUE); INimbus iNimbus = new INimbusTest(); - //create test DNSToSwitchMapping plugin + // create test DNSToSwitchMapping plugin DNSToSwitchMapping TestNetworkTopographyPlugin = - new TestDNSToSwitchMapping(supMapRack0, supMapRack1, supMapRack2, supMapRack3, supMapRack4, supMapRack5); + new TestDNSToSwitchMapping(supMapRack0, supMapRack1, supMapRack2, supMapRack3, + supMapRack4, supMapRack5); - //generate topologies - TopologyDetails topo1 = genTopology("topo-1", config, 8, 0, 2, 0, CURRENT_TIME - 2, 10, "user"); - TopologyDetails topo2 = genTopology("topo-2", config, 8, 0, 2, 0, CURRENT_TIME - 2, 10, "user"); + // generate topologies + TopologyDetails topo1 = genTopology("topo-1", config, 8, 0, 2, 0, CURRENT_TIME - 2, 10, + "user"); + TopologyDetails topo2 = genTopology("topo-2", config, 8, 0, 2, 0, CURRENT_TIME - 2, 10, + "user"); Topologies topologies = new Topologies(topo1, topo2); - Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config); + Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), + supMap, new HashMap<>(), topologies, config); List supHostnames = new LinkedList<>(); for (SupervisorDetails sup : supMap.values()) { supHostnames.add(sup.getHost()); } Map> rackToNodes = new HashMap<>(); - Map resolvedSuperVisors = TestNetworkTopographyPlugin.resolve(supHostnames); + Map resolvedSuperVisors = TestNetworkTopographyPlugin + .resolve(supHostnames); for (Map.Entry entry : resolvedSuperVisors.entrySet()) { String hostName = entry.getKey(); String rack = entry.getValue(); @@ -799,7 +908,8 @@ public void testMultipleRacks() { DefaultResourceAwareStrategyOld rs = new DefaultResourceAwareStrategyOld(); rs.prepareForScheduling(cluster, topo1); - INodeSorter nodeSorter = new NodeSorterHostProximity(cluster, topo1, BaseResourceAwareStrategy.NodeSortType.DEFAULT_RAS); + INodeSorter nodeSorter = new NodeSorterHostProximity(cluster, topo1, + BaseResourceAwareStrategy.NodeSortType.DEFAULT_RAS); nodeSorter.prepare(null); Iterable sortedRacks = nodeSorter.getSortedRacks(); @@ -812,22 +922,24 @@ public void testMultipleRacks() { assertEquals("rack-4", it.next().id, "rack-4 should be ordered third"); // Ranked fourth since rack-3 has alot of memory but not cpu assertEquals("rack-3", it.next().id, "rack-3 should be ordered fourth"); - //Ranked fifth since rack-2 has not cpu resources + // Ranked fifth since rack-2 has not cpu resources assertEquals("rack-2", it.next().id, "rack-2 should be ordered fifth"); - //Ranked last since rack-5 has neither CPU nor memory available + // Ranked last since rack-5 has neither CPU nor memory available assertEquals("rack-5", it.next().id, "Rack-5 should be ordered sixth"); SchedulingResult schedulingResult = rs.schedule(cluster, topo1); assertTrue(schedulingResult.isSuccess(), "Scheduling failed"); - SchedulerAssignment assignment = cluster.getAssignmentById(topo1.getId()); - for (WorkerSlot ws : assignment.getSlotToExecutors().keySet()) { - //make sure all workers on scheduled in rack-0 - assertEquals("rack-0", - resolvedSuperVisors.get(rs.idToNode(ws.getNodeId()).getHostname()), "assert worker scheduled on rack-0"); + SchedulerAssignment assignment = cluster.getAssignmentById(topo1.getId()); + for (WorkerSlot ws : assignment.getSlotToExecutors().keySet()) { + // make sure all workers on scheduled in rack-0 + assertEquals("rack-0", + resolvedSuperVisors.get(rs.idToNode(ws.getNodeId()) + .getHostname()), "assert worker scheduled on rack-0"); } - assertEquals(0, cluster.getUnassignedExecutors(topo1).size(), "All executors in topo-1 scheduled"); + assertEquals(0, cluster.getUnassignedExecutors(topo1).size(), + "All executors in topo-1 scheduled"); - //Test if topology is already partially scheduled on one rack + // Test if topology is already partially scheduled on one rack Iterator executorIterator = topo2.getExecutors().iterator(); List nodeHostnames = rackToNodes.get("rack-1"); for (int i = 0; i < topo2.getExecutors().size() / 2; i++) { @@ -843,35 +955,39 @@ public void testMultipleRacks() { // schedule topo2 schedulingResult = rs.schedule(cluster, topo2); assertTrue(schedulingResult.isSuccess(), "Scheduling failed"); - assignment = cluster.getAssignmentById(topo2.getId()); - for (WorkerSlot ws : assignment.getSlotToExecutors().keySet()) { - //make sure all workers on scheduled in rack-1 - assertEquals("rack-1", - resolvedSuperVisors.get(rs.idToNode(ws.getNodeId()).getHostname()), "assert worker scheduled on rack-1"); + assignment = cluster.getAssignmentById(topo2.getId()); + for (WorkerSlot ws : assignment.getSlotToExecutors().keySet()) { + // make sure all workers on scheduled in rack-1 + assertEquals("rack-1", + resolvedSuperVisors.get(rs.idToNode(ws.getNodeId()) + .getHostname()), "assert worker scheduled on rack-1"); } - assertEquals(0, cluster.getUnassignedExecutors(topo2).size(), "All executors in topo-2 scheduled"); + assertEquals(0, cluster.getUnassignedExecutors(topo2).size(), + "All executors in topo-2 scheduled"); } } /** - * Test whether strategy will choose correct rack + * Test whether strategy will choose correct rack. */ @Test public void testMultipleRacksWithFavoritism() { for (Class strategyClass : strategyClasses) { final Map supMap = new HashMap<>(); final Map supMapRack0 = genSupervisors(10, 4, 0, 400, 8000); - //generate another rack of supervisors with less resources + // generate another rack of supervisors with less resources final Map supMapRack1 = genSupervisors(10, 4, 10, 200, 4000); - //generate some supervisors that are depleted of one resource + // generate some supervisors that are depleted of one resource final Map supMapRack2 = genSupervisors(10, 4, 20, 0, 8000); - //generate some that has alot of memory but little of cpu - final Map supMapRack3 = genSupervisors(10, 4, 30, 10, 8000 * 2 + 4000); + // generate some that has alot of memory but little of cpu + final Map supMapRack3 = genSupervisors(10, 4, 30, 10, + 8000 * 2 + 4000); - //generate some that has alot of cpu but little of memory - final Map supMapRack4 = genSupervisors(10, 4, 40, 400 + 200 + 10, 1000); + // generate some that has alot of cpu but little of memory + final Map supMapRack4 = genSupervisors(10, 4, 40, 400 + 200 + + 10, 1000); supMap.putAll(supMapRack0); supMap.putAll(supMapRack1); @@ -883,9 +999,10 @@ public void testMultipleRacksWithFavoritism() { config.put(Config.TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB, Double.MAX_VALUE); INimbus iNimbus = new INimbusTest(); - //create test DNSToSwitchMapping plugin + // create test DNSToSwitchMapping plugin DNSToSwitchMapping TestNetworkTopographyPlugin = - new TestDNSToSwitchMapping(supMapRack0, supMapRack1, supMapRack2, supMapRack3, supMapRack4); + new TestDNSToSwitchMapping(supMapRack0, supMapRack1, supMapRack2, supMapRack3, + supMapRack4); Config t1Conf = new Config(); t1Conf.putAll(config); @@ -893,25 +1010,31 @@ public void testMultipleRacksWithFavoritism() { t1Conf.put(Config.TOPOLOGY_SCHEDULER_FAVORED_NODES, t1FavoredHostNames); final List t1UnfavoredHostIds = Arrays.asList("host-1", "host-2", "host-3"); t1Conf.put(Config.TOPOLOGY_SCHEDULER_UNFAVORED_NODES, t1UnfavoredHostIds); - //generate topologies - TopologyDetails topo1 = genTopology("topo-1", t1Conf, 8, 0, 2, 0, CURRENT_TIME - 2, 10, "user"); + // generate topologies + TopologyDetails topo1 = genTopology("topo-1", t1Conf, 8, 0, 2, 0, CURRENT_TIME - 2, 10, + "user"); Config t2Conf = new Config(); t2Conf.putAll(config); - t2Conf.put(Config.TOPOLOGY_SCHEDULER_FAVORED_NODES, Arrays.asList("host-31", "host-32", "host-33")); - t2Conf.put(Config.TOPOLOGY_SCHEDULER_UNFAVORED_NODES, Arrays.asList("host-11", "host-12", "host-13")); - TopologyDetails topo2 = genTopology("topo-2", t2Conf, 8, 0, 2, 0, CURRENT_TIME - 2, 10, "user"); + t2Conf.put(Config.TOPOLOGY_SCHEDULER_FAVORED_NODES, Arrays.asList("host-31", "host-32", + "host-33")); + t2Conf.put(Config.TOPOLOGY_SCHEDULER_UNFAVORED_NODES, Arrays.asList("host-11", + "host-12", "host-13")); + TopologyDetails topo2 = genTopology("topo-2", t2Conf, 8, 0, 2, 0, CURRENT_TIME - 2, 10, + "user"); Topologies topologies = new Topologies(topo1, topo2); - Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config); + Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), + supMap, new HashMap<>(), topologies, config); List supHostnames = new LinkedList<>(); for (SupervisorDetails sup : supMap.values()) { supHostnames.add(sup.getHost()); } Map> rackToNodes = new HashMap<>(); - Map resolvedSuperVisors = TestNetworkTopographyPlugin.resolve(supHostnames); + Map resolvedSuperVisors = TestNetworkTopographyPlugin + .resolve(supHostnames); for (Map.Entry entry : resolvedSuperVisors.entrySet()) { String hostName = entry.getKey(); String rack = entry.getValue(); @@ -927,7 +1050,8 @@ public void testMultipleRacksWithFavoritism() { DefaultResourceAwareStrategyOld rs = new DefaultResourceAwareStrategyOld(); rs.prepareForScheduling(cluster, topo1); - INodeSorter nodeSorter = new NodeSorterHostProximity(cluster, topo1, BaseResourceAwareStrategy.NodeSortType.DEFAULT_RAS); + INodeSorter nodeSorter = new NodeSorterHostProximity(cluster, topo1, + BaseResourceAwareStrategy.NodeSortType.DEFAULT_RAS); nodeSorter.prepare(null); Iterable sortedRacks = nodeSorter.getSortedRacks(); @@ -940,23 +1064,25 @@ public void testMultipleRacksWithFavoritism() { assertEquals("rack-4", it.next().id, "rack-4 should be ordered third"); // Ranked fourth since rack-3 has alot of memory but not cpu assertEquals("rack-3", it.next().id, "rack-3 should be ordered fourth"); - //Ranked last since rack-2 has not cpu resources + // Ranked last since rack-2 has not cpu resources assertEquals("rack-2", it.next().id, "rack-2 should be ordered fifth"); SchedulingResult schedulingResult = rs.schedule(cluster, topo1); assertTrue(schedulingResult.isSuccess(), "Scheduling failed"); - SchedulerAssignment assignment = cluster.getAssignmentById(topo1.getId()); - for (WorkerSlot ws : assignment.getSlotToExecutors().keySet()) { - String hostName = rs.idToNode(ws.getNodeId()).getHostname(); - String rackId = resolvedSuperVisors.get(hostName); - assertTrue(t1FavoredHostNames.contains(hostName) || "rack-0".equals(rackId), - ws + " is neither on a favored node " + t1FavoredHostNames + " nor the highest priority rack (rack-0)"); + SchedulerAssignment assignment = cluster.getAssignmentById(topo1.getId()); + for (WorkerSlot ws : assignment.getSlotToExecutors().keySet()) { + String hostName = rs.idToNode(ws.getNodeId()).getHostname(); + String rackId = resolvedSuperVisors.get(hostName); + assertTrue(t1FavoredHostNames.contains(hostName) || "rack-0".equals(rackId), + ws + " is neither on a favored node " + t1FavoredHostNames + + " nor the highest priority rack (rack-0)"); assertFalse(t1UnfavoredHostIds.contains(hostName), ws + " is a part of an unfavored node " + t1UnfavoredHostIds); } - assertEquals(0, cluster.getUnassignedExecutors(topo1).size(), "All executors in topo-1 scheduled"); + assertEquals(0, cluster.getUnassignedExecutors(topo1).size(), + "All executors in topo-1 scheduled"); - //Test if topology is already partially scheduled on one rack + // Test if topology is already partially scheduled on one rack Iterator executorIterator = topo2.getExecutors().iterator(); List nodeHostnames = rackToNodes.get("rack-1"); for (int i = 0; i < topo2.getExecutors().size() / 2; i++) { @@ -972,14 +1098,18 @@ public void testMultipleRacksWithFavoritism() { // schedule topo2 schedulingResult = rs.schedule(cluster, topo2); assertTrue(schedulingResult.isSuccess(), "Scheduling failed"); - assignment = cluster.getAssignmentById(topo2.getId()); - for (WorkerSlot ws : assignment.getSlotToExecutors().keySet()) { - //make sure all workers on scheduled in rack-1 - // The favored nodes would have put it on a different rack, but because that rack does not have free space to run the - // topology it falls back to this rack - assertEquals("rack-1", resolvedSuperVisors.get(rs.idToNode(ws.getNodeId()).getHostname()), "assert worker scheduled on rack-1"); + assignment = cluster.getAssignmentById(topo2.getId()); + for (WorkerSlot ws : assignment.getSlotToExecutors().keySet()) { + // make sure all workers on scheduled in rack-1 + // The favored nodes would have put it on a different rack, but because that rack + // does + // not have free space to run the + // topology it falls back to this rack + assertEquals("rack-1", resolvedSuperVisors.get(rs.idToNode(ws.getNodeId()) + .getHostname()), "assert worker scheduled on rack-1"); } - assertEquals(0, cluster.getUnassignedExecutors(topo2).size(), "All executors in topo-2 scheduled"); + assertEquals(0, cluster.getUnassignedExecutors(topo2).size(), + "All executors in topo-2 scheduled"); } } } diff --git a/storm-server/src/test/java/org/apache/storm/scheduler/resource/strategies/scheduling/TestGenericResourceAwareStrategy.java b/storm-server/src/test/java/org/apache/storm/scheduler/resource/strategies/scheduling/TestGenericResourceAwareStrategy.java index abdd9ae8661..82f64df6e84 100644 --- a/storm-server/src/test/java/org/apache/storm/scheduler/resource/strategies/scheduling/TestGenericResourceAwareStrategy.java +++ b/storm-server/src/test/java/org/apache/storm/scheduler/resource/strategies/scheduling/TestGenericResourceAwareStrategy.java @@ -18,13 +18,31 @@ package org.apache.storm.scheduler.resource.strategies.scheduling; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.INimbusTest; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.TestBolt; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.TestSpout; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.assertTopologiesFullyScheduled; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.assertTopologiesNotScheduled; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.createGrasClusterConfig; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.genExecsAndComps; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.genSupervisors; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.genSupervisorsWithRacks; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.genTopology; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.supervisorIdToRackName; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.topoToTopologyDetails; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.topologyBuilder; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; -import java.util.Map; import java.util.Map.Entry; +import java.util.Map; import java.util.Set; import java.util.TreeMap; import java.util.concurrent.atomic.AtomicLong; @@ -35,6 +53,7 @@ import org.apache.storm.generated.InvalidTopologyException; import org.apache.storm.generated.StormTopology; import org.apache.storm.generated.WorkerResources; +import org.apache.storm.metric.StormMetricsRegistry; import org.apache.storm.scheduler.Cluster; import org.apache.storm.scheduler.ExecutorDetails; import org.apache.storm.scheduler.INimbus; @@ -47,6 +66,7 @@ import org.apache.storm.scheduler.WorkerSlot; import org.apache.storm.scheduler.resource.ResourceAwareScheduler; import org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler; +import org.apache.storm.scheduler.resource.normalization.ResourceMetrics; import org.apache.storm.topology.SharedOffHeapWithinNode; import org.apache.storm.topology.SharedOffHeapWithinWorker; import org.apache.storm.topology.SharedOnHeap; @@ -59,17 +79,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.*; -import static org.junit.jupiter.api.Assertions.*; - -import org.apache.storm.metric.StormMetricsRegistry; -import org.apache.storm.scheduler.resource.normalization.ResourceMetrics; - public class TestGenericResourceAwareStrategy { - private static final Logger LOG = LoggerFactory.getLogger(TestGenericResourceAwareStrategy.class); + private static final Logger LOG = LoggerFactory + .getLogger(TestGenericResourceAwareStrategy.class); private static final Class[] strategyClasses = { - GenericResourceAwareStrategy.class, - GenericResourceAwareStrategyOld.class, + GenericResourceAwareStrategy.class, + GenericResourceAwareStrategyOld.class, }; private final int currentTime = 1450418597; @@ -83,19 +98,21 @@ public void cleanup() { } } - private Config createGrasClusterConfig(Class strategyClass, double compPcore, double compOnHeap, double compOffHeap, + private Config createGrasClusterConfig(Class strategyClass, double compPcore, double compOnHeap, + double compOffHeap, Map> pools, Map genericResourceMap) { - Config config = TestUtilsForResourceAwareScheduler.createGrasClusterConfig(compPcore, compOnHeap, compOffHeap, pools, genericResourceMap); + Config config = TestUtilsForResourceAwareScheduler.createGrasClusterConfig(compPcore, + compOnHeap, compOffHeap, pools, genericResourceMap); config.put(Config.TOPOLOGY_SCHEDULER_STRATEGY, strategyClass.getName()); return config; } /** - * test if the scheduling logic for the GenericResourceAwareStrategy is correct. + * Test if the scheduling logic for the GenericResourceAwareStrategy is correct. */ @Test public void testGenericResourceAwareStrategySharedMemory() { - for (Class strategyClass: strategyClasses) { + for (Class strategyClass : strategyClasses) { int spoutParallelism = 2; int boltParallelism = 2; int numBolts = 3; @@ -109,21 +126,27 @@ public void testGenericResourceAwareStrategySharedMemory() { builder.setSpout("spout", new TestSpout(), spoutParallelism).addResource("gpu.count", 1.0); builder.setBolt("bolt-1", new TestBolt(), - boltParallelism).addSharedMemory(new SharedOffHeapWithinWorker(sharedOffHeapWorker, "bolt-1 shared off heap worker")).shuffleGrouping("spout"); + boltParallelism) + .addSharedMemory(new SharedOffHeapWithinWorker(sharedOffHeapWorker, + "bolt-1 shared off heap worker")).shuffleGrouping("spout"); builder.setBolt("bolt-2", new TestBolt(), - boltParallelism).addSharedMemory(new SharedOffHeapWithinNode(sharedOffHeapNode, "bolt-2 shared node")).shuffleGrouping("bolt-1"); + boltParallelism).addSharedMemory(new SharedOffHeapWithinNode(sharedOffHeapNode, + "bolt-2 shared node")).shuffleGrouping("bolt-1"); builder.setBolt("bolt-3", new TestBolt(), - boltParallelism).addSharedMemory(new SharedOnHeap(sharedOnHeap, "bolt-3 shared worker")).shuffleGrouping("bolt-2"); + boltParallelism).addSharedMemory(new SharedOnHeap(sharedOnHeap, + "bolt-3 shared worker")).shuffleGrouping("bolt-2"); StormTopology stormTopology = builder.createTopology(); INimbus iNimbus = new INimbusTest(); - Config conf = createGrasClusterConfig(strategyClass, cpuPercent, memoryOnHeap, memoryOffHeap, null, Collections.emptyMap()); + Config conf = createGrasClusterConfig(strategyClass, cpuPercent, memoryOnHeap, + memoryOffHeap, null, Collections.emptyMap()); Map genericResourcesMap = new HashMap<>(); genericResourcesMap.put("gpu.count", 1.0); - Map supMap = genSupervisors(4, 4, 500, 2000, genericResourcesMap); + Map supMap = genSupervisors(4, 4, 500, 2000, + genericResourcesMap); conf.put(Config.TOPOLOGY_PRIORITY, 0); conf.put(Config.TOPOLOGY_NAME, "testTopology"); @@ -133,14 +156,16 @@ public void testGenericResourceAwareStrategySharedMemory() { Topologies topologies = new Topologies(topo); - Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, conf); + Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), + supMap, new HashMap<>(), topologies, conf); scheduler = new ResourceAwareScheduler(); scheduler.prepare(conf, new StormMetricsRegistry()); scheduler.schedule(topologies, cluster); - for (Entry entry : cluster.getSupervisorsResourcesMap().entrySet()) { + for (Entry entry : cluster.getSupervisorsResourcesMap() + .entrySet()) { String supervisorId = entry.getKey(); SupervisorResources resources = entry.getValue(); assertTrue(resources.getTotalCpu() >= resources.getUsedCpu(), supervisorId); @@ -151,18 +176,22 @@ public void testGenericResourceAwareStrategySharedMemory() { // But because there is only 1 GPU per node, and each of the 2 spouts needs a GPU // It has to be scheduled on at least 2 nodes, and hence 2 slots. // Because of this, all the bolts will be scheduled on a single slot with one of - // the spouts and the other spout is on its own slot. So everything that can be shared is + // the spouts and the other spout is on its own slot. So everything that can be shared + // is // shared. int totalNumberOfTasks = (spoutParallelism + (boltParallelism * numBolts)); double totalExpectedCPU = totalNumberOfTasks * cpuPercent; double totalExpectedOnHeap = (totalNumberOfTasks * memoryOnHeap) + sharedOnHeap; - double totalExpectedWorkerOffHeap = (totalNumberOfTasks * memoryOffHeap) + sharedOffHeapWorker; + double totalExpectedWorkerOffHeap = (totalNumberOfTasks * memoryOffHeap) + + sharedOffHeapWorker; SchedulerAssignment assignment = cluster.getAssignmentById(topo.getId()); Set slots = assignment.getSlots(); - Map nodeToTotalShared = assignment.getNodeIdToTotalSharedOffHeapNodeMemory(); + Map nodeToTotalShared = assignment + .getNodeIdToTotalSharedOffHeapNodeMemory(); LOG.info("NODE TO SHARED OFF HEAP {}", nodeToTotalShared); - Map scheduledResources = assignment.getScheduledResources(); + Map scheduledResources = assignment + .getScheduledResources(); assertEquals(2, slots.size()); assertEquals(2, nodeToTotalShared.size()); assertEquals(2, scheduledResources.size()); @@ -179,9 +208,12 @@ public void testGenericResourceAwareStrategySharedMemory() { assertEquals(totalExpectedCPU, totalFoundCPU, 0.01); assertEquals(totalExpectedOnHeap, totalFoundOnHeap, 0.01); assertEquals(totalExpectedWorkerOffHeap, totalFoundWorkerOffHeap, 0.01); - assertEquals(sharedOffHeapNode, nodeToTotalShared.values().stream().mapToDouble((d) -> d).sum(), 0.01); - assertEquals(sharedOnHeap, scheduledResources.values().stream().mapToDouble(WorkerResources::get_shared_mem_on_heap).sum(), 0.01); - assertEquals(sharedOffHeapWorker, scheduledResources.values().stream().mapToDouble(WorkerResources::get_shared_mem_off_heap).sum(), + assertEquals(sharedOffHeapNode, nodeToTotalShared.values().stream() + .mapToDouble((d) -> d).sum(), 0.01); + assertEquals(sharedOnHeap, scheduledResources.values().stream() + .mapToDouble(WorkerResources::get_shared_mem_on_heap).sum(), 0.01); + assertEquals(sharedOffHeapWorker, scheduledResources.values().stream() + .mapToDouble(WorkerResources::get_shared_mem_off_heap).sum(), 0.01); } } @@ -190,13 +222,14 @@ public void testGenericResourceAwareStrategySharedMemory() { * Test if the scheduling logic for the GenericResourceAwareStrategy is correct * without setting {@link Config#TOPOLOGY_ACKER_EXECUTORS}. * - * Test details refer to {@link TestDefaultResourceAwareStrategy#testDefaultResourceAwareStrategyWithoutSettingAckerExecutors(int)} + *

      Test details refer to {@link + * TestDefaultResourceAwareStrategy#testDefaultResourceAwareStrategyWithoutSettingAckerExecutors(int)} */ @ParameterizedTest @ValueSource(ints = {-1, 0, 1, 2}) public void testGenericResourceAwareStrategyWithoutSettingAckerExecutors(int numOfAckersPerWorker) throws InvalidTopologyException { - for (Class strategyClass: strategyClasses) { + for (Class strategyClass : strategyClasses) { int spoutParallelism = 1; int boltParallelism = 2; TopologyBuilder builder = new TopologyBuilder(); @@ -214,10 +247,12 @@ public void testGenericResourceAwareStrategyWithoutSettingAckerExecutors(int num INimbus iNimbus = new INimbusTest(); - Config conf = createGrasClusterConfig(strategyClass, 50, 500, 0, null, Collections.emptyMap()); + Config conf = createGrasClusterConfig(strategyClass, 50, 500, 0, null, Collections + .emptyMap()); Map genericResourcesMap = new HashMap<>(); genericResourcesMap.put("gpu.count", 2.0); - Map supMap = genSupervisors(4, 4, 200, 2000, genericResourcesMap); + Map supMap = genSupervisors(4, 4, 200, 2000, + genericResourcesMap); conf.put(Config.TOPOLOGY_PRIORITY, 0); @@ -229,30 +264,35 @@ public void testGenericResourceAwareStrategyWithoutSettingAckerExecutors(int num // but with ackers added, probably more worker will be launched. // Parameterized test on different numOfAckersPerWorker if (numOfAckersPerWorker == -1) { - // Both Config.TOPOLOGY_ACKER_EXECUTORS and Config.TOPOLOGY_RAS_ACKER_EXECUTORS_PER_WORKER are not set + // Both Config.TOPOLOGY_ACKER_EXECUTORS and + // Config.TOPOLOGY_RAS_ACKER_EXECUTORS_PER_WORKER are not set // Default will be 2 (estimate num of workers) and 1 respectively } else { conf.put(Config.TOPOLOGY_RAS_ACKER_EXECUTORS_PER_WORKER, numOfAckersPerWorker); } - int estimatedNumWorker = ServerUtils.getEstimatedWorkerCountForRasTopo(conf, stormTopology); + int estimatedNumWorker = ServerUtils.getEstimatedWorkerCountForRasTopo(conf, + stormTopology); Nimbus.setUpAckerExecutorConfigs(topoName, conf, conf, estimatedNumWorker); conf.put(Config.TOPOLOGY_ACKER_RESOURCES_ONHEAP_MEMORY_MB, 250); conf.put(Config.TOPOLOGY_ACKER_CPU_PCORE_PERCENT, 50); TopologyDetails topo = new TopologyDetails("testTopology-id", conf, stormTopology, 0, - genExecsAndComps(StormCommon.systemTopology(conf, stormTopology)), currentTime, "user"); + genExecsAndComps(StormCommon.systemTopology(conf, + stormTopology)), currentTime, "user"); Topologies topologies = new Topologies(topo); - Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, conf); + Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), + supMap, new HashMap<>(), topologies, conf); scheduler = new ResourceAwareScheduler(); scheduler.prepare(conf, new StormMetricsRegistry()); scheduler.schedule(topologies, cluster); - // We need to have 3 slots on 3 separate hosts. The topology needs 6 GPUs 3500 MB memory and 350% CPU + // We need to have 3 slots on 3 separate hosts. The topology needs 6 GPUs 3500 MB memory + // and 350% CPU // The bolt-3 instances must be on separate nodes because they each need 2 GPUs. // The bolt-2 instances must be on the same node as they each need 1 GPU // (this assumes that we are packing the components to avoid fragmentation). @@ -263,54 +303,54 @@ public void testGenericResourceAwareStrategyWithoutSettingAckerExecutors(int num HashSet> expectedScheduling = new HashSet<>(); if (numOfAckersPerWorker == -1 || numOfAckersPerWorker == 1) { expectedScheduling.add(new HashSet<>(Collections.singletonList( - new ExecutorDetails(3, 3)))); //bolt-3 - 500 MB, 50% CPU, 2 GPU - //Total 500 MB, 50% CPU, 2 - GPU -> this node has 1500 MB, 150% cpu, 0 GPU left + new ExecutorDetails(3, 3)))); // bolt-3 - 500 MB, 50% CPU, 2 GPU + // Total 500 MB, 50% CPU, 2 - GPU -> this node has 1500 MB, 150% cpu, 0 GPU left expectedScheduling.add(new HashSet<>(Arrays.asList( - new ExecutorDetails(6, 6), //bolt-2 - 500 MB, 50% CPU, 1 GPU - new ExecutorDetails(2, 2), //bolt-1 - 500 MB, 50% CPU, 0 GPU - new ExecutorDetails(5, 5), //bolt-2 - 500 MB, 50% CPU, 1 GPU - new ExecutorDetails(8, 8)))); //acker - 250 MB, 50% CPU, 0 GPU - //Total 1750 MB, 200% CPU, 2 GPU -> this node has 250 MB, 0% CPU, 0 GPU left + new ExecutorDetails(6, 6), // bolt-2 - 500 MB, 50% CPU, 1 GPU + new ExecutorDetails(2, 2), // bolt-1 - 500 MB, 50% CPU, 0 GPU + new ExecutorDetails(5, 5), // bolt-2 - 500 MB, 50% CPU, 1 GPU + new ExecutorDetails(8, 8)))); // acker - 250 MB, 50% CPU, 0 GPU + // Total 1750 MB, 200% CPU, 2 GPU -> this node has 250 MB, 0% CPU, 0 GPU left expectedScheduling.add(new HashSet<>(Arrays.asList( - new ExecutorDetails(4, 4), //bolt-3 500 MB, 50% cpu, 2 GPU - new ExecutorDetails(1, 1), //bolt-1 - 500 MB, 50% CPU, 0 GPU - new ExecutorDetails(0, 0), //Spout - 500 MB, 50% CPU, 0 GPU - new ExecutorDetails(7, 7)))); //acker - 250 MB, 50% CPU, 0 GPU - //Total 1750 MB, 200% CPU, 2 GPU -> this node has 250 MB, 0% CPU, 0 GPU left + new ExecutorDetails(4, 4), // bolt-3 500 MB, 50% cpu, 2 GPU + new ExecutorDetails(1, 1), // bolt-1 - 500 MB, 50% CPU, 0 GPU + new ExecutorDetails(0, 0), // Spout - 500 MB, 50% CPU, 0 GPU + new ExecutorDetails(7, 7)))); // acker - 250 MB, 50% CPU, 0 GPU + // Total 1750 MB, 200% CPU, 2 GPU -> this node has 250 MB, 0% CPU, 0 GPU left } else if (numOfAckersPerWorker == 0) { expectedScheduling.add(new HashSet<>(Collections.singletonList( - new ExecutorDetails(3, 3)))); //bolt-3 - 500 MB, 50% CPU, 2 GPU - //Total 500 MB, 50% CPU, 2 - GPU -> this node has 1500 MB, 150% cpu, 0 GPU left + new ExecutorDetails(3, 3)))); // bolt-3 - 500 MB, 50% CPU, 2 GPU + // Total 500 MB, 50% CPU, 2 - GPU -> this node has 1500 MB, 150% cpu, 0 GPU left expectedScheduling.add(new HashSet<>(Arrays.asList( - new ExecutorDetails(6, 6), //bolt-2 - 500 MB, 50% CPU, 1 GPU - new ExecutorDetails(2, 2), //bolt-1 - 500 MB, 50% CPU, 0 GPU - new ExecutorDetails(5, 5), //bolt-2 - 500 MB, 50% CPU, 1 GPU - new ExecutorDetails(1, 1)))); //bolt-1 - 500 MB, 50% CPU, 0 GPU - //Total 2000 MB, 200% CPU, 2 GPU -> this node has 0 MB, 0% CPU, 0 GPU left + new ExecutorDetails(6, 6), // bolt-2 - 500 MB, 50% CPU, 1 GPU + new ExecutorDetails(2, 2), // bolt-1 - 500 MB, 50% CPU, 0 GPU + new ExecutorDetails(5, 5), // bolt-2 - 500 MB, 50% CPU, 1 GPU + new ExecutorDetails(1, 1)))); // bolt-1 - 500 MB, 50% CPU, 0 GPU + // Total 2000 MB, 200% CPU, 2 GPU -> this node has 0 MB, 0% CPU, 0 GPU left expectedScheduling.add(new HashSet<>(Arrays.asList( - new ExecutorDetails(0, 0), //Spout - 500 MB, 50% CPU, 0 GPU - new ExecutorDetails(4, 4)))); //bolt-3 500 MB, 50% cpu, 2 GPU - //Total 1000 MB, 100% CPU, 2 GPU -> this node has 1000 MB, 100% CPU, 0 GPU left + new ExecutorDetails(0, 0), // Spout - 500 MB, 50% CPU, 0 GPU + new ExecutorDetails(4, 4)))); // bolt-3 500 MB, 50% cpu, 2 GPU + // Total 1000 MB, 100% CPU, 2 GPU -> this node has 1000 MB, 100% CPU, 0 GPU left } else if (numOfAckersPerWorker == 2) { expectedScheduling.add(new HashSet<>(Collections.singletonList( - new ExecutorDetails(3, 3)))); //bolt-3 - 500 MB, 50% CPU, 2 GPU - //Total 500 MB, 50% CPU, 2 - GPU -> this node has 1500 MB, 150% cpu, 0 GPU left + new ExecutorDetails(3, 3)))); // bolt-3 - 500 MB, 50% CPU, 2 GPU + // Total 500 MB, 50% CPU, 2 - GPU -> this node has 1500 MB, 150% cpu, 0 GPU left expectedScheduling.add(new HashSet<>(Arrays.asList( - new ExecutorDetails(7, 7), //acker - 250 MB, 50% CPU, 0 GPU - new ExecutorDetails(8, 8), //acker - 250 MB, 50% CPU, 0 GPU - new ExecutorDetails(6, 6), //bolt-2 - 500 MB, 50% CPU, 1 GPU - new ExecutorDetails(2, 2)))); //bolt-1 - 500 MB, 50% CPU, 0 GPU - //Total 1500 MB, 200% CPU, 2 GPU -> this node has 500 MB, 0% CPU, 0 GPU left + new ExecutorDetails(7, 7), // acker - 250 MB, 50% CPU, 0 GPU + new ExecutorDetails(8, 8), // acker - 250 MB, 50% CPU, 0 GPU + new ExecutorDetails(6, 6), // bolt-2 - 500 MB, 50% CPU, 1 GPU + new ExecutorDetails(2, 2)))); // bolt-1 - 500 MB, 50% CPU, 0 GPU + // Total 1500 MB, 200% CPU, 2 GPU -> this node has 500 MB, 0% CPU, 0 GPU left expectedScheduling.add(new HashSet<>(Arrays.asList( - new ExecutorDetails(9, 9), //acker- 250 MB, 50% CPU, 0 GPU - new ExecutorDetails(10, 10), //acker- 250 MB, 50% CPU, 0 GPU - new ExecutorDetails(1, 1), //bolt-1 - 500 MB, 50% CPU, 0 GPU - new ExecutorDetails(4, 4)))); //bolt-3 500 MB, 50% cpu, 2 GPU - //Total 1500 MB, 200% CPU, 2 GPU -> this node has 500 MB, 0% CPU, 0 GPU left + new ExecutorDetails(9, 9), // acker- 250 MB, 50% CPU, 0 GPU + new ExecutorDetails(10, 10), // acker- 250 MB, 50% CPU, 0 GPU + new ExecutorDetails(1, 1), // bolt-1 - 500 MB, 50% CPU, 0 GPU + new ExecutorDetails(4, 4)))); // bolt-3 500 MB, 50% cpu, 2 GPU + // Total 1500 MB, 200% CPU, 2 GPU -> this node has 500 MB, 0% CPU, 0 GPU left expectedScheduling.add(new HashSet<>(Arrays.asList( - new ExecutorDetails(0, 0), //Spout - 500 MB, 50% CPU, 0 GPU - new ExecutorDetails(5, 5)))); //bolt-2 - 500 MB, 50% CPU, 1 GPU - //Total 1000 MB, 100% CPU, 2 GPU -> this node has 1000 MB, 100% CPU, 0 GPU left + new ExecutorDetails(0, 0), // Spout - 500 MB, 50% CPU, 0 GPU + new ExecutorDetails(5, 5)))); // bolt-2 - 500 MB, 50% CPU, 1 GPU + // Total 1000 MB, 100% CPU, 2 GPU -> this node has 1000 MB, 100% CPU, 0 GPU left } HashSet> foundScheduling = new HashSet<>(); SchedulerAssignment assignment = cluster.getAssignmentById("testTopology-id"); @@ -326,13 +366,14 @@ public void testGenericResourceAwareStrategyWithoutSettingAckerExecutors(int num * Test if the scheduling logic for the GenericResourceAwareStrategy is correct * with setting {@link Config#TOPOLOGY_ACKER_EXECUTORS}. * - * Test details refer to {@link TestDefaultResourceAwareStrategy#testDefaultResourceAwareStrategyWithSettingAckerExecutors(int)} + *

      Test details refer to {@link + * TestDefaultResourceAwareStrategy#testDefaultResourceAwareStrategyWithSettingAckerExecutors(int)} */ @ParameterizedTest @ValueSource(ints = {-1, 0, 2, 200}) public void testGenericResourceAwareStrategyWithSettingAckerExecutors(int numOfAckersPerWorker) throws InvalidTopologyException { - for (Class strategyClass: strategyClasses) { + for (Class strategyClass : strategyClasses) { int spoutParallelism = 1; int boltParallelism = 2; TopologyBuilder builder = new TopologyBuilder(); @@ -350,10 +391,12 @@ public void testGenericResourceAwareStrategyWithSettingAckerExecutors(int numOfA INimbus iNimbus = new INimbusTest(); - Config conf = createGrasClusterConfig(strategyClass, 50, 500, 0, null, Collections.emptyMap()); + Config conf = createGrasClusterConfig(strategyClass, 50, 500, 0, null, Collections + .emptyMap()); Map genericResourcesMap = new HashMap<>(); genericResourcesMap.put("gpu.count", 2.0); - Map supMap = genSupervisors(4, 4, 200, 2000, genericResourcesMap); + Map supMap = genSupervisors(4, 4, 200, 2000, + genericResourcesMap); conf.put(Config.TOPOLOGY_PRIORITY, 0); @@ -368,24 +411,28 @@ public void testGenericResourceAwareStrategyWithSettingAckerExecutors(int numOfA conf.put(Config.TOPOLOGY_RAS_ACKER_EXECUTORS_PER_WORKER, numOfAckersPerWorker); } - int estimatedNumWorker = ServerUtils.getEstimatedWorkerCountForRasTopo(conf, stormTopology); + int estimatedNumWorker = ServerUtils.getEstimatedWorkerCountForRasTopo(conf, + stormTopology); Nimbus.setUpAckerExecutorConfigs(topoName, conf, conf, estimatedNumWorker); conf.put(Config.TOPOLOGY_ACKER_RESOURCES_ONHEAP_MEMORY_MB, 250); conf.put(Config.TOPOLOGY_ACKER_CPU_PCORE_PERCENT, 50); TopologyDetails topo = new TopologyDetails("testTopology-id", conf, stormTopology, 0, - genExecsAndComps(StormCommon.systemTopology(conf, stormTopology)), currentTime, "user"); + genExecsAndComps(StormCommon.systemTopology(conf, + stormTopology)), currentTime, "user"); Topologies topologies = new Topologies(topo); - Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, conf); + Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), + supMap, new HashMap<>(), topologies, conf); scheduler = new ResourceAwareScheduler(); scheduler.prepare(conf, new StormMetricsRegistry()); scheduler.schedule(topologies, cluster); - // We need to have 3 slots on 3 separate hosts. The topology needs 6 GPUs 3500 MB memory and 350% CPU + // We need to have 3 slots on 3 separate hosts. The topology needs 6 GPUs 3500 MB memory + // and 350% CPU // The bolt-3 instances must be on separate nodes because they each need 2 GPUs. // The bolt-2 instances must be on the same node as they each need 1 GPU // (this assumes that we are packing the components to avoid fragmentation). @@ -395,24 +442,24 @@ public void testGenericResourceAwareStrategyWithSettingAckerExecutors(int numOfA // Ackers: [[8, 8], [7, 7]] (+ [[9, 9], [10, 10]] when numOfAckersPerWorker=2) HashSet> expectedScheduling = new HashSet<>(); expectedScheduling.add(new HashSet<>(Collections.singletonList( - new ExecutorDetails(3, 3)))); //bolt-3 - 500 MB, 50% CPU, 2 GPU - //Total 500 MB, 50% CPU, 2 - GPU -> this node has 1500 MB, 150% cpu, 0 GPU left + new ExecutorDetails(3, 3)))); // bolt-3 - 500 MB, 50% CPU, 2 GPU + // Total 500 MB, 50% CPU, 2 - GPU -> this node has 1500 MB, 150% cpu, 0 GPU left expectedScheduling.add(new HashSet<>(Arrays.asList( - new ExecutorDetails(7, 7), //acker - 250 MB, 50% CPU, 0 GPU - new ExecutorDetails(8, 8), //acker - 250 MB, 50% CPU, 0 GPU - new ExecutorDetails(6, 6), //bolt-2 - 500 MB, 50% CPU, 1 GPU - new ExecutorDetails(2, 2)))); //bolt-1 - 500 MB, 50% CPU, 0 GPU - //Total 1500 MB, 200% CPU, 2 GPU -> this node has 500 MB, 0% CPU, 0 GPU left + new ExecutorDetails(7, 7), // acker - 250 MB, 50% CPU, 0 GPU + new ExecutorDetails(8, 8), // acker - 250 MB, 50% CPU, 0 GPU + new ExecutorDetails(6, 6), // bolt-2 - 500 MB, 50% CPU, 1 GPU + new ExecutorDetails(2, 2)))); // bolt-1 - 500 MB, 50% CPU, 0 GPU + // Total 1500 MB, 200% CPU, 2 GPU -> this node has 500 MB, 0% CPU, 0 GPU left expectedScheduling.add(new HashSet<>(Arrays.asList( - new ExecutorDetails(9, 9), //acker- 250 MB, 50% CPU, 0 GPU - new ExecutorDetails(10, 10), //acker- 250 MB, 50% CPU, 0 GPU - new ExecutorDetails(1, 1), //bolt-1 - 500 MB, 50% CPU, 0 GPU - new ExecutorDetails(4, 4)))); //bolt-3 500 MB, 50% cpu, 2 GPU - //Total 1500 MB, 200% CPU, 2 GPU -> this node has 500 MB, 0% CPU, 0 GPU left + new ExecutorDetails(9, 9), // acker- 250 MB, 50% CPU, 0 GPU + new ExecutorDetails(10, 10), // acker- 250 MB, 50% CPU, 0 GPU + new ExecutorDetails(1, 1), // bolt-1 - 500 MB, 50% CPU, 0 GPU + new ExecutorDetails(4, 4)))); // bolt-3 500 MB, 50% cpu, 2 GPU + // Total 1500 MB, 200% CPU, 2 GPU -> this node has 500 MB, 0% CPU, 0 GPU left expectedScheduling.add(new HashSet<>(Arrays.asList( - new ExecutorDetails(0, 0), //Spout - 500 MB, 50% CPU, 0 GPU - new ExecutorDetails(5, 5)))); //bolt-2 - 500 MB, 50% CPU, 1 GPU - //Total 1000 MB, 100% CPU, 2 GPU -> this node has 1000 MB, 100% CPU, 0 GPU left + new ExecutorDetails(0, 0), // Spout - 500 MB, 50% CPU, 0 GPU + new ExecutorDetails(5, 5)))); // bolt-2 - 500 MB, 50% CPU, 1 GPU + // Total 1000 MB, 100% CPU, 2 GPU -> this node has 1000 MB, 100% CPU, 0 GPU left HashSet> foundScheduling = new HashSet<>(); SchedulerAssignment assignment = cluster.getAssignmentById("testTopology-id"); @@ -424,10 +471,11 @@ public void testGenericResourceAwareStrategyWithSettingAckerExecutors(int numOfA } } - private TopologyDetails createTestStormTopology(StormTopology stormTopology, int priority, String name, Config conf) { + private TopologyDetails createTestStormTopology(StormTopology stormTopology, int priority, + String name, Config conf) { conf.put(Config.TOPOLOGY_PRIORITY, priority); conf.put(Config.TOPOLOGY_NAME, name); - return new TopologyDetails(name , conf, stormTopology, 0, + return new TopologyDetails(name, conf, stormTopology, 0, genExecsAndComps(stormTopology), currentTime, "user"); } @@ -436,16 +484,18 @@ private TopologyDetails createTestStormTopology(StormTopology stormTopology, int */ @Test public void testGrasRequiringEviction() { - for (Class strategyClass: strategyClasses) { + for (Class strategyClass : strategyClasses) { int spoutParallelism = 3; double cpuPercent = 10; double memoryOnHeap = 10; double memoryOffHeap = 10; - // Sufficient Cpu/Memory. But insufficient gpu to schedule all topologies (gpu1, noGpu, gpu2). + // Sufficient Cpu/Memory. But insufficient gpu to schedule all topologies (gpu1, noGpu, + // gpu2). // gpu topology (requires 3 gpu's in total) TopologyBuilder builder = new TopologyBuilder(); - builder.setSpout("spout", new TestSpout(), spoutParallelism).addResource("gpu.count", 1.0); + builder.setSpout("spout", new TestSpout(), spoutParallelism).addResource("gpu.count", + 1.0); StormTopology stormTopologyWithGpu = builder.createTopology(); // non-gpu topology @@ -453,13 +503,15 @@ public void testGrasRequiringEviction() { builder.setSpout("spout", new TestSpout(), spoutParallelism); StormTopology stormTopologyNoGpu = builder.createTopology(); - Config conf = createGrasClusterConfig(strategyClass, cpuPercent, memoryOnHeap, memoryOffHeap, null, Collections.emptyMap()); - conf.put(DaemonConfig.RESOURCE_AWARE_SCHEDULER_MAX_TOPOLOGY_SCHEDULING_ATTEMPTS, 2); // allow 1 round of evictions + Config conf = createGrasClusterConfig(strategyClass, cpuPercent, memoryOnHeap, + memoryOffHeap, null, Collections.emptyMap()); + conf.put(DaemonConfig.RESOURCE_AWARE_SCHEDULER_MAX_TOPOLOGY_SCHEDULING_ATTEMPTS, + 2); // allow 1 round of evictions String gpu1 = "hasGpu1"; String noGpu = "hasNoGpu"; String gpu2 = "hasGpu2"; - TopologyDetails topo[] = { + TopologyDetails[] topo = { createTestStormTopology(stormTopologyWithGpu, 10, gpu1, conf), createTestStormTopology(stormTopologyNoGpu, 10, noGpu, conf), createTestStormTopology(stormTopologyWithGpu, 9, gpu2, conf) @@ -468,8 +520,10 @@ public void testGrasRequiringEviction() { Map genericResourcesMap = new HashMap<>(); genericResourcesMap.put("gpu.count", 1.0); - Map supMap = genSupervisors(4, 4, 500, 2000, genericResourcesMap); - Cluster cluster = new Cluster(new INimbusTest(), new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, conf); + Map supMap = genSupervisors(4, 4, 500, 2000, + genericResourcesMap); + Cluster cluster = new Cluster(new INimbusTest(), + new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, conf); // should schedule gpu1 and noGpu successfully scheduler = new ResourceAwareScheduler(); @@ -478,9 +532,11 @@ public void testGrasRequiringEviction() { assertTopologiesFullyScheduled(cluster, strategyClass, gpu1); assertTopologiesFullyScheduled(cluster, strategyClass, noGpu); - // should evict gpu1 and noGpu topologies in order to schedule gpu2 topology; then fail to reschedule gpu1 topology; + // should evict gpu1 and noGpu topologies in order to schedule gpu2 topology; then fail + // to reschedule gpu1 topology; // then schedule noGpu topology. - // Scheduling used to ignore gpu resource when deciding when to stop evicting, and gpu2 would fail to schedule. + // Scheduling used to ignore gpu resource when deciding when to stop evicting, and gpu2 + // would fail to schedule. topologies = new Topologies(topo[0], topo[1], topo[2]); cluster = new Cluster(cluster, topologies); scheduler.schedule(topologies, cluster); @@ -491,12 +547,13 @@ public void testGrasRequiringEviction() { } /** - * test if the scheduling logic for the GenericResourceAwareStrategy (when in favor of shuffle) is correct. + * Test if the scheduling logic for the GenericResourceAwareStrategy (when in favor of shuffle) + * is correct. */ @Test public void testGenericResourceAwareStrategyInFavorOfShuffle() throws InvalidTopologyException { - for (Class strategyClass: strategyClasses) { + for (Class strategyClass : strategyClasses) { int spoutParallelism = 1; int boltParallelism = 2; TopologyBuilder builder = new TopologyBuilder(); @@ -513,10 +570,12 @@ public void testGenericResourceAwareStrategyInFavorOfShuffle() INimbus iNimbus = new INimbusTest(); - Config conf = createGrasClusterConfig(strategyClass, 50, 250, 250, null, Collections.emptyMap()); + Config conf = createGrasClusterConfig(strategyClass, 50, 250, 250, null, Collections + .emptyMap()); Map genericResourcesMap = new HashMap<>(); genericResourcesMap.put("gpu.count", 2.0); - Map supMap = genSupervisors(4, 4, 200, 2000, genericResourcesMap); + Map supMap = genSupervisors(4, 4, 200, 2000, + genericResourcesMap); conf.put(Config.TOPOLOGY_PRIORITY, 0); @@ -526,10 +585,12 @@ public void testGenericResourceAwareStrategyInFavorOfShuffle() conf.put(Config.TOPOLOGY_RAS_ORDER_EXECUTORS_BY_PROXIMITY_NEEDS, true); TopologyDetails topo = new TopologyDetails("testTopology-id", conf, stormTopology, 0, - genExecsAndComps(StormCommon.systemTopology(conf, stormTopology)), currentTime, "user"); + genExecsAndComps(StormCommon.systemTopology(conf, + stormTopology)), currentTime, "user"); Topologies topologies = new Topologies(topo); - Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, conf); + Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), + supMap, new HashMap<>(), topologies, conf); ResourceAwareScheduler rs = new ResourceAwareScheduler(); @@ -540,15 +601,17 @@ public void testGenericResourceAwareStrategyInFavorOfShuffle() HashSet> expectedScheduling = new HashSet<>(); expectedScheduling.add(new HashSet<>(Arrays.asList( - new ExecutorDetails(0, 0), //spout - new ExecutorDetails(2, 2), //bolt-1 - new ExecutorDetails(6, 6), //bolt-2 - new ExecutorDetails(7, 7)))); //acker + new ExecutorDetails(0, 0), // spout + new ExecutorDetails(2, 2), // bolt-1 + new ExecutorDetails(6, 6), // bolt-2 + new ExecutorDetails(7, 7)))); // acker expectedScheduling.add(new HashSet<>(Arrays.asList( - new ExecutorDetails(4, 4), //bolt-3 - new ExecutorDetails(1, 1)))); //bolt-1 - expectedScheduling.add(new HashSet<>(Collections.singletonList(new ExecutorDetails(5, 5)))); //bolt-2 - expectedScheduling.add(new HashSet<>(Collections.singletonList(new ExecutorDetails(3, 3)))); //bolt-3 + new ExecutorDetails(4, 4), // bolt-3 + new ExecutorDetails(1, 1)))); // bolt-1 + expectedScheduling.add(new HashSet<>(Collections.singletonList(new ExecutorDetails(5, + 5)))); // bolt-2 + expectedScheduling.add(new HashSet<>(Collections.singletonList(new ExecutorDetails(3, + 3)))); // bolt-3 HashSet> foundScheduling = new HashSet<>(); SchedulerAssignment assignment = cluster.getAssignmentById("testTopology-id"); for (Collection execs : assignment.getSlotToExecutors().values()) { @@ -560,9 +623,10 @@ public void testGenericResourceAwareStrategyInFavorOfShuffle() @Test public void testAntiAffinityWithMultipleTopologies() { - for (Class strategyClass: strategyClasses) { + for (Class strategyClass : strategyClasses) { INimbus iNimbus = new INimbusTest(); - Map supMap = genSupervisorsWithRacks(1, 40, 66, 0, 0, 4700, 226200, new HashMap<>()); + Map supMap = genSupervisorsWithRacks(1, 40, 66, 0, 0, 4700, + 226200, new HashMap<>()); HashMap extraResources = new HashMap<>(); extraResources.put("my.gpu", 1.0); supMap.putAll(genSupervisorsWithRacks(1, 40, 66, 1, 0, 4700, 226200, extraResources)); @@ -576,18 +640,20 @@ public void testAntiAffinityWithMultipleTopologies() { TopologyDetails tdSimple = genTopology("topology-simple", config, 1, 5, 100, 300, 0, 0, "user", 8192); - //Schedule the simple topology first + // Schedule the simple topology first Topologies topologies = new Topologies(tdSimple); - Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config); + Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), + supMap, new HashMap<>(), topologies, config); scheduler.schedule(topologies, cluster); TopologyBuilder builder = topologyBuilder(1, 5, 100, 300); builder.setBolt("gpu-bolt", new TestBolt(), 40) .addResource("my.gpu", 1.0) .shuffleGrouping("spout-0"); - TopologyDetails tdGpu = topoToTopologyDetails("topology-gpu", config, builder.createTopology(), 0, 0, "user", 8192); + TopologyDetails tdGpu = topoToTopologyDetails("topology-gpu", config, builder + .createTopology(), 0, 0, "user", 8192); - //Now schedule GPU but with the simple topology in place. + // Now schedule GPU but with the simple topology in place. topologies = new Topologies(tdSimple, tdGpu); cluster = new Cluster(cluster, topologies); scheduler.schedule(topologies, cluster); @@ -610,19 +676,21 @@ public void testAntiAffinityWithMultipleTopologies() { Map simpleCount = topoPerRackCount.get("topology-simple-0"); assertNotNull(simpleCount); - //Because the simple topology was scheduled first we want to be sure that it didn't put anything on + // Because the simple topology was scheduled first we want to be sure that it didn't put + // anything on // the GPU nodes. - assertEquals(1, simpleCount.size()); //Only 1 rack is in use - assertFalse(simpleCount.containsKey("r001")); //r001 is the second rack with GPUs - assertTrue(simpleCount.containsKey("r000")); //r000 is the first rack with no GPUs + assertEquals(1, simpleCount.size()); // Only 1 rack is in use + assertFalse(simpleCount.containsKey("r001")); // r001 is the second rack with GPUs + assertTrue(simpleCount.containsKey("r000")); // r000 is the first rack with no GPUs - //We don't really care too much about the scheduling of topology-gpu-0, because it was scheduled. + // We don't really care too much about the scheduling of topology-gpu-0, because it was + // scheduled. } } @Test public void testScheduleLeftOverAckers() throws Exception { - for (Class strategyClass: strategyClasses) { + for (Class strategyClass : strategyClasses) { int spoutParallelism = 1; TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("spout", new TestSpout(), spoutParallelism); @@ -631,7 +699,8 @@ public void testScheduleLeftOverAckers() throws Exception { StormTopology stormToplogy = builder.createTopology(); INimbus iNimbus = new INimbusTest(); - Config conf = createGrasClusterConfig(strategyClass, 50, 400, 0, null, Collections.emptyMap()); + Config conf = createGrasClusterConfig(strategyClass, 50, 400, 0, null, Collections + .emptyMap()); Map supMap = genSupervisors(1, 1, 100, 1100); Map tmpSupMap = genSupervisors(2, 1, 100, 400); @@ -649,10 +718,12 @@ public void testScheduleLeftOverAckers() throws Exception { conf.put(Config.TOPOLOGY_ACKER_CPU_PCORE_PERCENT, 0); TopologyDetails topo = new TopologyDetails("testTopology-id", conf, stormToplogy, 0, - genExecsAndComps(StormCommon.systemTopology(conf, stormToplogy)), currentTime, "user"); + genExecsAndComps(StormCommon.systemTopology(conf, + stormToplogy)), currentTime, "user"); Topologies topologies = new Topologies(topo); - Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, conf); + Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), + supMap, new HashMap<>(), topologies, conf); scheduler = new ResourceAwareScheduler(); @@ -660,7 +731,8 @@ public void testScheduleLeftOverAckers() throws Exception { scheduler.schedule(topologies, cluster); // First it tries to schedule spout [0, 0] with a bound acker [1, 1] to sup1 r000s000. - // However, sup2 r000s001 only has 400 on-heap mem which can not fit the left over acker [2, 2] + // However, sup2 r000s001 only has 400 on-heap mem which can not fit the left over acker + // [2, 2] // So it backtrack to [0, 0] and put it to sup2 r000s001. // Then put two ackers both as left-over ackers to sup1 r000s000. HashSet> expectedScheduling = new HashSet<>(); diff --git a/storm-server/src/test/java/org/apache/storm/scheduler/resource/strategies/scheduling/TestLargeCluster.java b/storm-server/src/test/java/org/apache/storm/scheduler/resource/strategies/scheduling/TestLargeCluster.java index ea15b2f620f..890d7878b2f 100644 --- a/storm-server/src/test/java/org/apache/storm/scheduler/resource/strategies/scheduling/TestLargeCluster.java +++ b/storm-server/src/test/java/org/apache/storm/scheduler/resource/strategies/scheduling/TestLargeCluster.java @@ -18,6 +18,26 @@ package org.apache.storm.scheduler.resource.strategies.scheduling; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.BufferedReader; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.core.config.Configurator; import org.apache.storm.Config; @@ -45,34 +65,13 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.BufferedReader; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeMap; -import java.util.concurrent.atomic.AtomicInteger; - -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - @ExtendWith({NormalizedResourcesExtension.class}) public class TestLargeCluster { private static final Logger LOG = LoggerFactory.getLogger(TestLargeCluster.class); private static final Class[] strategyClasses = { - DefaultResourceAwareStrategy.class, - RoundRobinResourceAwareStrategy.class, - GenericResourceAwareStrategy.class, + DefaultResourceAwareStrategy.class, + RoundRobinResourceAwareStrategy.class, + GenericResourceAwareStrategy.class, }; public enum TEST_CLUSTER_NAME { @@ -173,14 +172,17 @@ public static ClassLoader getContextClassLoader() { } /** - * Create an array of TopologyDetails by reading serialized files for topology and configuration in the + * Create an array of TopologyDetails by reading serialized files for topology and configuration + * in the * resource path. Skip topologies with no executors/components. * - * @param failOnParseError throw exception if there are unmatched files, otherwise ignore unmatched and read errors. + * @param failOnParseError throw exception if there are unmatched files, otherwise ignore + * unmatched and read errors. * @return An array of TopologyDetails representing resource files. * @throws Exception upon error in reading topology serialized files. */ - public static TopologyDetails[] createTopoDetailsArray(String resourcePath, boolean failOnParseError) throws Exception { + public static TopologyDetails[] createTopoDetailsArray(String resourcePath, + boolean failOnParseError) throws Exception { List topoDetailsList = new ArrayList<>(); List errors = new ArrayList<>(); List resources = getResourceFiles(resourcePath); @@ -191,7 +193,8 @@ public static TopologyDetails[] createTopoDetailsArray(String resourcePath, bool int idxOfDash = resource.lastIndexOf("-"); String nm = idxOfDash > idxOfSlash ? resource.substring(idxOfSlash + 1, idxOfDash) - : resource.substring(idxOfSlash + 1, resource.length() - COMPRESSED_SERIALIZED_TOPOLOGY_FILENAME_ENDING.length()); + : resource.substring(idxOfSlash + 1, resource + .length() - COMPRESSED_SERIALIZED_TOPOLOGY_FILENAME_ENDING.length()); if (resource.endsWith(COMPRESSED_SERIALIZED_TOPOLOGY_FILENAME_ENDING)) { codeResourceMap.put(nm, resource); } else if (resource.endsWith(COMPRESSED_SERIALIZED_CONFIG_FILENAME_ENDING)) { @@ -201,30 +204,34 @@ public static TopologyDetails[] createTopoDetailsArray(String resourcePath, bool } } String[] examinedConfParams = { - Config.TOPOLOGY_NAME, - Config.TOPOLOGY_SCHEDULER_STRATEGY, - Config.TOPOLOGY_PRIORITY, - Config.TOPOLOGY_WORKERS, - Config.TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB, - Config.TOPOLOGY_SUBMITTER_USER, - Config.TOPOLOGY_ACKER_CPU_PCORE_PERCENT, - Config.TOPOLOGY_ACKER_RESOURCES_OFFHEAP_MEMORY_MB, - Config.TOPOLOGY_ACKER_RESOURCES_ONHEAP_MEMORY_MB, + Config.TOPOLOGY_NAME, + Config.TOPOLOGY_SCHEDULER_STRATEGY, + Config.TOPOLOGY_PRIORITY, + Config.TOPOLOGY_WORKERS, + Config.TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB, + Config.TOPOLOGY_SUBMITTER_USER, + Config.TOPOLOGY_ACKER_CPU_PCORE_PERCENT, + Config.TOPOLOGY_ACKER_RESOURCES_OFFHEAP_MEMORY_MB, + Config.TOPOLOGY_ACKER_RESOURCES_ONHEAP_MEMORY_MB, }; for (String topoId : codeResourceMap.keySet()) { String codeResource = codeResourceMap.get(topoId); if (!confResourceMap.containsKey(topoId)) { - String err = String.format("Ignoring topology file %s because of missing config file for %s", codeResource, topoId); + String err = String + .format("Ignoring topology file %s because of missing config file for %s", + codeResource, topoId); errors.add(err); LOG.error(err); continue; } String confResource = confResourceMap.get(topoId); - LOG.info("Found matching topology and config files: {}, {}", codeResource, confResource); + LOG.info("Found matching topology and config files: {}, {}", codeResource, + confResource); StormTopology stormTopology; try { - stormTopology = Utils.deserialize(getResourceAsBytes(codeResource), StormTopology.class); + stormTopology = Utils.deserialize(getResourceAsBytes(codeResource), + StormTopology.class); } catch (Exception ex) { String err = String.format("Cannot read topology from resource %s", codeResource); errors.add(err); @@ -236,15 +243,17 @@ public static TopologyDetails[] createTopoDetailsArray(String resourcePath, bool try { conf = Utils.fromCompressedJsonConf(getResourceAsBytes(confResource)); } catch (RuntimeException | IOException ex) { - String err = String.format("Cannot read configuration from resource %s", confResource); + String err = String.format("Cannot read configuration from resource %s", + confResource); errors.add(err); LOG.error(err, ex); continue; } // fix 0.10 conf class names - String[] configParamsToFix = {Config.TOPOLOGY_SCHEDULER_STRATEGY, Config.STORM_NETWORK_TOPOGRAPHY_PLUGIN, - DaemonConfig.RESOURCE_AWARE_SCHEDULER_PRIORITY_STRATEGY }; - for (String configParam: configParamsToFix) { + String[] configParamsToFix = + {Config.TOPOLOGY_SCHEDULER_STRATEGY, Config.STORM_NETWORK_TOPOGRAPHY_PLUGIN, + DaemonConfig.RESOURCE_AWARE_SCHEDULER_PRIORITY_STRATEGY }; + for (String configParam : configParamsToFix) { if (!conf.containsKey(configParam)) { continue; } @@ -276,9 +285,11 @@ public static TopologyDetails[] createTopoDetailsArray(String resourcePath, bool LOG.info(sb.toString()); // topo - Map execToComp = TestUtilsForResourceAwareScheduler.genExecsAndComps(stormTopology); + Map execToComp = TestUtilsForResourceAwareScheduler + .genExecsAndComps(stormTopology); LOG.info("Topology \"{}\" spouts={}, bolts={}, execToComp size is {}", topoName, - stormTopology.get_spouts_size(), stormTopology.get_bolts_size(), execToComp.size()); + stormTopology.get_spouts_size(), stormTopology.get_bolts_size(), execToComp + .size()); if (execToComp.isEmpty()) { LOG.error("Topology \"{}\" Ignoring BAD topology with zero executors", topoName); continue; @@ -291,20 +302,22 @@ public static TopologyDetails[] createTopoDetailsArray(String resourcePath, bool topoDetailsList.add(topo); } if (!errors.isEmpty() && failOnParseError) { - throw new Exception("Unable to parse all serialized objects\n\t" + String.join("\n\t", errors)); + throw new Exception("Unable to parse all serialized objects\n\t" + String.join("\n\t", + errors)); } return topoDetailsList.toArray(new TopologyDetails[0]); } /** - * Check if the files in the resource directory are matched, can be read properly, and code/config files occur + * Check if the files in the resource directory are matched, can be read properly, and + * code/config files occur * in matched pairs. * * @throws Exception showing bad and unmatched resource files. */ @Test public void testReadSerializedTopologiesAndConfigs() throws Exception { - for (TEST_CLUSTER_NAME testClusterName: TEST_CLUSTER_NAME.values()) { + for (TEST_CLUSTER_NAME testClusterName : TEST_CLUSTER_NAME.values()) { String resourcePath = testClusterName.getResourcePath(); List resources = getResourceFiles(resourcePath); assertFalse(resources.isEmpty(), "No resource files found in " + resourcePath); @@ -353,10 +366,13 @@ private static void createAndAddOneSupervisor( private static Map createSupervisors( TEST_CLUSTER_NAME testClusterName, int reducedSupervisorsPerRack) { - Collection supervisorDistributions = SupervisorDistribution.getSupervisorDistribution(testClusterName); - Map> byRackId = SupervisorDistribution.mapByRackId(supervisorDistributions); + Collection supervisorDistributions = SupervisorDistribution + .getSupervisorDistribution(testClusterName); + Map> byRackId = SupervisorDistribution + .mapByRackId(supervisorDistributions); LOG.info("Cluster={}, Designed capacity: {}", - testClusterName.getClusterName(), SupervisorDistribution.clusterCapacity(supervisorDistributions)); + testClusterName.getClusterName(), SupervisorDistribution + .clusterCapacity(supervisorDistributions)); Map retList = new HashMap<>(); Map seenRacks = new HashMap<>(); @@ -369,12 +385,14 @@ private static Map createSupervisors( list.forEach(x -> { int supervisorCnt = x.supervisorCnt; for (int i = 0; i < supervisorCnt; i++) { - int superInRack = seenRacks.computeIfAbsent(rackId, z -> new AtomicInteger(-1)).incrementAndGet(); + int superInRack = seenRacks.computeIfAbsent(rackId, z -> new AtomicInteger(-1)) + .incrementAndGet(); int rackNum = seenRacks.size() - 1; if (superInRack >= adjustedRackSupervisorCnt) { continue; } - createAndAddOneSupervisor(rackNum, superInRack, x.cpuPercent, x.memoryMb, x.slotCnt, retList); + createAndAddOneSupervisor(rackNum, superInRack, x.cpuPercent, x.memoryMb, + x.slotCnt, retList); } }); }); @@ -382,33 +400,39 @@ private static Map createSupervisors( } /** - * Create a large cluster, read topologies and configuration from resource directory and schedule. + * Create a large cluster, read topologies and configuration from resource directory and + * schedule. * * @throws Exception upon error. */ @Test public void testLargeCluster() throws Exception { - for (Class strategyClass: strategyClasses) { + for (Class strategyClass : strategyClasses) { for (TEST_CLUSTER_NAME testClusterName : TEST_CLUSTER_NAME.values()) { LOG.info("********************************************"); - LOG.info("testLargeCluster: Start Processing cluster {} using ", testClusterName.getClusterName(), strategyClass.getName()); + LOG.info("testLargeCluster: Start Processing cluster {} using ", testClusterName + .getClusterName(), strategyClass.getName()); String resourcePath = testClusterName.getResourcePath(); Map supervisors = createSupervisors(testClusterName, 0); TopologyDetails[] topoDetailsArray = createTopoDetailsArray(resourcePath, false); - assertTrue(topoDetailsArray.length > 0, "No topologies found for cluster " + testClusterName.getClusterName()); + assertTrue(topoDetailsArray.length > 0, "No topologies found for cluster " + + testClusterName.getClusterName()); Topologies topologies = new Topologies(topoDetailsArray); Config confWithDefaultStrategy = new Config(); confWithDefaultStrategy.putAll(topoDetailsArray[0].getConf()); - confWithDefaultStrategy.put(Config.TOPOLOGY_SCHEDULER_STRATEGY, strategyClass.getName()); + confWithDefaultStrategy.put(Config.TOPOLOGY_SCHEDULER_STRATEGY, strategyClass + .getName()); confWithDefaultStrategy.put( Config.STORM_NETWORK_TOPOGRAPHY_PLUGIN, - TestUtilsForResourceAwareScheduler.GenSupervisorsDnsToSwitchMapping.class.getName()); + TestUtilsForResourceAwareScheduler.GenSupervisorsDnsToSwitchMapping.class + .getName()); INimbus iNimbus = new INimbusTest(); - Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supervisors, new HashMap<>(), + Cluster cluster = new Cluster(iNimbus, + new ResourceMetrics(new StormMetricsRegistry()), supervisors, new HashMap<>(), topologies, confWithDefaultStrategy); scheduler = new ResourceAwareScheduler(); @@ -417,17 +441,20 @@ public void testLargeCluster() throws Exception { GenericResourceAwareStrategy.class, ResourceAwareScheduler.class, Cluster.class ); - Level logLevel = Level.INFO; // switch to Level.DEBUG for verbose otherwise Level.INFO + Level logLevel = + Level.INFO; // switch to Level.DEBUG for verbose otherwise Level.INFO classesToDebug.forEach(x -> Configurator.setLevel(x.getName(), logLevel)); long startTime = System.currentTimeMillis(); scheduler.prepare(confWithDefaultStrategy, new StormMetricsRegistry()); scheduler.schedule(topologies, cluster); long endTime = System.currentTimeMillis(); LOG.info("Cluster={} Scheduling Time: {} topologies in {} seconds", - testClusterName.getClusterName(), topoDetailsArray.length, (endTime - startTime) / 1000.0); + testClusterName + .getClusterName(), topoDetailsArray.length, (endTime - startTime) / 1000.0); for (TopologyDetails td : topoDetailsArray) { - TestUtilsForResourceAwareScheduler.assertTopologiesFullyScheduled(cluster, strategyClass, td.getName()); + TestUtilsForResourceAwareScheduler.assertTopologiesFullyScheduled(cluster, + strategyClass, td.getName()); } // Remove topology and reschedule it @@ -435,18 +462,23 @@ public void testLargeCluster() throws Exception { startTime = System.currentTimeMillis(); TopologyDetails topoDetails = topoDetailsArray[i]; cluster.unassign(topoDetails.getId()); - LOG.info("Cluster={}, ({}) Removed topology {}", testClusterName.getClusterName(), i, topoDetails.getName()); + LOG.info("Cluster={}, ({}) Removed topology {}", testClusterName + .getClusterName(), i, topoDetails.getName()); IScheduler rescheduler = new ResourceAwareScheduler(); rescheduler.prepare(confWithDefaultStrategy, new StormMetricsRegistry()); rescheduler.schedule(topologies, cluster); - TestUtilsForResourceAwareScheduler.assertTopologiesFullyScheduled(cluster, strategyClass, topoDetails.getName()); + TestUtilsForResourceAwareScheduler.assertTopologiesFullyScheduled(cluster, + strategyClass, topoDetails.getName()); endTime = System.currentTimeMillis(); - LOG.info("Cluster={}, ({}) Scheduling Time: Removed topology {} and rescheduled in {} seconds", - testClusterName.getClusterName(), i, topoDetails.getName(), (endTime - startTime) / 1000.0); + LOG.info("Cluster={}, ({}) Scheduling Time: Removed topology {} and " + + "rescheduled in {} seconds", + testClusterName.getClusterName(), i, topoDetails + .getName(), (endTime - startTime) / 1000.0); } classesToDebug.forEach(x -> Configurator.setLevel(x.getName(), Level.INFO)); - LOG.info("testLargeCluster: End Processing cluster {}", testClusterName.getClusterName()); + LOG.info("testLargeCluster: End Processing cluster {}", testClusterName + .getClusterName()); LOG.info("********************************************"); } } @@ -459,7 +491,8 @@ public static class SupervisorDistribution { final int memoryMb; final int cpuPercent; - public SupervisorDistribution(int supervisorCnt, String rackId, int slotCnt, int memoryMb, int cpuPercent) { + public SupervisorDistribution(int supervisorCnt, String rackId, int slotCnt, int memoryMb, + int cpuPercent) { this.rackId = rackId; this.supervisorCnt = supervisorCnt; this.slotCnt = slotCnt; @@ -469,7 +502,8 @@ public SupervisorDistribution(int supervisorCnt, String rackId, int slotCnt, int public static Map> mapByRackId(Collection supervisors) { Map> retVal = new HashMap<>(); - supervisors.forEach(x -> retVal.computeIfAbsent(x.rackId, rackId -> new ArrayList<>()).add(x)); + supervisors.forEach(x -> retVal.computeIfAbsent(x.rackId, rackId -> new ArrayList<>()) + .add(x)); return retVal; } @@ -498,24 +532,30 @@ private static Collection getSupervisorDistribution01() int cpu = 3600; // %percent int mem = 178_000; // MB int adjustedCpu = cpu - 100; - ret.add(new SupervisorDistribution(numSupersPerRackEven, rackId, numPorts, mem, cpu)); - ret.add(new SupervisorDistribution(numSupersPerRackOdd, rackId, numPorts, mem, adjustedCpu)); + ret.add(new SupervisorDistribution(numSupersPerRackEven, rackId, numPorts, mem, + cpu)); + ret.add(new SupervisorDistribution(numSupersPerRackOdd, rackId, numPorts, mem, + adjustedCpu)); } for (int rack = 12; rack < 14; rack++) { String rackId = String.format("r%03d", rack); int cpu = 2400; // %percent int mem = 118_100; // MB int adjustedCpu = cpu - 100; - ret.add(new SupervisorDistribution(numSupersPerRackEven, rackId, numPorts, mem, cpu)); - ret.add(new SupervisorDistribution(numSupersPerRackOdd, rackId, numPorts, mem, adjustedCpu)); + ret.add(new SupervisorDistribution(numSupersPerRackEven, rackId, numPorts, mem, + cpu)); + ret.add(new SupervisorDistribution(numSupersPerRackOdd, rackId, numPorts, mem, + adjustedCpu)); } for (int rack = 14; rack < 16; rack++) { String rackId = String.format("r%03d", rack); int cpu = 1200; // %percent int mem = 42_480; // MB int adjustedCpu = cpu - 100; - ret.add(new SupervisorDistribution(numSupersPerRackEven, rackId, numPorts, mem, cpu)); - ret.add(new SupervisorDistribution(numSupersPerRackOdd, rackId, numPorts, mem, adjustedCpu)); + ret.add(new SupervisorDistribution(numSupersPerRackEven, rackId, numPorts, mem, + cpu)); + ret.add(new SupervisorDistribution(numSupersPerRackOdd, rackId, numPorts, mem, + adjustedCpu)); } return ret; } @@ -564,13 +604,14 @@ public static String clusterCapacity(Collection supervis int supervisorCnt = 0; Set racks = new HashSet<>(); - for (SupervisorDistribution x: supervisorDistributions) { + for (SupervisorDistribution x : supervisorDistributions) { memoryMb += ((long) x.supervisorCnt * x.memoryMb); cpuPercent += ((long) x.supervisorCnt * x.cpuPercent); supervisorCnt += x.supervisorCnt; racks.add(x.rackId); } - return String.format("Cluster summary: Racks=%d, Supervisors=%d, memoryMb=%d, cpuPercent=%d", + return String + .format("Cluster summary: Racks=%d, Supervisors=%d, memoryMb=%d, cpuPercent=%d", racks.size(), supervisorCnt, memoryMb, cpuPercent); } } @@ -584,7 +625,7 @@ public void prepare(Map topoConf, String schedulerLocalDir) { @Override public Collection allSlotsAvailableForScheduling(Collection existingSupervisors, Topologies topologies, Set topologiesMissingAssignments) { - //return null; + // return null; Set ret = new HashSet<>(); for (SupervisorDetails sd : existingSupervisors) { String id = sd.getId(); @@ -596,12 +637,14 @@ public Collection allSlotsAvailableForScheduling(Collection> newSlotsByTopologyId) { + public void assignSlots(Topologies topologies, Map> newSlotsByTopologyId) { } @Override - public String getHostName(Map existingSupervisors, String nodeId) { + public String getHostName(Map existingSupervisors, + String nodeId) { if (existingSupervisors.containsKey(nodeId)) { return existingSupervisors.get(nodeId).getHost(); } diff --git a/storm-server/src/test/java/org/apache/storm/scheduler/resource/strategies/scheduling/TestRoundRobinNodeSorterHostIsolation.java b/storm-server/src/test/java/org/apache/storm/scheduler/resource/strategies/scheduling/TestRoundRobinNodeSorterHostIsolation.java index 3d436a63d3f..5aedf05e4b2 100644 --- a/storm-server/src/test/java/org/apache/storm/scheduler/resource/strategies/scheduling/TestRoundRobinNodeSorterHostIsolation.java +++ b/storm-server/src/test/java/org/apache/storm/scheduler/resource/strategies/scheduling/TestRoundRobinNodeSorterHostIsolation.java @@ -18,7 +18,26 @@ package org.apache.storm.scheduler.resource.strategies.scheduling; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.INimbusTest; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.createRoundRobinClusterConfig; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.genSupervisorsWithRacksAndNuma; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.genTopology; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.supervisorIdToRackName; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + import com.google.common.collect.Sets; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; import org.apache.storm.Config; import org.apache.storm.metric.StormMetricsRegistry; import org.apache.storm.networktopography.DNSToSwitchMapping; @@ -39,35 +58,17 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; - -import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.INimbusTest; -import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.createRoundRobinClusterConfig; -import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.genSupervisorsWithRacksAndNuma; -import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.genTopology; -import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.supervisorIdToRackName; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; - @ExtendWith({NormalizedResourcesExtension.class}) public class TestRoundRobinNodeSorterHostIsolation { - private static final Logger LOG = LoggerFactory.getLogger(TestRoundRobinNodeSorterHostIsolation.class); + private static final Logger LOG = LoggerFactory + .getLogger(TestRoundRobinNodeSorterHostIsolation.class); private static final int CURRENT_TIME = 1450418597; private static final Class strategyClass = RoundRobinResourceAwareStrategy.class; private Config createClusterConfig(double compPcore, double compOnHeap, double compOffHeap, Map> pools) { - Config config = TestUtilsForResourceAwareScheduler.createClusterConfig(compPcore, compOnHeap, compOffHeap, pools); + Config config = TestUtilsForResourceAwareScheduler.createClusterConfig(compPcore, + compOnHeap, compOffHeap, pools); config.put(Config.TOPOLOGY_SCHEDULER_STRATEGY, strategyClass.getName()); return config; } @@ -106,7 +107,7 @@ public TestDNSToSwitchMapping(Collection supervisorDetailsCol Map hostToRackMap = new HashMap<>(); Map> rackToHosts = new HashMap<>(); - for (SupervisorDetails supervisorDetails: supervisorDetailsCollection) { + for (SupervisorDetails supervisorDetails : supervisorDetailsCollection) { String rackId = supervisorIdToRackName(supervisorDetails.getId()); hostToRackMap.put(supervisorDetails.getHost(), rackId); String host = supervisorDetails.getHost(); @@ -134,10 +135,10 @@ public Map> getRackToHosts() { */ private void freeSomeWorkerSlots(Cluster cluster) { Map assignmentMap = cluster.getAssignments(); - for (SchedulerAssignment schedulerAssignment: assignmentMap.values()) { + for (SchedulerAssignment schedulerAssignment : assignmentMap.values()) { int i = 0; List slotsToKill = new ArrayList<>(); - for (WorkerSlot workerSlot: schedulerAssignment.getSlots()) { + for (WorkerSlot workerSlot : schedulerAssignment.getSlots()) { i++; if (i % 5 == 0) { slotsToKill.add(workerSlot); @@ -148,8 +149,10 @@ private void freeSomeWorkerSlots(Cluster cluster) { } /** - * Test whether number of nodes is limited by {@link Config#TOPOLOGY_ISOLATED_MACHINES} by scheduling - * two topologies and verifying the number of nodes that each one occupies and are not overlapping. + * Test whether number of nodes is limited by {@link Config#TOPOLOGY_ISOLATED_MACHINES} by + * scheduling + * two topologies and verifying the number of nodes that each one occupies and are not + * overlapping. */ @Test void testTopologyIsolation() { @@ -157,8 +160,8 @@ void testTopologyIsolation() { double compPcore = 100; double compOnHeap = 775; double compOffHeap = 25; - int[] topoNumSpouts = {1,1}; - int[] topoNumBolts = {1,1}; + int[] topoNumSpouts = {1, 1}; + int[] topoNumBolts = {1, 1}; int[] topoSpoutParallelism = {100, 100}; int[] topoBoltParallelism = {200, 200}; final int numRacks = 3; @@ -168,7 +171,8 @@ void testTopologyIsolation() { final double numaResourceMultiplier = 1.0; int rackStartNum = 0; int supStartNum = 0; - long compPerRack = (topoNumSpouts[0] * topoSpoutParallelism[0] + topoNumBolts[0] * topoBoltParallelism[0] + long compPerRack = (topoNumSpouts[0] * topoSpoutParallelism[0] + + topoNumBolts[0] * topoBoltParallelism[0] + topoNumSpouts[1] * topoSpoutParallelism[1]); // enough for topo1 but not topo1+topo2 long compPerSuper = compPerRack / numSupersPerRack; double cpuPerSuper = compPcore * compPerSuper; @@ -184,9 +188,10 @@ void testTopologyIsolation() { Config[] configs = new Config[topoNames.length]; TopologyDetails[] topos = new TopologyDetails[topoNames.length]; - for (int i = 0 ; i < topoNames.length ; i++) { + for (int i = 0; i < topoNames.length; i++) { configs[i] = new Config(); - configs[i].putAll(createRoundRobinClusterConfig(compPcore, compOnHeap, compOffHeap, null, null)); + configs[i].putAll(createRoundRobinClusterConfig(compPcore, compOnHeap, compOffHeap, + null, null)); configs[i].put(Config.TOPOLOGY_ISOLATED_MACHINES, maxNodes[i]); topos[i] = genTopology(topoNames[i], configs[i], topoNumSpouts[i], topoNumBolts[i], topoSpoutParallelism[i], topoBoltParallelism[i], 0, 0, "user", topoMaxHeapSize[i]); @@ -197,32 +202,37 @@ void testTopologyIsolation() { IScheduler scheduler = new ResourceAwareScheduler(); scheduler.prepare(configs[0], new StormMetricsRegistry()); - //Schedule the topo1 topology and ensure it uses limited number of nodes + // Schedule the topo1 topology and ensure it uses limited number of nodes Topologies topologies = new Topologies(td1); - Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, configs[0]); + Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), + supMap, new HashMap<>(), topologies, configs[0]); cluster.setNetworkTopography(testDNSToSwitchMapping.getRackToHosts()); scheduler.schedule(topologies, cluster); Set assignedRacks = cluster.getAssignedRacks(topos[0].getId()); - assertEquals(2 , assignedRacks.size(), "Racks for topology=" + td1.getId() + " is " + assignedRacks); + assertEquals(2, assignedRacks.size(), "Racks for topology=" + td1.getId() + " is " + + assignedRacks); - //Now schedule GPU but with the simple topology in place. + // Now schedule GPU but with the simple topology in place. topologies = new Topologies(td1, td2); cluster = new Cluster(cluster, topologies); scheduler.schedule(topologies, cluster); assignedRacks = cluster.getAssignedRacks(td1.getId(), td2.getId()); - assertEquals(numRacks, assignedRacks.size(), "Racks for topologies=" + td1.getId() + "/" + td2.getId() + " is " + assignedRacks); + assertEquals(numRacks, assignedRacks.size(), "Racks for topologies=" + td1.getId() + "/" + + td2.getId() + " is " + assignedRacks); SchedulerAssignment[] assignments = new SchedulerAssignment[topoNames.length]; Collection[] assignmentNodes = new Collection[topoNames.length]; - for (int i = 0 ; i < topoNames.length ; i++) { + for (int i = 0; i < topoNames.length; i++) { assignments[i] = cluster.getAssignmentById(topos[i].getId()); if (assignments[i] == null) { fail("Topology " + topoNames[i] + " cannot be scheduled"); } - assignmentNodes[i] = assignments[i].getSlots().stream().map(WorkerSlot::getNodeId).collect(Collectors.toList()); - assertEquals(maxNodes[i], assignmentNodes[i].size(), "Max Nodes for " + topoNames[i] + " assignment"); + assignmentNodes[i] = assignments[i].getSlots().stream().map(WorkerSlot::getNodeId) + .collect(Collectors.toList()); + assertEquals(maxNodes[i], assignmentNodes[i].size(), "Max Nodes for " + topoNames[i] + + " assignment"); } // confirm no overlap in nodes Set nodes1 = new HashSet<>(assignmentNodes[0]); @@ -236,18 +246,23 @@ void testTopologyIsolation() { } nodes2.removeAll(nodes1); - // topo2 gets scheduled on across the two racks even if there is one rack with enough capacity + // topo2 gets scheduled on across the two racks even if there is one rack with enough + // capacity assignedRacks = cluster.getAssignedRacks(td2.getId()); - assertEquals(numRacks -1, assignedRacks.size(), "Racks for topologies=" + td2.getId() + " is " + assignedRacks); + assertEquals(numRacks - 1, assignedRacks.size(), "Racks for topologies=" + td2.getId() + + " is " + assignedRacks); - // now unassign topo2, expect only two of three racks to be in use; free some slots and reschedule topo1 some topo1 executors + // now unassign topo2, expect only two of three racks to be in use; free some slots and + // reschedule topo1 some topo1 executors cluster.unassign(td2.getId()); assignedRacks = cluster.getAssignedRacks(td2.getId()); assertEquals(0, assignedRacks.size(), - "After unassigning topology " + td2.getId() + ", racks for topology=" + td2.getId() + " is " + assignedRacks); + "After unassigning topology " + td2.getId() + ", racks for topology=" + td2.getId() + + " is " + assignedRacks); assignedRacks = cluster.getAssignedRacks(td1.getId()); assertEquals(numRacks - 1, assignedRacks.size(), - "After unassigning topology " + td2.getId() + ", racks for topology=" + td1.getId() + " is " + assignedRacks); + "After unassigning topology " + td2.getId() + ", racks for topology=" + td1.getId() + + " is " + assignedRacks); assertFalse(cluster.needsSchedulingRas(td1), "Topology " + td1.getId() + " should be fully assigned before freeing slots"); freeSomeWorkerSlots(cluster); @@ -260,6 +275,7 @@ void testTopologyIsolation() { // only two of three racks should be in use still assignedRacks = cluster.getAssignedRacks(td1.getId()); assertEquals(numRacks - 1, assignedRacks.size(), - "After reassigning topology " + td2.getId() + ", racks for topology=" + td1.getId() + " is " + assignedRacks); + "After reassigning topology " + td2.getId() + ", racks for topology=" + td1.getId() + + " is " + assignedRacks); } -} \ No newline at end of file +} diff --git a/storm-server/src/test/java/org/apache/storm/scheduler/resource/strategies/scheduling/TestTopologyAnonymizerUtils.java b/storm-server/src/test/java/org/apache/storm/scheduler/resource/strategies/scheduling/TestTopologyAnonymizerUtils.java index bd677d0d62b..1189cf7544b 100644 --- a/storm-server/src/test/java/org/apache/storm/scheduler/resource/strategies/scheduling/TestTopologyAnonymizerUtils.java +++ b/storm-server/src/test/java/org/apache/storm/scheduler/resource/strategies/scheduling/TestTopologyAnonymizerUtils.java @@ -6,10 +6,10 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - *

      - * http://www.apache.org/licenses/LICENSE-2.0 - *

      - * Unless required by applicable law or agreed to in writing, software + * + *

      http://www.apache.org/licenses/LICENSE-2.0 + * + *

      Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -18,19 +18,6 @@ package org.apache.storm.scheduler.resource.strategies.scheduling; -import org.apache.storm.Config; -import org.apache.storm.DaemonConfig; -import org.apache.storm.generated.Bolt; -import org.apache.storm.generated.GlobalStreamId; -import org.apache.storm.generated.SpoutSpec; -import org.apache.storm.generated.StormTopology; -import org.apache.storm.serialization.GzipThriftSerializationDelegate; -import org.apache.storm.utils.Utils; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.BufferedReader; import java.io.ByteArrayOutputStream; import java.io.File; @@ -43,32 +30,51 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import org.apache.storm.Config; +import org.apache.storm.DaemonConfig; +import org.apache.storm.generated.Bolt; +import org.apache.storm.generated.GlobalStreamId; +import org.apache.storm.generated.SpoutSpec; +import org.apache.storm.generated.StormTopology; +import org.apache.storm.serialization.GzipThriftSerializationDelegate; +import org.apache.storm.utils.Utils; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** - * Anonymize Serialized Topologies and Configs with the goal of taking internally developed topologies and configuration + * Anonymize Serialized Topologies and Configs with the goal of taking internally developed + * topologies and configuration * and make them publicly available for testing. * - * Assume that topologies and configurations exist in the specified resource directory with names ending in - * {@link #COMPRESSED_SERIALIZED_TOPOLOGY_FILENAME_ENDING} and {@link #COMPRESSED_SERIALIZED_CONFIG_FILENAME_ENDING} + *

      Assume that topologies and configurations exist in the specified resource directory with names + * ending in + * {@link #COMPRESSED_SERIALIZED_TOPOLOGY_FILENAME_ENDING} and {@link + * #COMPRESSED_SERIALIZED_CONFIG_FILENAME_ENDING} * respectively as they exist in blobstore. Also, when both these files exist for the same topology, * they share the same file name prefix. * - *

    • Rename topologies and its corresponding configuration (as identified by its resource name). Ensure that renamed + *
    • Rename topologies and its corresponding configuration (as identified by its resource name). + * Ensure that renamed * configuration file for a topology retains the proper linkage so that: - *

      <old-topo-name>-stormcode.ser -> <new-topo-name>-stormcode.ser

      and its old conf + *

      <old-topo-name>-stormcode.ser -> <new-topo-name>-stormcode.ser

      and its old + * conf *

      <old-topo-name>-stormconf.ser -> <new-topo-name>-stormconf.ser

      *
    • * *
    • Rename components in each of the topologies.
    • * - * The new converted resource files can be copied to a resource directory under "clusterconf" and made available for use + *

      The new converted resource files can be copied to a resource directory under "clusterconf" and + * made available for use * in TestLargeCluster class. */ public class TestTopologyAnonymizerUtils { private static final Logger LOG = LoggerFactory.getLogger(TestTopologyAnonymizerUtils.class); private static final String DEFAULT_ORIGINAL_RESOURCES_PATH = "clusterconf/ebonyred"; - private static final String DEFAULT_ANONYMIZED_RESOURCES_OUTDIR = "src/test/resources/clusterconf/largeCluster03"; + private static final String DEFAULT_ANONYMIZED_RESOURCES_OUTDIR = + "src/test/resources/clusterconf/largeCluster03"; public static final String COMPRESSED_SERIALIZED_TOPOLOGY_FILENAME_ENDING = "stormcode.ser"; public static final String COMPRESSED_SERIALIZED_CONFIG_FILENAME_ENDING = "stormconf.ser"; @@ -93,7 +99,8 @@ public void testResourceAvailability() throws Exception { } /** - * Take all compressed serialized files in {@link #originalResourcePath} and create anonymized versions or the + * Take all compressed serialized files in {@link #originalResourcePath} and create anonymized + * versions or the * topology and configuration in the {@link #outputDirPath}. * */ @@ -109,9 +116,11 @@ public void anonymizeDirectory() throws Exception { LOG.error(err); continue; } - String resType = resource.substring(resource.length() - COMPRESSED_SERIALIZED_TOPOLOGY_FILENAME_ENDING.length()); + String resType = resource.substring(resource + .length() - COMPRESSED_SERIALIZED_TOPOLOGY_FILENAME_ENDING.length()); String entryName = getEntryName( - resource.substring(0, resource.length() - COMPRESSED_SERIALIZED_TOPOLOGY_FILENAME_ENDING.length()), + resource.substring(0, resource + .length() - COMPRESSED_SERIALIZED_TOPOLOGY_FILENAME_ENDING.length()), seenTopoNameIndex); int entryNum = seenTopoNameIndex.get(entryName); String topoName = String.format("TopologyName%05d", entryNum); @@ -121,9 +130,11 @@ public void anonymizeDirectory() throws Exception { switch (resType) { case COMPRESSED_SERIALIZED_TOPOLOGY_FILENAME_ENDING: // anonymize StormTopology - LOG.info("Anonymizing Topology {} as {}, with topoId={}", resource, newResourceName, topoId); + LOG.info("Anonymizing Topology {} as {}, with topoId={}", resource, + newResourceName, topoId); StormTopology stormTopology = readAndAnonymizeTopology(resource, errs); - writeCompressedResource(newResourceName, new GzipThriftSerializationDelegate().serialize(stormTopology)); + writeCompressedResource(newResourceName, new GzipThriftSerializationDelegate() + .serialize(stormTopology)); break; case COMPRESSED_SERIALIZED_CONFIG_FILENAME_ENDING: @@ -134,13 +145,16 @@ public void anonymizeDirectory() throws Exception { break; default: - String err = String.format("Resource %s is not recognized as one of supported types", resource); + String err = String + .format("Resource %s is not recognized as one of supported types", + resource); errs.add(err); LOG.warn(err); } } if (!errs.isEmpty()) { - throw new Exception("Unable to parse all serialized objects\n\t" + String.join("\n\t", errs)); + throw new Exception("Unable to parse all serialized objects\n\t" + String.join("\n\t", + errs)); } } @@ -150,12 +164,15 @@ public void anonymizeDirectory() throws Exception { * @param resourcePath to read */ private static InputStream getResourceAsStream(String resourcePath) { - final InputStream in = Thread.currentThread().getContextClassLoader().getResourceAsStream(resourcePath); - return in == null ? ClassLoader.getSystemClassLoader().getResourceAsStream(resourcePath) : in; + final InputStream in = Thread.currentThread().getContextClassLoader() + .getResourceAsStream(resourcePath); + return in == null ? ClassLoader.getSystemClassLoader() + .getResourceAsStream(resourcePath) : in; } /** - * Get the list of serialized topology (ending with {@link #COMPRESSED_SERIALIZED_TOPOLOGY_FILENAME_ENDING} + * Get the list of serialized topology (ending with {@link + * #COMPRESSED_SERIALIZED_TOPOLOGY_FILENAME_ENDING} * and configuration (ending with {@link #COMPRESSED_SERIALIZED_TOPOLOGY_FILENAME_ENDING}) * resource files in the path. * @@ -220,7 +237,7 @@ private StormTopology readAndAnonymizeTopology(String resource, List err } Map renameMap = new HashMap<>(); - if (stormTopology.get_spouts() != null){ + if (stormTopology.get_spouts() != null) { for (String name : stormTopology.get_spouts().keySet()) { String newName = String.format("Spout-%d", renameMap.size()); renameMap.putIfAbsent(name, newName); @@ -238,7 +255,7 @@ private StormTopology readAndAnonymizeTopology(String resource, List err StormTopology retVal = stormTopology.deepCopy(); if (spoutCnt > 0) { Map spouts = retVal.get_spouts(); - for (String name: renameMap.keySet()) { + for (String name : renameMap.keySet()) { if (spouts.containsKey(name)) { spouts.put(renameMap.get(name), spouts.remove(name)); } @@ -254,7 +271,7 @@ private StormTopology readAndAnonymizeTopology(String resource, List err if (boltCnt > 0) { Map bolts = retVal.get_bolts(); - for (String name: renameMap.keySet()) { + for (String name : renameMap.keySet()) { if (bolts.containsKey(name)) { bolts.put(renameMap.get(name), bolts.remove(name)); } @@ -270,7 +287,8 @@ private StormTopology readAndAnonymizeTopology(String resource, List err return retVal; } - private Map readAndAnonymizeConfig(String confResource, String topoName, List errors) { + private Map readAndAnonymizeConfig(String confResource, String topoName, + List errors) { Map conf; try { conf = Utils.fromCompressedJsonConf(getResourceAsBytes(confResource)); @@ -298,7 +316,8 @@ private Map readAndAnonymizeConfig(String confResource, String t return conf; } - private void writeCompressedResource(String newResourceName, byte[] compressedBytes) throws IOException { + private void writeCompressedResource(String newResourceName, + byte[] compressedBytes) throws IOException { File dir = new File(outputDirPath); if (!dir.exists()) { dir.mkdirs(); @@ -313,10 +332,13 @@ private void writeCompressedResource(String newResourceName, byte[] compressedBy * In order to create resources as part of a test run: *

    • Download compressed topologies and configurations (from blobstore) into resource path * {@link #DEFAULT_ORIGINAL_RESOURCES_PATH}. The resource names must end with either - * {@link #COMPRESSED_SERIALIZED_TOPOLOGY_FILENAME_ENDING} or {@link #COMPRESSED_SERIALIZED_CONFIG_FILENAME_ENDING}
    • - *
    • Change pathnames for {@link #DEFAULT_ORIGINAL_RESOURCES_PATH} and {@link #DEFAULT_ANONYMIZED_RESOURCES_OUTDIR}
    • + * {@link #COMPRESSED_SERIALIZED_TOPOLOGY_FILENAME_ENDING} or {@link + * #COMPRESSED_SERIALIZED_CONFIG_FILENAME_ENDING} + *
    • Change pathnames for {@link #DEFAULT_ORIGINAL_RESOURCES_PATH} and {@link + * #DEFAULT_ANONYMIZED_RESOURCES_OUTDIR}
    • *
    • Uncomment annotation so that this method is executed as a test
    • - *
    • add files in {@link #DEFAULT_ANONYMIZED_RESOURCES_OUTDIR} to the resource path "clusterconf/new-cluster-name"
    • + *
    • add files in {@link #DEFAULT_ANONYMIZED_RESOURCES_OUTDIR} to the resource path + * "clusterconf/new-cluster-name"
    • *
    • use TestLargeCluster to test these newly generated files after changing * {@link TestLargeCluster.TEST_CLUSTER_NAME} to "new-cluster-name"
    • * @@ -330,7 +352,8 @@ public void testAnonymizer() throws Exception { instance.outputDirPath = args[1]; instance.testResourceAvailability(); instance.anonymizeDirectory(); - LOG.info("Read resources in {} and wrote anonymized files to {}", instance.originalResourcePath, instance.outputDirPath); + LOG.info("Read resources in {} and wrote anonymized files to {}", + instance.originalResourcePath, instance.outputDirPath); } public static void main(String[] args) { @@ -338,7 +361,8 @@ public static void main(String[] args) { args = new String[]{DEFAULT_ORIGINAL_RESOURCES_PATH, DEFAULT_ANONYMIZED_RESOURCES_OUTDIR}; } if (args.length != 2) { - LOG.error("Expecting two arguments , received {} args", args.length); + LOG.error("Expecting two arguments , received {} args", + args.length); System.exit(-1); } diff --git a/storm-server/src/test/java/org/apache/storm/scheduler/resource/strategies/scheduling/sorter/TestNodeSorterHostProximity.java b/storm-server/src/test/java/org/apache/storm/scheduler/resource/strategies/scheduling/sorter/TestNodeSorterHostProximity.java index 50e50326fe2..675d3a155a1 100644 --- a/storm-server/src/test/java/org/apache/storm/scheduler/resource/strategies/scheduling/sorter/TestNodeSorterHostProximity.java +++ b/storm-server/src/test/java/org/apache/storm/scheduler/resource/strategies/scheduling/sorter/TestNodeSorterHostProximity.java @@ -18,6 +18,39 @@ package org.apache.storm.scheduler.resource.strategies.scheduling.sorter; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.INimbusTest; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.TestBolt; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.createClusterConfig; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.createGrasClusterConfig; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.genSupervisorsWithRacks; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.genSupervisorsWithRacksAndNuma; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.genTopology; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.supervisorIdToRackName; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.topoToTopologyDetails; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.topologyBuilder; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; import org.apache.storm.Config; import org.apache.storm.metric.StormMetricsRegistry; import org.apache.storm.networktopography.DNSToSwitchMapping; @@ -46,31 +79,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeMap; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; -import java.util.stream.Collectors; -import java.util.stream.StreamSupport; - -import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.*; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNotEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; - @ExtendWith({NormalizedResourcesExtension.class}) public class TestNodeSorterHostProximity { private static final Logger LOG = LoggerFactory.getLogger(TestNodeSorterHostProximity.class); @@ -82,8 +90,10 @@ protected Class getDefaultResourceAwareStrategyClass() { private Config createClusterConfig(double compPcore, double compOnHeap, double compOffHeap, Map> pools) { - Config config = TestUtilsForResourceAwareScheduler.createClusterConfig(compPcore, compOnHeap, compOffHeap, pools); - config.put(Config.TOPOLOGY_SCHEDULER_STRATEGY, getDefaultResourceAwareStrategyClass().getName()); + Config config = TestUtilsForResourceAwareScheduler.createClusterConfig(compPcore, + compOnHeap, compOffHeap, pools); + config.put(Config.TOPOLOGY_SCHEDULER_STRATEGY, getDefaultResourceAwareStrategyClass() + .getName()); return config; } @@ -121,7 +131,7 @@ public TestDNSToSwitchMapping(Collection supervisorDetailsCol Map hostToRackMap = new HashMap<>(); Map> rackToHosts = new HashMap<>(); - for (SupervisorDetails supervisorDetails: supervisorDetailsCollection) { + for (SupervisorDetails supervisorDetails : supervisorDetailsCollection) { String rackId = supervisorIdToRackName(supervisorDetails.getId()); hostToRackMap.put(supervisorDetails.getHost(), rackId); String host = supervisorDetails.getHost(); @@ -162,32 +172,34 @@ public void testMultipleRacks() { numRacks, numSupersPerRack, numZonesPerHost, numPortsPerSuper, rackStartNum++, supStartNum, 400, 8000, Collections.emptyMap(), numaResourceMultiplier); - //generate another rack of supervisors with less resources + // generate another rack of supervisors with less resources supStartNum += numSupersPerRack; final Map supMapRack1 = genSupervisorsWithRacksAndNuma( numRacks, numSupersPerRack, numZonesPerHost, numPortsPerSuper, rackStartNum++, supStartNum, 200, 4000, Collections.emptyMap(), numaResourceMultiplier); - //generate some supervisors that are depleted of one resource + // generate some supervisors that are depleted of one resource supStartNum += numSupersPerRack; final Map supMapRack2 = genSupervisorsWithRacksAndNuma( numRacks, numSupersPerRack, numZonesPerHost, numPortsPerSuper, rackStartNum++, supStartNum, 0, 8000, Collections.emptyMap(), numaResourceMultiplier); - //generate some that has a lot of memory but little of cpu + // generate some that has a lot of memory but little of cpu supStartNum += numSupersPerRack; final Map supMapRack3 = genSupervisorsWithRacksAndNuma( numRacks, numSupersPerRack, numZonesPerHost, numPortsPerSuper, rackStartNum++, supStartNum, - 10, 8000 * 2 + 4000, Collections.emptyMap(),numaResourceMultiplier); + 10, 8000 * 2 + 4000, Collections.emptyMap(), numaResourceMultiplier); - //generate some that has a lot of cpu but little of memory + // generate some that has a lot of cpu but little of memory supStartNum += numSupersPerRack; final Map supMapRack4 = genSupervisorsWithRacksAndNuma( numRacks, numSupersPerRack, numZonesPerHost, numPortsPerSuper, rackStartNum++, supStartNum, 400 + 200 + 10, 1000, Collections.emptyMap(), numaResourceMultiplier); - //Generate some that have neither resource, to verify that the strategy will prioritize this last - //Also put a generic resource with 0 value in the resources list, to verify that it doesn't affect the sorting + // Generate some that have neither resource, to verify that the strategy will prioritize + // this last + // Also put a generic resource with 0 value in the resources list, to verify that it doesn't + // affect the sorting supStartNum += numSupersPerRack; final Map supMapRack5 = genSupervisorsWithRacksAndNuma( numRacks, numSupersPerRack, numZonesPerHost, numPortsPerSuper, rackStartNum++, supStartNum, @@ -204,16 +216,20 @@ public void testMultipleRacks() { config.put(Config.TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB, Double.MAX_VALUE); INimbus iNimbus = new INimbusTest(); - //create test DNSToSwitchMapping plugin + // create test DNSToSwitchMapping plugin TestDNSToSwitchMapping testDNSToSwitchMapping = - new TestDNSToSwitchMapping(supMapRack0, supMapRack1, supMapRack2, supMapRack3, supMapRack4, supMapRack5); + new TestDNSToSwitchMapping(supMapRack0, supMapRack1, supMapRack2, supMapRack3, + supMapRack4, supMapRack5); - //generate topologies - TopologyDetails topo1 = genTopology("topo-1", config, 8, 0, 2, 0, CURRENT_TIME - 2, 10, "user"); - TopologyDetails topo2 = genTopology("topo-2", config, 8, 0, 2, 0, CURRENT_TIME - 2, 10, "user"); + // generate topologies + TopologyDetails topo1 = genTopology("topo-1", config, 8, 0, 2, 0, CURRENT_TIME - 2, 10, + "user"); + TopologyDetails topo2 = genTopology("topo-2", config, 8, 0, 2, 0, CURRENT_TIME - 2, 10, + "user"); Topologies topologies = new Topologies(topo1, topo2); - Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config); + Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), + supMap, new HashMap<>(), topologies, config); List supHostnames = new LinkedList<>(); for (SupervisorDetails sup : supMap.values()) { @@ -222,25 +238,39 @@ public void testMultipleRacks() { Map> rackToHosts = testDNSToSwitchMapping.getRackToHosts(); cluster.setNetworkTopography(rackToHosts); - NodeSorterHostProximity nodeSorter = new NodeSorterHostProximity(cluster, topo1, BaseResourceAwareStrategy.NodeSortType.DEFAULT_RAS); + NodeSorterHostProximity nodeSorter = new NodeSorterHostProximity(cluster, topo1, + BaseResourceAwareStrategy.NodeSortType.DEFAULT_RAS); nodeSorter.prepare(null); - List sortedRacks = StreamSupport.stream(nodeSorter.getSortedRacks().spliterator(), false) + List sortedRacks = StreamSupport.stream(nodeSorter.getSortedRacks() + .spliterator(), false) .collect(Collectors.toList()); String rackSummaries = sortedRacks.stream() - .map(x -> String.format("Rack %s -> scheduled-cnt %d, min-avail %f, avg-avail %f, cpu %f, mem %f", - x.id, nodeSorter.getScheduledExecCntByRackId().getOrDefault(x.id, new AtomicInteger(-1)).get(), + .map(x -> String + .format("Rack %s -> scheduled-cnt %d, min-avail %f, avg-avail %f, cpu %f, mem %f", + x.id, nodeSorter.getScheduledExecCntByRackId().getOrDefault(x.id, + new AtomicInteger(-1)).get(), x.minResourcePercent, x.avgResourcePercent, x.availableResources.getTotalCpu(), x.availableResources.getTotalMemoryMb())) .collect(Collectors.joining("\n\t")); assertEquals(6, sortedRacks.size(), rackSummaries + "\n# of racks sorted"); Iterator it = sortedRacks.iterator(); - assertEquals("rack-000", it.next().id, rackSummaries + "\nrack-000 should be ordered first since it has the most balanced set of resources"); - assertEquals("rack-001", it.next().id, rackSummaries + "\nrack-001 should be ordered second since it has a balanced set of resources but less than rack-000"); - assertEquals("rack-004", it.next().id, rackSummaries + "\nrack-004 should be ordered third since it has a lot of cpu but not a lot of memory"); - assertEquals("rack-003", it.next().id, rackSummaries + "\nrack-003 should be ordered fourth since it has a lot of memory but not cpu"); - assertEquals("rack-002", it.next().id, rackSummaries + "\nrack-002 should be ordered fifth since it has not cpu resources"); - assertEquals("rack-005", it.next().id, rackSummaries + "\nRack-005 should be ordered sixth since it has neither CPU nor memory available"); + assertEquals("rack-000", it.next().id, rackSummaries + + "\nrack-000 should be ordered first since it has the most balanced set of " + + "resources"); + assertEquals("rack-001", it.next().id, rackSummaries + + "\nrack-001 should be ordered second since it has a balanced set of resources " + + "but less than rack-000"); + assertEquals("rack-004", it.next().id, rackSummaries + + "\nrack-004 should be ordered third since it has a lot of cpu but not a lot of " + + "memory"); + assertEquals("rack-003", it.next().id, rackSummaries + + "\nrack-003 should be ordered fourth since it has a lot of memory but not cpu"); + assertEquals("rack-002", it.next().id, rackSummaries + + "\nrack-002 should be ordered fifth since it has not cpu resources"); + assertEquals("rack-005", it.next().id, rackSummaries + + "\nRack-005 should be ordered sixth since it has neither CPU nor memory " + + "available"); } /** @@ -259,25 +289,25 @@ public void testMultipleRacksWithFavoritism() { numRacks, numSupersPerRack, numZonesPerHost, numPortsPerSuper, rackStartNum++, supStartNum, 400, 8000, Collections.emptyMap(), 1.0); - //generate another rack of supervisors with less resources + // generate another rack of supervisors with less resources supStartNum += numSupersPerRack; final Map supMapRack1 = genSupervisorsWithRacksAndNuma( numRacks, numSupersPerRack, numZonesPerHost, numPortsPerSuper, rackStartNum++, supStartNum, 200, 4000, Collections.emptyMap(), 1.0); - //generate some supervisors that are depleted of one resource + // generate some supervisors that are depleted of one resource supStartNum += numSupersPerRack; final Map supMapRack2 = genSupervisorsWithRacksAndNuma( numRacks, numSupersPerRack, numZonesPerHost, numPortsPerSuper, rackStartNum++, supStartNum, 0, 8000, Collections.emptyMap(), 1.0); - //generate some that has a lot of memory but little of cpu + // generate some that has a lot of memory but little of cpu supStartNum += numSupersPerRack; final Map supMapRack3 = genSupervisorsWithRacksAndNuma( numRacks, numSupersPerRack, numZonesPerHost, numPortsPerSuper, rackStartNum++, supStartNum, 10, 8000 * 2 + 4000, Collections.emptyMap(), 1.0); - //generate some that has a lot of cpu but little of memory + // generate some that has a lot of cpu but little of memory supStartNum += numSupersPerRack; final Map supMapRack4 = genSupervisorsWithRacksAndNuma( numRacks, numSupersPerRack, numZonesPerHost, numPortsPerSuper, rackStartNum++, supStartNum, @@ -293,9 +323,10 @@ public void testMultipleRacksWithFavoritism() { config.put(Config.TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB, Double.MAX_VALUE); INimbus iNimbus = new INimbusTest(); - //create test DNSToSwitchMapping plugin + // create test DNSToSwitchMapping plugin TestDNSToSwitchMapping testDNSToSwitchMapping = - new TestDNSToSwitchMapping(supMapRack0, supMapRack1, supMapRack2, supMapRack3, supMapRack4); + new TestDNSToSwitchMapping(supMapRack0, supMapRack1, supMapRack2, supMapRack3, + supMapRack4); Config t1Conf = new Config(); t1Conf.putAll(config); @@ -303,18 +334,23 @@ public void testMultipleRacksWithFavoritism() { t1Conf.put(Config.TOPOLOGY_SCHEDULER_FAVORED_NODES, t1FavoredHostNames); final List t1UnfavoredHostIds = Arrays.asList("host-1", "host-2", "host-3"); t1Conf.put(Config.TOPOLOGY_SCHEDULER_UNFAVORED_NODES, t1UnfavoredHostIds); - //generate topologies - TopologyDetails topo1 = genTopology("topo-1", t1Conf, 8, 0, 2, 0, CURRENT_TIME - 2, 10, "user"); + // generate topologies + TopologyDetails topo1 = genTopology("topo-1", t1Conf, 8, 0, 2, 0, CURRENT_TIME - 2, 10, + "user"); Config t2Conf = new Config(); t2Conf.putAll(config); - t2Conf.put(Config.TOPOLOGY_SCHEDULER_FAVORED_NODES, Arrays.asList("host-31", "host-32", "host-33")); - t2Conf.put(Config.TOPOLOGY_SCHEDULER_UNFAVORED_NODES, Arrays.asList("host-11", "host-12", "host-13")); - TopologyDetails topo2 = genTopology("topo-2", t2Conf, 8, 0, 2, 0, CURRENT_TIME - 2, 10, "user"); + t2Conf.put(Config.TOPOLOGY_SCHEDULER_FAVORED_NODES, Arrays.asList("host-31", "host-32", + "host-33")); + t2Conf.put(Config.TOPOLOGY_SCHEDULER_UNFAVORED_NODES, Arrays.asList("host-11", "host-12", + "host-13")); + TopologyDetails topo2 = genTopology("topo-2", t2Conf, 8, 0, 2, 0, CURRENT_TIME - 2, 10, + "user"); Topologies topologies = new Topologies(topo1, topo2); - Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config); + Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), + supMap, new HashMap<>(), topologies, config); List supHostnames = new LinkedList<>(); for (SupervisorDetails sup : supMap.values()) { @@ -323,13 +359,17 @@ public void testMultipleRacksWithFavoritism() { Map> rackToHosts = testDNSToSwitchMapping.getRackToHosts(); cluster.setNetworkTopography(rackToHosts); - NodeSorterHostProximity nodeSorter = new NodeSorterHostProximity(cluster, topo1, BaseResourceAwareStrategy.NodeSortType.DEFAULT_RAS); + NodeSorterHostProximity nodeSorter = new NodeSorterHostProximity(cluster, topo1, + BaseResourceAwareStrategy.NodeSortType.DEFAULT_RAS); nodeSorter.prepare(null); - List sortedRacks = StreamSupport.stream(nodeSorter.getSortedRacks().spliterator(), false) + List sortedRacks = StreamSupport.stream(nodeSorter.getSortedRacks() + .spliterator(), false) .collect(Collectors.toList()); String rackSummaries = sortedRacks.stream() - .map(x -> String.format("Rack %s -> scheduled-cnt %d, min-avail %f, avg-avail %f, cpu %f, mem %f", - x.id, nodeSorter.getScheduledExecCntByRackId().getOrDefault(x.id, new AtomicInteger(-1)).get(), + .map(x -> String + .format("Rack %s -> scheduled-cnt %d, min-avail %f, avg-avail %f, cpu %f, mem %f", + x.id, nodeSorter.getScheduledExecCntByRackId().getOrDefault(x.id, + new AtomicInteger(-1)).get(), x.minResourcePercent, x.avgResourcePercent, x.availableResources.getTotalCpu(), x.availableResources.getTotalMemoryMb())) @@ -344,13 +384,14 @@ public void testMultipleRacksWithFavoritism() { assertEquals("rack-004", it.next().id, "rack-004 should be ordered third"); // Ranked fourth since rack-3 has alot of memory but not cpu assertEquals("rack-003", it.next().id, "rack-003 should be ordered fourth"); - //Ranked last since rack-2 has not cpu resources + // Ranked last since rack-2 has not cpu resources assertEquals("rack-002", it.next().id, "rack-00s2 should be ordered fifth"); } /** * Test if hosts are presented together regardless of resource availability. - * Supervisors are created with multiple Numa zones in such a manner that resources on two numa zones on the same host + * Supervisors are created with multiple Numa zones in such a manner that resources on two numa + * zones on the same host * differ widely in resource availability. */ @Test @@ -368,25 +409,25 @@ public void testMultipleRacksWithHostProximity() { numRacks, numSupersPerRack, numZonesPerHost, numPortsPerSuper, rackStartNum++, supStartNum, 400, 8000, Collections.emptyMap(), numaResourceMultiplier); - //generate another rack of supervisors with less resources + // generate another rack of supervisors with less resources supStartNum += numSupersPerRack; final Map supMapRack1 = genSupervisorsWithRacksAndNuma( numRacks, numSupersPerRack, numZonesPerHost, numPortsPerSuper, rackStartNum++, supStartNum, 200, 4000, Collections.emptyMap(), numaResourceMultiplier); - //generate some supervisors that are depleted of one resource + // generate some supervisors that are depleted of one resource supStartNum += numSupersPerRack; final Map supMapRack2 = genSupervisorsWithRacksAndNuma( numRacks, numSupersPerRack, numZonesPerHost, numPortsPerSuper, rackStartNum++, supStartNum, 0, 8000, Collections.emptyMap(), numaResourceMultiplier); - //generate some that has a lot of memory but little of cpu + // generate some that has a lot of memory but little of cpu supStartNum += numSupersPerRack; final Map supMapRack3 = genSupervisorsWithRacksAndNuma( numRacks, numSupersPerRack, numZonesPerHost, numPortsPerSuper, rackStartNum++, supStartNum, - 10, 8000 * 2 + 4000, Collections.emptyMap(),numaResourceMultiplier); + 10, 8000 * 2 + 4000, Collections.emptyMap(), numaResourceMultiplier); - //generate some that has a lot of cpu but little of memory + // generate some that has a lot of cpu but little of memory supStartNum += numSupersPerRack; final Map supMapRack4 = genSupervisorsWithRacksAndNuma( numRacks, numSupersPerRack, numZonesPerHost, numPortsPerSuper, rackStartNum++, supStartNum, @@ -402,9 +443,10 @@ public void testMultipleRacksWithHostProximity() { config.put(Config.TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB, Double.MAX_VALUE); INimbus iNimbus = new INimbusTest(); - //create test DNSToSwitchMapping plugin + // create test DNSToSwitchMapping plugin TestDNSToSwitchMapping testDNSToSwitchMapping = - new TestDNSToSwitchMapping(supMapRack0, supMapRack1, supMapRack2, supMapRack3, supMapRack4); + new TestDNSToSwitchMapping(supMapRack0, supMapRack1, supMapRack2, supMapRack3, + supMapRack4); Config t1Conf = new Config(); t1Conf.putAll(config); @@ -412,18 +454,23 @@ public void testMultipleRacksWithHostProximity() { t1Conf.put(Config.TOPOLOGY_SCHEDULER_FAVORED_NODES, t1FavoredHostNames); final List t1UnfavoredHostIds = Arrays.asList("host-1", "host-2", "host-3"); t1Conf.put(Config.TOPOLOGY_SCHEDULER_UNFAVORED_NODES, t1UnfavoredHostIds); - //generate topologies - TopologyDetails topo1 = genTopology("topo-1", t1Conf, 8, 0, 2, 0, CURRENT_TIME - 2, 10, "user"); + // generate topologies + TopologyDetails topo1 = genTopology("topo-1", t1Conf, 8, 0, 2, 0, CURRENT_TIME - 2, 10, + "user"); Config t2Conf = new Config(); t2Conf.putAll(config); - t2Conf.put(Config.TOPOLOGY_SCHEDULER_FAVORED_NODES, Arrays.asList("host-31", "host-32", "host-33")); - t2Conf.put(Config.TOPOLOGY_SCHEDULER_UNFAVORED_NODES, Arrays.asList("host-11", "host-12", "host-13")); - TopologyDetails topo2 = genTopology("topo-2", t2Conf, 8, 0, 2, 0, CURRENT_TIME - 2, 10, "user"); + t2Conf.put(Config.TOPOLOGY_SCHEDULER_FAVORED_NODES, Arrays.asList("host-31", "host-32", + "host-33")); + t2Conf.put(Config.TOPOLOGY_SCHEDULER_UNFAVORED_NODES, Arrays.asList("host-11", "host-12", + "host-13")); + TopologyDetails topo2 = genTopology("topo-2", t2Conf, 8, 0, 2, 0, CURRENT_TIME - 2, 10, + "user"); Topologies topologies = new Topologies(topo1, topo2); - Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config); + Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), + supMap, new HashMap<>(), topologies, config); cluster.setNetworkTopography(testDNSToSwitchMapping.getRackToHosts()); @@ -434,11 +481,12 @@ public void testMultipleRacksWithHostProximity() { String prevHost = null; List errLines = new ArrayList(); Map nodeToHost = new RasNodes(cluster).getNodeIdToHostname(); - for (String nodeId: nodeSorter.sortAllNodes()) { + for (String nodeId : nodeSorter.sortAllNodes()) { String host = nodeToHost.getOrDefault(nodeId, "no-host-for-node-" + nodeId); errLines.add(String.format("\tnodeId:%s, host:%s", nodeId, host)); if (!host.equals(prevHost) && seenHosts.contains(host)) { - String err = String.format("Host %s for node %s is out of order:\n\t%s", host, nodeId, String.join("\n\t", errLines)); + String err = String.format("Host %s for node %s is out of order:\n\t%s", host, + nodeId, String.join("\n\t", errLines)); fail(err); } seenHosts.add(host); @@ -477,7 +525,7 @@ public void testMultipleRacksOrderedByCapacity() { supStartNum += numSupersPerRack; final Map supMapRack3 = genSupervisorsWithRacksAndNuma( numRacks, numSupersPerRack, numZonesPerHost, numPortsPerSuper, rackStartNum++, supStartNum, - 300, 8000 - rackStartNum, Collections.emptyMap(),numaResourceMultiplier); + 300, 8000 - rackStartNum, Collections.emptyMap(), numaResourceMultiplier); supStartNum += numSupersPerRack; final Map supMapRack4 = genSupervisorsWithRacksAndNuma( @@ -488,7 +536,8 @@ public void testMultipleRacksOrderedByCapacity() { supStartNum += numSupersPerRack; final Map supMapRack5 = genSupervisorsWithRacksAndNuma( numRacks, numSupersPerRack, numZonesPerHost, numPortsPerSuper, rackStartNum++, supStartNum, - 100, 8000 - rackStartNum, Collections.singletonMap("gpu.count", 0.0), numaResourceMultiplier); + 100, 8000 - rackStartNum, Collections.singletonMap("gpu.count", + 0.0), numaResourceMultiplier); supMap.putAll(supMapRack0); supMap.putAll(supMapRack1); @@ -501,40 +550,54 @@ public void testMultipleRacksOrderedByCapacity() { config.put(Config.TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB, Double.MAX_VALUE); INimbus iNimbus = new INimbusTest(); - //create test DNSToSwitchMapping plugin + // create test DNSToSwitchMapping plugin TestDNSToSwitchMapping testDNSToSwitchMapping = - new TestDNSToSwitchMapping(supMapRack0, supMapRack1, supMapRack2, supMapRack3, supMapRack4, supMapRack5); + new TestDNSToSwitchMapping(supMapRack0, supMapRack1, supMapRack2, supMapRack3, + supMapRack4, supMapRack5); - //generate topologies - TopologyDetails topo1 = genTopology("topo-1", config, 8, 0, 2, 0, CURRENT_TIME - 2, 10, "user"); - TopologyDetails topo2 = genTopology("topo-2", config, 8, 0, 2, 0, CURRENT_TIME - 2, 10, "user"); + // generate topologies + TopologyDetails topo1 = genTopology("topo-1", config, 8, 0, 2, 0, CURRENT_TIME - 2, 10, + "user"); + TopologyDetails topo2 = genTopology("topo-2", config, 8, 0, 2, 0, CURRENT_TIME - 2, 10, + "user"); Topologies topologies = new Topologies(topo1, topo2); - Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config); + Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), + supMap, new HashMap<>(), topologies, config); cluster.setNetworkTopography(testDNSToSwitchMapping.getRackToHosts()); NodeSorterHostProximity nodeSorter = new NodeSorterHostProximity(cluster, topo1); nodeSorter.prepare(null); - List sortedRacks = StreamSupport.stream(nodeSorter.getSortedRacks().spliterator(), false) + List sortedRacks = StreamSupport.stream(nodeSorter.getSortedRacks() + .spliterator(), false) .collect(Collectors.toList()); String rackSummaries = sortedRacks .stream() - .map(x -> String.format("Rack %s -> scheduled-cnt %d, min-avail %f, avg-avail %f, cpu %f, mem %f", - x.id, nodeSorter.getScheduledExecCntByRackId().getOrDefault(x.id, new AtomicInteger(-1)).get(), + .map(x -> String + .format("Rack %s -> scheduled-cnt %d, min-avail %f, avg-avail %f, cpu %f, mem %f", + x.id, nodeSorter.getScheduledExecCntByRackId().getOrDefault(x.id, + new AtomicInteger(-1)).get(), x.minResourcePercent, x.avgResourcePercent, x.availableResources.getTotalCpu(), x.availableResources.getTotalMemoryMb())) .collect(Collectors.joining("\n\t")); NormalizedResourceRequest topoResourceRequest = topo1.getApproximateTotalResources(); - String topoRequest = String.format("Topo %s, approx-requested-resources %s", topo1.getId(), topoResourceRequest.toString()); + String topoRequest = String.format("Topo %s, approx-requested-resources %s", topo1.getId(), + topoResourceRequest.toString()); Iterator it = sortedRacks.iterator(); - assertEquals("rack-000", it.next().id, topoRequest + "\n\t" + rackSummaries + "\nRack-000 should be ordered first since it has the largest capacity"); - assertEquals("rack-001", it.next().id, topoRequest + "\n\t" + rackSummaries + "\nrack-001 should be ordered second since it smaller than rack-000"); - assertEquals("rack-002", it.next().id, topoRequest + "\n\t" + rackSummaries + "\nrack-002 should be ordered third since it is smaller than rack-001"); - assertEquals("rack-003", it.next().id, topoRequest + "\n\t" + rackSummaries + "\nrack-003 should be ordered fourth since it since it is smaller than rack-002"); - assertEquals("rack-004", it.next().id, topoRequest + "\n\t" + rackSummaries + "\nrack-004 should be ordered fifth since it since it is smaller than rack-003"); - assertEquals("rack-005", it.next().id, topoRequest + "\n\t" + rackSummaries + "\nrack-005 should be ordered last since it since it is has smallest capacity"); + assertEquals("rack-000", it.next().id, topoRequest + "\n\t" + rackSummaries + + "\nRack-000 should be ordered first since it has the largest capacity"); + assertEquals("rack-001", it.next().id, topoRequest + "\n\t" + rackSummaries + + "\nrack-001 should be ordered second since it smaller than rack-000"); + assertEquals("rack-002", it.next().id, topoRequest + "\n\t" + rackSummaries + + "\nrack-002 should be ordered third since it is smaller than rack-001"); + assertEquals("rack-003", it.next().id, topoRequest + "\n\t" + rackSummaries + + "\nrack-003 should be ordered fourth since it since it is smaller than rack-002"); + assertEquals("rack-004", it.next().id, topoRequest + "\n\t" + rackSummaries + + "\nrack-004 should be ordered fifth since it since it is smaller than rack-003"); + assertEquals("rack-005", it.next().id, topoRequest + "\n\t" + rackSummaries + + "\nrack-005 should be ordered last since it since it is has smallest capacity"); } /** @@ -545,7 +608,8 @@ public void testMultipleRacksOrderedByCapacity() { @Test public void testAntiAffinityWithMultipleTopologies() { INimbus iNimbus = new INimbusTest(); - Map supMap = genSupervisorsWithRacks(1, 40, 66, 0, 0, 4700, 226200, new HashMap<>()); + Map supMap = genSupervisorsWithRacks(1, 40, 66, 0, 0, 4700, + 226200, new HashMap<>()); HashMap extraResources = new HashMap<>(); extraResources.put("my.gpu", 1.0); supMap.putAll(genSupervisorsWithRacks(1, 40, 66, 1, 0, 4700, 226200, extraResources)); @@ -559,9 +623,10 @@ public void testAntiAffinityWithMultipleTopologies() { TopologyDetails tdSimple = genTopology("topology-simple", config, 1, 5, 100, 300, 0, 0, "user", 8192); - //Schedule the simple topology first + // Schedule the simple topology first Topologies topologies = new Topologies(tdSimple); - Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config); + Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), + supMap, new HashMap<>(), topologies, config); { NodeSorterHostProximity nodeSorter = new NodeSorterHostProximity(cluster, tdSimple); @@ -572,17 +637,24 @@ public void testAntiAffinityWithMultipleTopologies() { .collect(Collectors.toList()); String rackSummaries = StreamSupport .stream(sortedRacks.spliterator(), false) - .map(x -> String.format("Rack %s -> scheduled-cnt %d, min-avail %f, avg-avail %f, cpu %f, mem %f", - x.id, nodeSorter.getScheduledExecCntByRackId().getOrDefault(x.id, new AtomicInteger(-1)).get(), + .map(x -> String + .format("Rack %s -> scheduled-cnt %d, min-avail %f, avg-avail %f, cpu %f, " + + "mem %f", + x.id, nodeSorter.getScheduledExecCntByRackId().getOrDefault(x.id, + new AtomicInteger(-1)).get(), x.minResourcePercent, x.avgResourcePercent, x.availableResources.getTotalCpu(), x.availableResources.getTotalMemoryMb())) .collect(Collectors.joining("\n\t")); - NormalizedResourceRequest topoResourceRequest = tdSimple.getApproximateTotalResources(); - String topoRequest = String.format("Topo %s, approx-requested-resources %s", tdSimple.getId(), topoResourceRequest.toString()); + NormalizedResourceRequest topoResourceRequest = tdSimple + .getApproximateTotalResources(); + String topoRequest = String.format("Topo %s, approx-requested-resources %s", + tdSimple.getId(), topoResourceRequest.toString()); assertEquals(2, sortedRacks.size(), rackSummaries + "\n# of racks sorted"); - assertEquals("rack-000", sortedRacks.get(0).id, rackSummaries + "\nFirst rack sorted"); - assertEquals("rack-001", sortedRacks.get(1).id, rackSummaries + "\nSecond rack sorted"); + assertEquals("rack-000", sortedRacks.get(0).id, rackSummaries + + "\nFirst rack sorted"); + assertEquals("rack-001", sortedRacks.get(1).id, rackSummaries + + "\nSecond rack sorted"); } } @@ -592,9 +664,10 @@ public void testAntiAffinityWithMultipleTopologies() { builder.setBolt("gpu-bolt", new TestBolt(), 40) .addResource("my.gpu", 1.0) .shuffleGrouping("spout-0"); - TopologyDetails tdGpu = topoToTopologyDetails("topology-gpu", config, builder.createTopology(), 0, 0,"user", 8192); + TopologyDetails tdGpu = topoToTopologyDetails("topology-gpu", config, builder + .createTopology(), 0, 0, "user", 8192); - //Now schedule GPU but with the simple topology in place. + // Now schedule GPU but with the simple topology in place. topologies = new Topologies(tdSimple, tdGpu); cluster = new Cluster(cluster, topologies); { @@ -603,23 +676,33 @@ public void testAntiAffinityWithMultipleTopologies() { String comp = tdGpu.getComponentFromExecutor(exec); nodeSorter.prepare(exec); List sortedRacks = StreamSupport - .stream(nodeSorter.getSortedRacks().spliterator(), false).collect(Collectors.toList()); + .stream(nodeSorter.getSortedRacks().spliterator(), false).collect(Collectors + .toList()); String rackSummaries = sortedRacks.stream() - .map(x -> String.format("Rack %s -> scheduled-cnt %d, min-avail %f, avg-avail %f, cpu %f, mem %f", - x.id, nodeSorter.getScheduledExecCntByRackId().getOrDefault(x.id, new AtomicInteger(-1)).get(), + .map(x -> String + .format("Rack %s -> scheduled-cnt %d, min-avail %f, avg-avail %f, cpu %f, " + + "mem %f", + x.id, nodeSorter.getScheduledExecCntByRackId().getOrDefault(x.id, + new AtomicInteger(-1)).get(), x.minResourcePercent, x.avgResourcePercent, x.availableResources.getTotalCpu(), x.availableResources.getTotalMemoryMb())) .collect(Collectors.joining("\n\t")); - NormalizedResourceRequest topoResourceRequest = tdSimple.getApproximateTotalResources(); - String topoRequest = String.format("Topo %s, approx-requested-resources %s", tdSimple.getId(), topoResourceRequest.toString()); + NormalizedResourceRequest topoResourceRequest = tdSimple + .getApproximateTotalResources(); + String topoRequest = String.format("Topo %s, approx-requested-resources %s", + tdSimple.getId(), topoResourceRequest.toString()); assertEquals(2, sortedRacks.size(), rackSummaries + "\n# of racks sorted"); if (comp.equals("gpu-bolt")) { - assertEquals("rack-001", sortedRacks.get(0).id, rackSummaries + "\nFirst rack sorted for " + comp); - assertEquals("rack-000", sortedRacks.get(1).id, rackSummaries + "\nSecond rack sorted for " + comp); + assertEquals("rack-001", sortedRacks.get(0).id, rackSummaries + + "\nFirst rack sorted for " + comp); + assertEquals("rack-000", sortedRacks.get(1).id, rackSummaries + + "\nSecond rack sorted for " + comp); } else { - assertEquals("rack-000", sortedRacks.get(0).id, rackSummaries + "\nFirst rack sorted for " + comp); - assertEquals("rack-001", sortedRacks.get(1).id, rackSummaries + "\nSecond rack sorted for " + comp); + assertEquals("rack-000", sortedRacks.get(0).id, rackSummaries + + "\nFirst rack sorted for " + comp); + assertEquals("rack-001", sortedRacks.get(1).id, rackSummaries + + "\nSecond rack sorted for " + comp); } } } @@ -630,7 +713,7 @@ public void testAntiAffinityWithMultipleTopologies() { assertEquals(2, assignments.size()); Map> topoPerRackCount = new HashMap<>(); - for (Map.Entry entry: assignments.entrySet()) { + for (Map.Entry entry : assignments.entrySet()) { SchedulerAssignment sa = entry.getValue(); Map slotsPerRack = new TreeMap<>(); for (WorkerSlot slot : sa.getSlots()) { @@ -644,13 +727,15 @@ public void testAntiAffinityWithMultipleTopologies() { Map simpleCount = topoPerRackCount.get("topology-simple-0"); assertNotNull(simpleCount); - //Because the simple topology was scheduled first we want to be sure that it didn't put anything on + // Because the simple topology was scheduled first we want to be sure that it didn't put + // anything on // the GPU nodes. - assertEquals(1, simpleCount.size()); //Only 1 rack is in use - assertFalse(simpleCount.containsKey("r001")); //r001 is the second rack with GPUs - assertTrue(simpleCount.containsKey("r000")); //r000 is the first rack with no GPUs + assertEquals(1, simpleCount.size()); // Only 1 rack is in use + assertFalse(simpleCount.containsKey("r001")); // r001 is the second rack with GPUs + assertTrue(simpleCount.containsKey("r000")); // r000 is the first rack with no GPUs - //We don't really care too much about the scheduling of topology-gpu-0, because it was scheduled. + // We don't really care too much about the scheduling of topology-gpu-0, because it was + // scheduled. } /** @@ -658,10 +743,10 @@ public void testAntiAffinityWithMultipleTopologies() { */ private void freeSomeWorkerSlots(Cluster cluster) { Map assignmentMap = cluster.getAssignments(); - for (SchedulerAssignment schedulerAssignment: assignmentMap.values()) { + for (SchedulerAssignment schedulerAssignment : assignmentMap.values()) { int i = 0; List slotsToKill = new ArrayList<>(); - for (WorkerSlot workerSlot: schedulerAssignment.getSlots()) { + for (WorkerSlot workerSlot : schedulerAssignment.getSlots()) { i++; if (i % 5 == 0) { slotsToKill.add(workerSlot); @@ -672,7 +757,8 @@ private void freeSomeWorkerSlots(Cluster cluster) { } /** - * If the topology is too large for one rack, it should be partially scheduled onto the next rack (and next rack only). + * If the topology is too large for one rack, it should be partially scheduled onto the next + * rack (and next rack only). */ @Test public void testFillUpRackAndSpilloverToNextRack() { @@ -691,7 +777,8 @@ public void testFillUpRackAndSpilloverToNextRack() { final double numaResourceMultiplier = 1.0; int rackStartNum = 0; int supStartNum = 0; - long compPerRack = (topo1NumSpouts * topo1SpoutParallelism + topo1NumBolts * topo1BoltParallelism) * 4/5; // not enough for topo1 + long compPerRack = (topo1NumSpouts * topo1SpoutParallelism + + topo1NumBolts * topo1BoltParallelism) * 4 / 5; // not enough for topo1 long compPerSuper = compPerRack / numSupersPerRack; double cpuPerSuper = compPcore * compPerSuper; double memPerSuper = (compOnHeap + compOffHeap) * compPerSuper; @@ -705,7 +792,8 @@ public void testFillUpRackAndSpilloverToNextRack() { Config config = new Config(); config.putAll(createGrasClusterConfig(compPcore, compOnHeap, compOffHeap, null, null)); - config.put(Config.TOPOLOGY_SCHEDULER_STRATEGY, GenericResourceAwareStrategy.class.getName()); + config.put(Config.TOPOLOGY_SCHEDULER_STRATEGY, GenericResourceAwareStrategy.class + .getName()); IScheduler scheduler = new ResourceAwareScheduler(); scheduler.prepare(config, new StormMetricsRegistry()); @@ -713,18 +801,21 @@ public void testFillUpRackAndSpilloverToNextRack() { TopologyDetails td1 = genTopology(topoName1, config, topo1NumSpouts, topo1NumBolts, topo1SpoutParallelism, topo1BoltParallelism, 0, 0, "user", topo1MaxHeapSize); - //Schedule the topo1 topology and ensure it fits on 2 racks + // Schedule the topo1 topology and ensure it fits on 2 racks Topologies topologies = new Topologies(td1); - Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config); + Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), + supMap, new HashMap<>(), topologies, config); cluster.setNetworkTopography(testDNSToSwitchMapping.getRackToHosts()); scheduler.schedule(topologies, cluster); Set assignedRacks = cluster.getAssignedRacks(td1.getId()); - assertEquals(2, assignedRacks.size(), "Racks for topology=" + td1.getId() + " is " + assignedRacks); + assertEquals(2, assignedRacks.size(), "Racks for topology=" + td1.getId() + " is " + + assignedRacks); } /** - * Rack with low resources should be used to schedule an executor if it has other executors for the same topology. + * Rack with low resources should be used to schedule an executor if it has other executors for + * the same topology. *
    • Schedule topo1 on one rack
    • *
    • unassign some executors
    • *
    • schedule another topology to partially fill up rack1
    • @@ -752,7 +843,8 @@ public void testPreferRackWithTopoExecutors() { final double numaResourceMultiplier = 1.0; int rackStartNum = 0; int supStartNum = 0; - long compPerRack = (topo1NumSpouts * topo1SpoutParallelism + topo1NumBolts * topo1BoltParallelism + long compPerRack = (topo1NumSpouts * topo1SpoutParallelism + + topo1NumBolts * topo1BoltParallelism + topo2NumSpouts * topo2SpoutParallelism); // enough for topo1 but not topo1+topo2 long compPerSuper = compPerRack / numSupersPerRack; double cpuPerSuper = compPcore * compPerSuper; @@ -769,7 +861,8 @@ public void testPreferRackWithTopoExecutors() { Config config = new Config(); config.putAll(createGrasClusterConfig(compPcore, compOnHeap, compOffHeap, null, null)); - config.put(Config.TOPOLOGY_SCHEDULER_STRATEGY, GenericResourceAwareStrategy.class.getName()); + config.put(Config.TOPOLOGY_SCHEDULER_STRATEGY, GenericResourceAwareStrategy.class + .getName()); IScheduler scheduler = new ResourceAwareScheduler(); scheduler.prepare(config, new StormMetricsRegistry()); @@ -777,38 +870,47 @@ public void testPreferRackWithTopoExecutors() { TopologyDetails td1 = genTopology(topoName1, config, topo1NumSpouts, topo1NumBolts, topo1SpoutParallelism, topo1BoltParallelism, 0, 0, "user", topo1MaxHeapSize); - //Schedule the topo1 topology and ensure it fits on 1 rack + // Schedule the topo1 topology and ensure it fits on 1 rack Topologies topologies = new Topologies(td1); - Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config); + Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), + supMap, new HashMap<>(), topologies, config); cluster.setNetworkTopography(testDNSToSwitchMapping.getRackToHosts()); scheduler.schedule(topologies, cluster); Set assignedRacks = cluster.getAssignedRacks(td1.getId()); - assertEquals(1, assignedRacks.size(), "Racks for topology=" + td1.getId() + " is " + assignedRacks); + assertEquals(1, assignedRacks.size(), "Racks for topology=" + td1.getId() + " is " + + assignedRacks); - TopologyBuilder builder = topologyBuilder(topo2NumSpouts, topo2NumBolts, topo2SpoutParallelism, topo2BoltParallelism); - TopologyDetails td2 = topoToTopologyDetails(topoName2, config, builder.createTopology(), 0, 0,"user", topo2MaxHeapSize); + TopologyBuilder builder = topologyBuilder(topo2NumSpouts, topo2NumBolts, + topo2SpoutParallelism, topo2BoltParallelism); + TopologyDetails td2 = topoToTopologyDetails(topoName2, config, builder.createTopology(), 0, + 0, "user", topo2MaxHeapSize); - //Now schedule GPU but with the simple topology in place. + // Now schedule GPU but with the simple topology in place. topologies = new Topologies(td1, td2); cluster = new Cluster(cluster, topologies); scheduler.schedule(topologies, cluster); assignedRacks = cluster.getAssignedRacks(td1.getId(), td2.getId()); - assertEquals(2, assignedRacks.size(), "Racks for topologies=" + td1.getId() + "/" + td2.getId() + " is " + assignedRacks); + assertEquals(2, assignedRacks.size(), "Racks for topologies=" + td1.getId() + "/" + td2 + .getId() + " is " + assignedRacks); // topo2 gets scheduled on its own rack because it is empty and available assignedRacks = cluster.getAssignedRacks(td2.getId()); - assertEquals(1, assignedRacks.size(), "Racks for topologies=" + td2.getId() + " is " + assignedRacks); + assertEquals(1, assignedRacks.size(), "Racks for topologies=" + td2.getId() + " is " + + assignedRacks); - // now unassign topo2, expect only one rack to be in use; free some slots and reschedule topo1 some topo1 executors + // now unassign topo2, expect only one rack to be in use; free some slots and reschedule + // topo1 some topo1 executors cluster.unassign(td2.getId()); assignedRacks = cluster.getAssignedRacks(td2.getId()); assertEquals(0, assignedRacks.size(), - "After unassigning topology " + td2.getId() + ", racks for topology=" + td2.getId() + " is " + assignedRacks); + "After unassigning topology " + td2.getId() + ", racks for topology=" + td2.getId() + + " is " + assignedRacks); assignedRacks = cluster.getAssignedRacks(td1.getId()); assertEquals(1, assignedRacks.size(), - "After unassigning topology " + td2.getId() + ", racks for topology=" + td1.getId() + " is " + assignedRacks); + "After unassigning topology " + td2.getId() + ", racks for topology=" + td1.getId() + + " is " + assignedRacks); assertFalse(cluster.needsSchedulingRas(td1), "Topology " + td1.getId() + " should be fully assigned before freeing slots"); freeSomeWorkerSlots(cluster); @@ -821,18 +923,22 @@ public void testPreferRackWithTopoExecutors() { // only one rack should be in use by topology1 assignedRacks = cluster.getAssignedRacks(td1.getId()); assertEquals(1, assignedRacks.size(), - "After reassigning topology " + td2.getId() + ", racks for topology=" + td1.getId() + " is " + assignedRacks); + "After reassigning topology " + td2.getId() + ", racks for topology=" + td1.getId() + + " is " + assignedRacks); } /** * Assign and then clear out a rack to host list mapping in cluster.networkTopography. * Expected behavior is that: - *
    • the rack without hosts does not show up in {@link NodeSorterHostProximity#getSortedRacks()}
    • - *
    • all the supervisor nodes still get returned in {@link NodeSorterHostProximity#sortAllNodes()} ()}
    • + *
    • the rack without hosts does not show up in {@link + * NodeSorterHostProximity#getSortedRacks()}
    • + *
    • all the supervisor nodes still get returned in {@link + * NodeSorterHostProximity#sortAllNodes()} ()}
    • *
    • supervisors on cleared rack show up under {@link DNSToSwitchMapping#DEFAULT_RACK}
    • * - *

      - * Force an usual condition, where one of the racks is still passed to LazyNodeSortingIterator with + *

      Force an usual condition, where one of the racks is still passed to + * LazyNodeSortingIterator + * with * an empty list and then ensure that code is resilient. *

      */ @@ -848,7 +954,8 @@ void testWithImpairedClusterNetworkTopography() { int topo1BoltParallelism = 200; final int numSupersPerRack = 10; final int numPortsPerSuper = 66; - long compPerRack = (topo1NumSpouts * topo1SpoutParallelism + topo1NumBolts * topo1BoltParallelism + 10); + long compPerRack = (topo1NumSpouts * topo1SpoutParallelism + + topo1NumBolts * topo1BoltParallelism + 10); long compPerSuper = compPerRack / numSupersPerRack; double cpuPerSuper = compPcore * compPerSuper; double memPerSuper = (compOnHeap + compOffHeap) * compPerSuper; @@ -856,13 +963,15 @@ void testWithImpairedClusterNetworkTopography() { final String topoName1 = "topology1"; int numRacks = 3; - Map supMap = genSupervisorsWithRacks(numRacks, numSupersPerRack, numPortsPerSuper, + Map supMap = genSupervisorsWithRacks(numRacks, numSupersPerRack, + numPortsPerSuper, 0, 0, cpuPerSuper, memPerSuper, new HashMap<>()); TestDNSToSwitchMapping testDNSToSwitchMapping = new TestDNSToSwitchMapping(supMap.values()); Config config = new Config(); config.putAll(createGrasClusterConfig(compPcore, compOnHeap, compOffHeap, null, null)); - config.put(Config.TOPOLOGY_SCHEDULER_STRATEGY, GenericResourceAwareStrategy.class.getName()); + config.put(Config.TOPOLOGY_SCHEDULER_STRATEGY, GenericResourceAwareStrategy.class + .getName()); IScheduler scheduler = new ResourceAwareScheduler(); scheduler.prepare(config, new StormMetricsRegistry()); @@ -871,51 +980,63 @@ void testWithImpairedClusterNetworkTopography() { topo1NumBolts, topo1SpoutParallelism, topo1BoltParallelism, 0, 0, "user", topo1MaxHeapSize); Topologies topologies = new Topologies(td1); - Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config); + Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), + supMap, new HashMap<>(), topologies, config); cluster.setNetworkTopography(testDNSToSwitchMapping.getRackToHosts()); Map> networkTopography = cluster.getNetworkTopography(); - assertEquals(numRacks, networkTopography.size(), "Expecting " + numRacks + " racks found " + networkTopography.size()); - assertTrue(networkTopography.size() >= 3, "Expecting racks count to be >= 3, found " + networkTopography.size()); - - // Impair cluster.networkTopography and set one rack to have zero hosts, getSortedRacks should exclude this rack. - // Keep, the supervisorDetails unchanged - confirm that these nodes are not lost even with incomplete networkTopography + assertEquals(numRacks, networkTopography.size(), "Expecting " + numRacks + " racks found " + + networkTopography.size()); + assertTrue(networkTopography.size() >= 3, "Expecting racks count to be >= 3, found " + + networkTopography.size()); + + // Impair cluster.networkTopography and set one rack to have zero hosts, getSortedRacks + // should exclude this rack. + // Keep, the supervisorDetails unchanged - confirm that these nodes are not lost even with + // incomplete networkTopography String rackIdToZero = networkTopography.keySet().stream().findFirst().get(); impairClusterRack(cluster, rackIdToZero, true, false); NodeSorterHostProximity nodeSorterHostProximity = new NodeSorterHostProximity(cluster, td1); nodeSorterHostProximity.getSortedRacks().forEach(x -> assertNotEquals(x.id, rackIdToZero)); - // confirm that the above action has not lost the hosts and that they appear under the DEFAULT rack + // confirm that the above action has not lost the hosts and that they appear under the + // DEFAULT rack { Set seenRacks = new HashSet<>(); nodeSorterHostProximity.getSortedRacks().forEach(x -> seenRacks.add(x.id)); assertEquals(numRacks, seenRacks.size(), "Expecting rack cnt to be still " + numRacks); assertTrue(seenRacks.contains(DNSToSwitchMapping.DEFAULT_RACK), - "Expecting to see default-rack=" + DNSToSwitchMapping.DEFAULT_RACK + " in sortedRacks"); + "Expecting to see default-rack=" + DNSToSwitchMapping.DEFAULT_RACK + + " in sortedRacks"); } // now check if node/supervisor is missing when sorting all nodes Set expectedNodes = supMap.keySet(); Set seenNodes = new HashSet<>(); nodeSorterHostProximity.prepare(null); - nodeSorterHostProximity.sortAllNodes().forEach( n -> seenNodes.add(n)); + nodeSorterHostProximity.sortAllNodes().forEach(n -> seenNodes.add(n)); assertEquals(expectedNodes, seenNodes, "Expecting see all supervisors "); // Now fully impair the cluster - confirm no default rack { - cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config); - cluster.setNetworkTopography(new TestDNSToSwitchMapping(supMap.values()).getRackToHosts()); + cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, + new HashMap<>(), topologies, config); + cluster.setNetworkTopography(new TestDNSToSwitchMapping(supMap.values()) + .getRackToHosts()); impairClusterRack(cluster, rackIdToZero, true, true); Set seenRacks = new HashSet<>(); - NodeSorterHostProximity nodeSorterHostProximity2 = new NodeSorterHostProximity(cluster, td1); + NodeSorterHostProximity nodeSorterHostProximity2 = new NodeSorterHostProximity(cluster, + td1); nodeSorterHostProximity2.getSortedRacks().forEach(x -> seenRacks.add(x.id)); Map> rackIdToHosts = nodeSorterHostProximity2.getRackIdToHosts(); String dumpOfRacks = rackIdToHosts.entrySet().stream() - .map(x -> String.format("rack %s -> hosts [%s]", x.getKey(), String.join(",", x.getValue()))) + .map(x -> String.format("rack %s -> hosts [%s]", x.getKey(), String.join(",", x + .getValue()))) .collect(Collectors.joining("\n\t")); assertEquals(numRacks - 1, seenRacks.size(), - "Expecting rack cnt to be " + (numRacks - 1) + " but found " + seenRacks.size() + "\n\t" + dumpOfRacks); + "Expecting rack cnt to be " + (numRacks - 1) + " but found " + seenRacks.size() + + "\n\t" + dumpOfRacks); assertFalse(seenRacks.contains(DNSToSwitchMapping.DEFAULT_RACK), "Found default-rack=" + DNSToSwitchMapping.DEFAULT_RACK + " in \n\t" + dumpOfRacks); } @@ -938,7 +1059,8 @@ void testWithBlackListedHosts() { int topo1BoltParallelism = 200; final int numSupersPerRack = 10; final int numPortsPerSuper = 66; - long compPerRack = (topo1NumSpouts * topo1SpoutParallelism + topo1NumBolts * topo1BoltParallelism + 10); + long compPerRack = (topo1NumSpouts * topo1SpoutParallelism + + topo1NumBolts * topo1BoltParallelism + 10); long compPerSuper = compPerRack / numSupersPerRack; double cpuPerSuper = compPcore * compPerSuper; double memPerSuper = (compOnHeap + compOffHeap) * compPerSuper; @@ -946,13 +1068,15 @@ void testWithBlackListedHosts() { final String topoName1 = "topology1"; int numRacks = 3; - Map supMap = genSupervisorsWithRacks(numRacks, numSupersPerRack, numPortsPerSuper, + Map supMap = genSupervisorsWithRacks(numRacks, numSupersPerRack, + numPortsPerSuper, 0, 0, cpuPerSuper, memPerSuper, new HashMap<>()); TestDNSToSwitchMapping testDNSToSwitchMapping = new TestDNSToSwitchMapping(supMap.values()); Config config = new Config(); config.putAll(createGrasClusterConfig(compPcore, compOnHeap, compOffHeap, null, null)); - config.put(Config.TOPOLOGY_SCHEDULER_STRATEGY, GenericResourceAwareStrategy.class.getName()); + config.put(Config.TOPOLOGY_SCHEDULER_STRATEGY, GenericResourceAwareStrategy.class + .getName()); IScheduler scheduler = new ResourceAwareScheduler(); scheduler.prepare(config, new StormMetricsRegistry()); @@ -961,16 +1085,19 @@ void testWithBlackListedHosts() { topo1NumBolts, topo1SpoutParallelism, topo1BoltParallelism, 0, 0, "user", topo1MaxHeapSize); Topologies topologies = new Topologies(td1); - Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config); + Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), + supMap, new HashMap<>(), topologies, config); cluster.setNetworkTopography(testDNSToSwitchMapping.getRackToHosts()); Map> networkTopography = cluster.getNetworkTopography(); - assertEquals(numRacks, networkTopography.size(), "Expecting " + numRacks + " racks found " + networkTopography.size()); - assertTrue(networkTopography.size() >= 3, "Expecting racks count to be >= 3, found " + networkTopography.size()); + assertEquals(numRacks, networkTopography.size(), "Expecting " + numRacks + " racks found " + + networkTopography.size()); + assertTrue(networkTopography.size() >= 3, "Expecting racks count to be >= 3, found " + + networkTopography.size()); Set blackListedHosts = new HashSet<>(); List supArray = new ArrayList<>(supMap.values()); - for (int i = 0 ; i < numSupersPerRack ; i++) { + for (int i = 0; i < numSupersPerRack; i++) { blackListedHosts.add(supArray.get(i).getHost()); } blacklistHostsAndSortNodes(blackListedHosts, supMap.values(), cluster, td1); @@ -993,7 +1120,7 @@ private void blacklistHostsAndSortNodes( Set seenRacks = new HashSet<>(); nodeSorterHostProximity.getSortedRacks().forEach(x -> seenRacks.add(x.id)); Set seenHosts = new HashSet<>(); - nodeSorterHostProximity.getRackIdToHosts().forEach((k,v) -> seenHosts.addAll(v)); + nodeSorterHostProximity.getRackIdToHosts().forEach((k, v) -> seenHosts.addAll(v)); allHosts.removeAll(seenHosts); assertEquals(allHosts, blackListedHosts, "Expecting only blacklisted hosts removed"); } @@ -1001,11 +1128,11 @@ private void blacklistHostsAndSortNodes( // now check if sortAllNodes still works Set expectedNodes = sups.stream() .filter(x -> !blackListedHosts.contains(x.getHost())) - .map(x ->x.getId()) + .map(x -> x.getId()) .collect(Collectors.toSet()); Set seenNodes = new HashSet<>(); - nodeSorterHostProximity.prepare(null); - nodeSorterHostProximity.sortAllNodes().forEach( n -> seenNodes.add(n)); + nodeSorterHostProximity.prepare(null); + nodeSorterHostProximity.sortAllNodes().forEach(n -> seenNodes.add(n)); assertEquals(expectedNodes, seenNodes, "Expecting see all supervisors "); } @@ -1016,23 +1143,26 @@ private void blacklistHostsAndSortNodes( * * @param cluster cluster to impair * @param rackId rackId to clear - * @param clearNetworkTopography if true, then clear (but not remove) the hosts in list for the rack. + * @param clearNetworkTopography if true, then clear (but not remove) the hosts in list for the + * rack. * @param clearSupervisorMap if true, then remove supervisors for the rack. */ - private void impairClusterRack(Cluster cluster, String rackId, boolean clearNetworkTopography, boolean clearSupervisorMap) { - Set hostIds = new HashSet<>(cluster.getNetworkTopography().computeIfAbsent(rackId, k -> new ArrayList<>())); + private void impairClusterRack(Cluster cluster, String rackId, boolean clearNetworkTopography, + boolean clearSupervisorMap) { + Set hostIds = new HashSet<>(cluster.getNetworkTopography().computeIfAbsent(rackId, + k -> new ArrayList<>())); if (clearNetworkTopography) { cluster.getNetworkTopography().computeIfAbsent(rackId, k -> new ArrayList<>()).clear(); } if (clearSupervisorMap) { Set supToRemove = new HashSet<>(); - for (String hostId: hostIds) { + for (String hostId : hostIds) { cluster.getSupervisorsByHost(hostId).forEach(s -> supToRemove.add(s.getId())); } Map supervisorDetailsMap = cluster.getSupervisors(); - for (String supId: supToRemove) { + for (String supId : supToRemove) { supervisorDetailsMap.remove(supId); } } } -} \ No newline at end of file +} diff --git a/storm-server/src/test/java/org/apache/storm/scheduler/resource/strategies/scheduling/sorter/TestRoundRobinNodeSorterHostProximity.java b/storm-server/src/test/java/org/apache/storm/scheduler/resource/strategies/scheduling/sorter/TestRoundRobinNodeSorterHostProximity.java index c43ffa71ad9..55f8dde3c25 100644 --- a/storm-server/src/test/java/org/apache/storm/scheduler/resource/strategies/scheduling/sorter/TestRoundRobinNodeSorterHostProximity.java +++ b/storm-server/src/test/java/org/apache/storm/scheduler/resource/strategies/scheduling/sorter/TestRoundRobinNodeSorterHostProximity.java @@ -18,6 +18,38 @@ package org.apache.storm.scheduler.resource.strategies.scheduling.sorter; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.INimbusTest; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.TestBolt; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.createRoundRobinClusterConfig; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.genSupervisorsWithRacks; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.genSupervisorsWithRacksAndNuma; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.genTopology; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.supervisorIdToRackName; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.topoToTopologyDetails; +import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.topologyBuilder; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; import org.apache.storm.Config; import org.apache.storm.metric.StormMetricsRegistry; import org.apache.storm.networktopography.DNSToSwitchMapping; @@ -45,48 +77,17 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeMap; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; -import java.util.stream.Collectors; -import java.util.stream.StreamSupport; - -import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.INimbusTest; -import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.TestBolt; -import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.createRoundRobinClusterConfig; -import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.genSupervisorsWithRacks; -import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.genSupervisorsWithRacksAndNuma; -import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.genTopology; -import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.supervisorIdToRackName; -import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.topoToTopologyDetails; -import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.topologyBuilder; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNotEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; - @ExtendWith({NormalizedResourcesExtension.class}) public class TestRoundRobinNodeSorterHostProximity { - private static final Logger LOG = LoggerFactory.getLogger(TestRoundRobinNodeSorterHostProximity.class); + private static final Logger LOG = LoggerFactory + .getLogger(TestRoundRobinNodeSorterHostProximity.class); private static final int CURRENT_TIME = 1450418597; private static final Class strategyClass = RoundRobinResourceAwareStrategy.class; private Config createClusterConfig(double compPcore, double compOnHeap, double compOffHeap, Map> pools) { - Config config = TestUtilsForResourceAwareScheduler.createClusterConfig(compPcore, compOnHeap, compOffHeap, pools); + Config config = TestUtilsForResourceAwareScheduler.createClusterConfig(compPcore, + compOnHeap, compOffHeap, pools); config.put(Config.TOPOLOGY_SCHEDULER_STRATEGY, strategyClass.getName()); return config; } @@ -125,7 +126,7 @@ public TestDNSToSwitchMapping(Collection supervisorDetailsCol Map hostToRackMap = new HashMap<>(); Map> rackToHosts = new HashMap<>(); - for (SupervisorDetails supervisorDetails: supervisorDetailsCollection) { + for (SupervisorDetails supervisorDetails : supervisorDetailsCollection) { String rackId = supervisorIdToRackName(supervisorDetails.getId()); hostToRackMap.put(supervisorDetails.getHost(), rackId); String host = supervisorDetails.getHost(); @@ -164,25 +165,25 @@ public void testMultipleRacksWithFavoritism() { numRacks, numSupersPerRack, numZonesPerHost, numPortsPerSuper, rackStartNum++, supStartNum, 400, 8000, Collections.emptyMap(), 1.0); - //generate another rack of supervisors with less resources + // generate another rack of supervisors with less resources supStartNum += numSupersPerRack; final Map supMapRack1 = genSupervisorsWithRacksAndNuma( numRacks, numSupersPerRack, numZonesPerHost, numPortsPerSuper, rackStartNum++, supStartNum, 200, 4000, Collections.emptyMap(), 1.0); - //generate some supervisors that are depleted of one resource + // generate some supervisors that are depleted of one resource supStartNum += numSupersPerRack; final Map supMapRack2 = genSupervisorsWithRacksAndNuma( numRacks, numSupersPerRack, numZonesPerHost, numPortsPerSuper, rackStartNum++, supStartNum, 0, 8000, Collections.emptyMap(), 1.0); - //generate some that has a lot of memory but little of cpu + // generate some that has a lot of memory but little of cpu supStartNum += numSupersPerRack; final Map supMapRack3 = genSupervisorsWithRacksAndNuma( numRacks, numSupersPerRack, numZonesPerHost, numPortsPerSuper, rackStartNum++, supStartNum, 10, 8000 * 2 + 4000, Collections.emptyMap(), 1.0); - //generate some that has a lot of cpu but little of memory + // generate some that has a lot of cpu but little of memory supStartNum += numSupersPerRack; final Map supMapRack4 = genSupervisorsWithRacksAndNuma( numRacks, numSupersPerRack, numZonesPerHost, numPortsPerSuper, rackStartNum++, supStartNum, @@ -198,9 +199,10 @@ public void testMultipleRacksWithFavoritism() { config.put(Config.TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB, Double.MAX_VALUE); INimbus iNimbus = new INimbusTest(); - //create test DNSToSwitchMapping plugin + // create test DNSToSwitchMapping plugin TestDNSToSwitchMapping testDNSToSwitchMapping = - new TestDNSToSwitchMapping(supMapRack0, supMapRack1, supMapRack2, supMapRack3, supMapRack4); + new TestDNSToSwitchMapping(supMapRack0, supMapRack1, supMapRack2, supMapRack3, + supMapRack4); Config t1Conf = new Config(); t1Conf.putAll(config); @@ -208,18 +210,23 @@ public void testMultipleRacksWithFavoritism() { t1Conf.put(Config.TOPOLOGY_SCHEDULER_FAVORED_NODES, t1FavoredHostNames); final List t1UnfavoredHostIds = Arrays.asList("host-1", "host-2", "host-3"); t1Conf.put(Config.TOPOLOGY_SCHEDULER_UNFAVORED_NODES, t1UnfavoredHostIds); - //generate topologies - TopologyDetails topo1 = genTopology("topo-1", t1Conf, 8, 0, 2, 0, CURRENT_TIME - 2, 10, "user"); + // generate topologies + TopologyDetails topo1 = genTopology("topo-1", t1Conf, 8, 0, 2, 0, CURRENT_TIME - 2, 10, + "user"); Config t2Conf = new Config(); t2Conf.putAll(config); - t2Conf.put(Config.TOPOLOGY_SCHEDULER_FAVORED_NODES, Arrays.asList("host-31", "host-32", "host-33")); - t2Conf.put(Config.TOPOLOGY_SCHEDULER_UNFAVORED_NODES, Arrays.asList("host-11", "host-12", "host-13")); - TopologyDetails topo2 = genTopology("topo-2", t2Conf, 8, 0, 2, 0, CURRENT_TIME - 2, 10, "user"); + t2Conf.put(Config.TOPOLOGY_SCHEDULER_FAVORED_NODES, Arrays.asList("host-31", "host-32", + "host-33")); + t2Conf.put(Config.TOPOLOGY_SCHEDULER_UNFAVORED_NODES, Arrays.asList("host-11", "host-12", + "host-13")); + TopologyDetails topo2 = genTopology("topo-2", t2Conf, 8, 0, 2, 0, CURRENT_TIME - 2, 10, + "user"); Topologies topologies = new Topologies(topo1, topo2); - Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config); + Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), + supMap, new HashMap<>(), topologies, config); List supHostnames = new LinkedList<>(); for (SupervisorDetails sup : supMap.values()) { @@ -228,13 +235,17 @@ public void testMultipleRacksWithFavoritism() { Map> rackToHosts = testDNSToSwitchMapping.getRackToHosts(); cluster.setNetworkTopography(rackToHosts); - NodeSorterHostProximity nodeSorter = new NodeSorterHostProximity(cluster, topo1, BaseResourceAwareStrategy.NodeSortType.COMMON); + NodeSorterHostProximity nodeSorter = new NodeSorterHostProximity(cluster, topo1, + BaseResourceAwareStrategy.NodeSortType.COMMON); nodeSorter.prepare(null); - List sortedRacks = StreamSupport.stream(nodeSorter.getSortedRacks().spliterator(), false) + List sortedRacks = StreamSupport.stream(nodeSorter.getSortedRacks() + .spliterator(), false) .collect(Collectors.toList()); String rackSummaries = sortedRacks.stream() - .map(x -> String.format("Rack %s -> scheduled-cnt %d, min-avail %f, avg-avail %f, cpu %f, mem %f", - x.id, nodeSorter.getScheduledExecCntByRackId().getOrDefault(x.id, new AtomicInteger(-1)).get(), + .map(x -> String + .format("Rack %s -> scheduled-cnt %d, min-avail %f, avg-avail %f, cpu %f, mem %f", + x.id, nodeSorter.getScheduledExecCntByRackId().getOrDefault(x.id, + new AtomicInteger(-1)).get(), x.minResourcePercent, x.avgResourcePercent, x.availableResources.getTotalCpu(), x.availableResources.getTotalMemoryMb())) @@ -242,20 +253,25 @@ public void testMultipleRacksWithFavoritism() { Iterator it = sortedRacks.iterator(); // Ranked first since rack-000 has the most balanced set of resources - assertEquals("rack-004", it.next().id, "rack-004 should be ordered first\n\t" + rackSummaries); + assertEquals("rack-004", it.next().id, "rack-004 should be ordered first\n\t" + + rackSummaries); // Ranked second since rack-1 has a balanced set of resources but less than rack-0 - assertEquals("rack-000", it.next().id, "rack-000 should be ordered second\n\t" + rackSummaries); + assertEquals("rack-000", it.next().id, "rack-000 should be ordered second\n\t" + + rackSummaries); // Ranked third since rack-4 has a lot of cpu but not a lot of memory assertEquals("rack-003", it.next().id, "rack-003 should be ordered\n\t" + rackSummaries); // Ranked fourth since rack-3 has alot of memory but not cpu - assertEquals("rack-001", it.next().id, "rack-001 should be ordered fourth\n\t" + rackSummaries); - //Ranked last since rack-2 has not cpu resources - assertEquals("rack-002", it.next().id, "rack-002 should be ordered fifth\n\t" + rackSummaries); + assertEquals("rack-001", it.next().id, "rack-001 should be ordered fourth\n\t" + + rackSummaries); + // Ranked last since rack-2 has not cpu resources + assertEquals("rack-002", it.next().id, "rack-002 should be ordered fifth\n\t" + + rackSummaries); } /** * Test if hosts are presented together regardless of resource availability. - * Supervisors are created with multiple Numa zones in such a manner that resources on two numa zones on the same host + * Supervisors are created with multiple Numa zones in such a manner that resources on two numa + * zones on the same host * differ widely in resource availability. */ @Test @@ -273,25 +289,25 @@ public void testMultipleRacksWithHostProximity() { numRacks, numSupersPerRack, numZonesPerHost, numPortsPerSuper, rackStartNum++, supStartNum, 400, 8000, Collections.emptyMap(), numaResourceMultiplier); - //generate another rack of supervisors with less resources + // generate another rack of supervisors with less resources supStartNum += numSupersPerRack; final Map supMapRack1 = genSupervisorsWithRacksAndNuma( numRacks, numSupersPerRack, numZonesPerHost, numPortsPerSuper, rackStartNum++, supStartNum, 200, 4000, Collections.emptyMap(), numaResourceMultiplier); - //generate some supervisors that are depleted of one resource + // generate some supervisors that are depleted of one resource supStartNum += numSupersPerRack; final Map supMapRack2 = genSupervisorsWithRacksAndNuma( numRacks, numSupersPerRack, numZonesPerHost, numPortsPerSuper, rackStartNum++, supStartNum, 0, 8000, Collections.emptyMap(), numaResourceMultiplier); - //generate some that has a lot of memory but little of cpu + // generate some that has a lot of memory but little of cpu supStartNum += numSupersPerRack; final Map supMapRack3 = genSupervisorsWithRacksAndNuma( numRacks, numSupersPerRack, numZonesPerHost, numPortsPerSuper, rackStartNum++, supStartNum, - 10, 8000 * 2 + 4000, Collections.emptyMap(),numaResourceMultiplier); + 10, 8000 * 2 + 4000, Collections.emptyMap(), numaResourceMultiplier); - //generate some that has a lot of cpu but little of memory + // generate some that has a lot of cpu but little of memory supStartNum += numSupersPerRack; final Map supMapRack4 = genSupervisorsWithRacksAndNuma( numRacks, numSupersPerRack, numZonesPerHost, numPortsPerSuper, rackStartNum++, supStartNum, @@ -307,9 +323,10 @@ public void testMultipleRacksWithHostProximity() { config.put(Config.TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB, Double.MAX_VALUE); INimbus iNimbus = new INimbusTest(); - //create test DNSToSwitchMapping plugin + // create test DNSToSwitchMapping plugin TestDNSToSwitchMapping testDNSToSwitchMapping = - new TestDNSToSwitchMapping(supMapRack0, supMapRack1, supMapRack2, supMapRack3, supMapRack4); + new TestDNSToSwitchMapping(supMapRack0, supMapRack1, supMapRack2, supMapRack3, + supMapRack4); Config t1Conf = new Config(); t1Conf.putAll(config); @@ -317,18 +334,23 @@ public void testMultipleRacksWithHostProximity() { t1Conf.put(Config.TOPOLOGY_SCHEDULER_FAVORED_NODES, t1FavoredHostNames); final List t1UnfavoredHostIds = Arrays.asList("host-1", "host-2", "host-3"); t1Conf.put(Config.TOPOLOGY_SCHEDULER_UNFAVORED_NODES, t1UnfavoredHostIds); - //generate topologies - TopologyDetails topo1 = genTopology("topo-1", t1Conf, 8, 0, 2, 0, CURRENT_TIME - 2, 10, "user"); + // generate topologies + TopologyDetails topo1 = genTopology("topo-1", t1Conf, 8, 0, 2, 0, CURRENT_TIME - 2, 10, + "user"); Config t2Conf = new Config(); t2Conf.putAll(config); - t2Conf.put(Config.TOPOLOGY_SCHEDULER_FAVORED_NODES, Arrays.asList("host-31", "host-32", "host-33")); - t2Conf.put(Config.TOPOLOGY_SCHEDULER_UNFAVORED_NODES, Arrays.asList("host-11", "host-12", "host-13")); - TopologyDetails topo2 = genTopology("topo-2", t2Conf, 8, 0, 2, 0, CURRENT_TIME - 2, 10, "user"); + t2Conf.put(Config.TOPOLOGY_SCHEDULER_FAVORED_NODES, Arrays.asList("host-31", "host-32", + "host-33")); + t2Conf.put(Config.TOPOLOGY_SCHEDULER_UNFAVORED_NODES, Arrays.asList("host-11", "host-12", + "host-13")); + TopologyDetails topo2 = genTopology("topo-2", t2Conf, 8, 0, 2, 0, CURRENT_TIME - 2, 10, + "user"); Topologies topologies = new Topologies(topo1, topo2); - Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config); + Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), + supMap, new HashMap<>(), topologies, config); cluster.setNetworkTopography(testDNSToSwitchMapping.getRackToHosts()); @@ -339,11 +361,12 @@ public void testMultipleRacksWithHostProximity() { String prevHost = null; List errLines = new ArrayList(); Map nodeToHost = new RasNodes(cluster).getNodeIdToHostname(); - for (String nodeId: nodeSorter.sortAllNodes()) { + for (String nodeId : nodeSorter.sortAllNodes()) { String host = nodeToHost.getOrDefault(nodeId, "no-host-for-node-" + nodeId); errLines.add(String.format("\tnodeId:%s, host:%s", nodeId, host)); if (!host.equals(prevHost) && seenHosts.contains(host)) { - String err = String.format("Host %s for node %s is out of order:\n\t%s", host, nodeId, String.join("\n\t", errLines)); + String err = String.format("Host %s for node %s is out of order:\n\t%s", host, + nodeId, String.join("\n\t", errLines)); fail(err); } seenHosts.add(host); @@ -382,7 +405,7 @@ public void testMultipleRacksOrderedByCapacity() { supStartNum += numSupersPerRack; final Map supMapRack3 = genSupervisorsWithRacksAndNuma( numRacks, numSupersPerRack, numZonesPerHost, numPortsPerSuper, rackStartNum++, supStartNum, - 300, 8000 - rackStartNum, Collections.emptyMap(),numaResourceMultiplier); + 300, 8000 - rackStartNum, Collections.emptyMap(), numaResourceMultiplier); supStartNum += numSupersPerRack; final Map supMapRack4 = genSupervisorsWithRacksAndNuma( @@ -393,7 +416,8 @@ public void testMultipleRacksOrderedByCapacity() { supStartNum += numSupersPerRack; final Map supMapRack5 = genSupervisorsWithRacksAndNuma( numRacks, numSupersPerRack, numZonesPerHost, numPortsPerSuper, rackStartNum++, supStartNum, - 100, 8000 - rackStartNum, Collections.singletonMap("gpu.count", 0.0), numaResourceMultiplier); + 100, 8000 - rackStartNum, Collections.singletonMap("gpu.count", + 0.0), numaResourceMultiplier); supMap.putAll(supMapRack0); supMap.putAll(supMapRack1); @@ -406,40 +430,54 @@ public void testMultipleRacksOrderedByCapacity() { config.put(Config.TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB, Double.MAX_VALUE); INimbus iNimbus = new INimbusTest(); - //create test DNSToSwitchMapping plugin + // create test DNSToSwitchMapping plugin TestDNSToSwitchMapping testDNSToSwitchMapping = - new TestDNSToSwitchMapping(supMapRack0, supMapRack1, supMapRack2, supMapRack3, supMapRack4, supMapRack5); + new TestDNSToSwitchMapping(supMapRack0, supMapRack1, supMapRack2, supMapRack3, + supMapRack4, supMapRack5); - //generate topologies - TopologyDetails topo1 = genTopology("topo-1", config, 8, 0, 2, 0, CURRENT_TIME - 2, 10, "user"); - TopologyDetails topo2 = genTopology("topo-2", config, 8, 0, 2, 0, CURRENT_TIME - 2, 10, "user"); + // generate topologies + TopologyDetails topo1 = genTopology("topo-1", config, 8, 0, 2, 0, CURRENT_TIME - 2, 10, + "user"); + TopologyDetails topo2 = genTopology("topo-2", config, 8, 0, 2, 0, CURRENT_TIME - 2, 10, + "user"); Topologies topologies = new Topologies(topo1, topo2); - Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config); + Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), + supMap, new HashMap<>(), topologies, config); cluster.setNetworkTopography(testDNSToSwitchMapping.getRackToHosts()); NodeSorterHostProximity nodeSorter = new NodeSorterHostProximity(cluster, topo1); nodeSorter.prepare(null); - List sortedRacks = StreamSupport.stream(nodeSorter.getSortedRacks().spliterator(), false) + List sortedRacks = StreamSupport.stream(nodeSorter.getSortedRacks() + .spliterator(), false) .collect(Collectors.toList()); String rackSummaries = sortedRacks .stream() - .map(x -> String.format("Rack %s -> scheduled-cnt %d, min-avail %f, avg-avail %f, cpu %f, mem %f", - x.id, nodeSorter.getScheduledExecCntByRackId().getOrDefault(x.id, new AtomicInteger(-1)).get(), + .map(x -> String + .format("Rack %s -> scheduled-cnt %d, min-avail %f, avg-avail %f, cpu %f, mem %f", + x.id, nodeSorter.getScheduledExecCntByRackId().getOrDefault(x.id, + new AtomicInteger(-1)).get(), x.minResourcePercent, x.avgResourcePercent, x.availableResources.getTotalCpu(), x.availableResources.getTotalMemoryMb())) .collect(Collectors.joining("\n\t")); NormalizedResourceRequest topoResourceRequest = topo1.getApproximateTotalResources(); - String topoRequest = String.format("Topo %s, approx-requested-resources %s", topo1.getId(), topoResourceRequest.toString()); + String topoRequest = String.format("Topo %s, approx-requested-resources %s", topo1.getId(), + topoResourceRequest.toString()); Iterator it = sortedRacks.iterator(); - assertEquals("rack-000", it.next().id, topoRequest + "\n\t" + rackSummaries + "\nRack-000 should be ordered first since it has the largest capacity"); - assertEquals("rack-001", it.next().id, topoRequest + "\n\t" + rackSummaries + "\nrack-001 should be ordered second since it smaller than rack-000"); - assertEquals("rack-002", it.next().id, topoRequest + "\n\t" + rackSummaries + "\nrack-002 should be ordered third since it is smaller than rack-001"); - assertEquals("rack-003", it.next().id, topoRequest + "\n\t" + rackSummaries + "\nrack-003 should be ordered fourth since it since it is smaller than rack-002"); - assertEquals("rack-004", it.next().id, topoRequest + "\n\t" + rackSummaries + "\nrack-004 should be ordered fifth since it since it is smaller than rack-003"); - assertEquals("rack-005", it.next().id, topoRequest + "\n\t" + rackSummaries + "\nrack-005 should be ordered last since it since it is has smallest capacity"); + assertEquals("rack-000", it.next().id, topoRequest + "\n\t" + rackSummaries + + "\nRack-000 should be ordered first since it has the largest capacity"); + assertEquals("rack-001", it.next().id, topoRequest + "\n\t" + rackSummaries + + "\nrack-001 should be ordered second since it smaller than rack-000"); + assertEquals("rack-002", it.next().id, topoRequest + "\n\t" + rackSummaries + + "\nrack-002 should be ordered third since it is smaller than rack-001"); + assertEquals("rack-003", it.next().id, topoRequest + "\n\t" + rackSummaries + + "\nrack-003 should be ordered fourth since it since it is smaller than rack-002"); + assertEquals("rack-004", it.next().id, topoRequest + "\n\t" + rackSummaries + + "\nrack-004 should be ordered fifth since it since it is smaller than rack-003"); + assertEquals("rack-005", it.next().id, topoRequest + "\n\t" + rackSummaries + + "\nrack-005 should be ordered last since it since it is has smallest capacity"); } /** @@ -450,7 +488,8 @@ public void testMultipleRacksOrderedByCapacity() { @Test public void testAntiAffinityWithMultipleTopologies() { INimbus iNimbus = new INimbusTest(); - Map supMap = genSupervisorsWithRacks(1, 40, 66, 0, 0, 4700, 226200, new HashMap<>()); + Map supMap = genSupervisorsWithRacks(1, 40, 66, 0, 0, 4700, + 226200, new HashMap<>()); HashMap extraResources = new HashMap<>(); extraResources.put("my.gpu", 1.0); supMap.putAll(genSupervisorsWithRacks(1, 40, 66, 1, 0, 4700, 226200, extraResources)); @@ -464,9 +503,10 @@ public void testAntiAffinityWithMultipleTopologies() { TopologyDetails tdSimple = genTopology("topology-simple", config, 1, 5, 100, 300, 0, 0, "user", 8192); - //Schedule the simple topology first + // Schedule the simple topology first Topologies topologies = new Topologies(tdSimple); - Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config); + Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), + supMap, new HashMap<>(), topologies, config); { NodeSorterHostProximity nodeSorter = new NodeSorterHostProximity(cluster, tdSimple); @@ -477,17 +517,24 @@ public void testAntiAffinityWithMultipleTopologies() { .collect(Collectors.toList()); String rackSummaries = StreamSupport .stream(sortedRacks.spliterator(), false) - .map(x -> String.format("Rack %s -> scheduled-cnt %d, min-avail %f, avg-avail %f, cpu %f, mem %f", - x.id, nodeSorter.getScheduledExecCntByRackId().getOrDefault(x.id, new AtomicInteger(-1)).get(), + .map(x -> String + .format("Rack %s -> scheduled-cnt %d, min-avail %f, avg-avail %f, cpu %f, " + + "mem %f", + x.id, nodeSorter.getScheduledExecCntByRackId().getOrDefault(x.id, + new AtomicInteger(-1)).get(), x.minResourcePercent, x.avgResourcePercent, x.availableResources.getTotalCpu(), x.availableResources.getTotalMemoryMb())) .collect(Collectors.joining("\n\t")); - NormalizedResourceRequest topoResourceRequest = tdSimple.getApproximateTotalResources(); - String topoRequest = String.format("Topo %s, approx-requested-resources %s", tdSimple.getId(), topoResourceRequest.toString()); + NormalizedResourceRequest topoResourceRequest = tdSimple + .getApproximateTotalResources(); + String topoRequest = String.format("Topo %s, approx-requested-resources %s", + tdSimple.getId(), topoResourceRequest.toString()); assertEquals(2, sortedRacks.size(), rackSummaries + "\n# of racks sorted"); - assertEquals("rack-000", sortedRacks.get(0).id, rackSummaries + "\nFirst rack sorted"); - assertEquals("rack-001", sortedRacks.get(1).id, rackSummaries + "\nSecond rack sorted"); + assertEquals("rack-000", sortedRacks.get(0).id, rackSummaries + + "\nFirst rack sorted"); + assertEquals("rack-001", sortedRacks.get(1).id, rackSummaries + + "\nSecond rack sorted"); } } @@ -499,9 +546,10 @@ public void testAntiAffinityWithMultipleTopologies() { builder.setBolt("gpu-bolt", new TestBolt(), 40) .addResource("my.gpu", 1.0) .shuffleGrouping("spout-0"); - TopologyDetails tdGpu = topoToTopologyDetails("topology-gpu", config, builder.createTopology(), 0, 0,"user", 8192); + TopologyDetails tdGpu = topoToTopologyDetails("topology-gpu", config, builder + .createTopology(), 0, 0, "user", 8192); - //Now schedule GPU but with the simple topology in place. + // Now schedule GPU but with the simple topology in place. topologies = new Topologies(tdSimple, tdGpu); cluster = new Cluster(cluster, topologies); { @@ -510,23 +558,33 @@ public void testAntiAffinityWithMultipleTopologies() { String comp = tdGpu.getComponentFromExecutor(exec); nodeSorter.prepare(exec); List sortedRacks = StreamSupport - .stream(nodeSorter.getSortedRacks().spliterator(), false).collect(Collectors.toList()); + .stream(nodeSorter.getSortedRacks().spliterator(), false).collect(Collectors + .toList()); String rackSummaries = sortedRacks.stream() - .map(x -> String.format("Rack %s -> scheduled-cnt %d, min-avail %f, avg-avail %f, cpu %f, mem %f", - x.id, nodeSorter.getScheduledExecCntByRackId().getOrDefault(x.id, new AtomicInteger(-1)).get(), + .map(x -> String + .format("Rack %s -> scheduled-cnt %d, min-avail %f, avg-avail %f, cpu %f, " + + "mem %f", + x.id, nodeSorter.getScheduledExecCntByRackId().getOrDefault(x.id, + new AtomicInteger(-1)).get(), x.minResourcePercent, x.avgResourcePercent, x.availableResources.getTotalCpu(), x.availableResources.getTotalMemoryMb())) .collect(Collectors.joining("\n\t")); - NormalizedResourceRequest topoResourceRequest = tdSimple.getApproximateTotalResources(); - String topoRequest = String.format("Topo %s, approx-requested-resources %s", tdSimple.getId(), topoResourceRequest.toString()); + NormalizedResourceRequest topoResourceRequest = tdSimple + .getApproximateTotalResources(); + String topoRequest = String.format("Topo %s, approx-requested-resources %s", + tdSimple.getId(), topoResourceRequest.toString()); assertEquals(2, sortedRacks.size(), rackSummaries + "\n# of racks sorted"); if (comp.equals("gpu-bolt")) { - assertEquals("rack-001", sortedRacks.get(0).id, rackSummaries + "\nFirst rack sorted for " + comp); - assertEquals("rack-000", sortedRacks.get(1).id, rackSummaries + "\nSecond rack sorted for " + comp); + assertEquals("rack-001", sortedRacks.get(0).id, rackSummaries + + "\nFirst rack sorted for " + comp); + assertEquals("rack-000", sortedRacks.get(1).id, rackSummaries + + "\nSecond rack sorted for " + comp); } else { - assertEquals("rack-000", sortedRacks.get(0).id, rackSummaries + "\nFirst rack sorted for " + comp); - assertEquals("rack-001", sortedRacks.get(1).id, rackSummaries + "\nSecond rack sorted for " + comp); + assertEquals("rack-000", sortedRacks.get(0).id, rackSummaries + + "\nFirst rack sorted for " + comp); + assertEquals("rack-001", sortedRacks.get(1).id, rackSummaries + + "\nSecond rack sorted for " + comp); } } } @@ -537,7 +595,7 @@ public void testAntiAffinityWithMultipleTopologies() { assertEquals(1, assignments.size()); // second topology is not expected to be assigned Map> topoPerRackCount = new HashMap<>(); - for (Map.Entry entry: assignments.entrySet()) { + for (Map.Entry entry : assignments.entrySet()) { SchedulerAssignment sa = entry.getValue(); Map slotsPerRack = new TreeMap<>(); for (WorkerSlot slot : sa.getSlots()) { @@ -551,13 +609,15 @@ public void testAntiAffinityWithMultipleTopologies() { Map simpleCount = topoPerRackCount.get("topology-simple-0"); assertNotNull(simpleCount); - //Because the simple topology was scheduled first we want to be sure that it didn't put anything on + // Because the simple topology was scheduled first we want to be sure that it didn't put + // anything on // the GPU nodes. - assertEquals(2, simpleCount.size()); //Both racks are in use - assertTrue(simpleCount.containsKey("r001")); //r001 is the second rack with GPUs - assertTrue(simpleCount.containsKey("r000")); //r000 is the first rack with no GPUs + assertEquals(2, simpleCount.size()); // Both racks are in use + assertTrue(simpleCount.containsKey("r001")); // r001 is the second rack with GPUs + assertTrue(simpleCount.containsKey("r000")); // r000 is the first rack with no GPUs - //We don't really care too much about the scheduling of topology-gpu-0, because it was scheduled. + // We don't really care too much about the scheduling of topology-gpu-0, because it was + // scheduled. } /** @@ -565,10 +625,10 @@ public void testAntiAffinityWithMultipleTopologies() { */ private void freeSomeWorkerSlots(Cluster cluster) { Map assignmentMap = cluster.getAssignments(); - for (SchedulerAssignment schedulerAssignment: assignmentMap.values()) { + for (SchedulerAssignment schedulerAssignment : assignmentMap.values()) { int i = 0; List slotsToKill = new ArrayList<>(); - for (WorkerSlot workerSlot: schedulerAssignment.getSlots()) { + for (WorkerSlot workerSlot : schedulerAssignment.getSlots()) { i++; if (i % 5 == 0) { slotsToKill.add(workerSlot); @@ -599,7 +659,8 @@ public void testDistributeOverRacks() { final double numaResourceMultiplier = 1.0; int rackStartNum = 0; int supStartNum = 0; - long compPerRack = (topo1NumSpouts * topo1SpoutParallelism + topo1NumBolts * topo1BoltParallelism) * 4/5; // not enough for topo1 + long compPerRack = (topo1NumSpouts * topo1SpoutParallelism + + topo1NumBolts * topo1BoltParallelism) * 4 / 5; // not enough for topo1 long compPerSuper = compPerRack / numSupersPerRack; double cpuPerSuper = compPcore * compPerSuper; double memPerSuper = (compOnHeap + compOffHeap) * compPerSuper; @@ -612,7 +673,8 @@ public void testDistributeOverRacks() { TestDNSToSwitchMapping testDNSToSwitchMapping = new TestDNSToSwitchMapping(supMap.values()); Config config = new Config(); - config.putAll(createRoundRobinClusterConfig(compPcore, compOnHeap, compOffHeap, null, null)); + config.putAll(createRoundRobinClusterConfig(compPcore, compOnHeap, compOffHeap, null, + null)); IScheduler scheduler = new ResourceAwareScheduler(); scheduler.prepare(config, new StormMetricsRegistry()); @@ -620,18 +682,21 @@ public void testDistributeOverRacks() { TopologyDetails td1 = genTopology(topoName1, config, topo1NumSpouts, topo1NumBolts, topo1SpoutParallelism, topo1BoltParallelism, 0, 0, "user", topo1MaxHeapSize); - //Schedule the topo1 topology and ensure it fits on 2 racks + // Schedule the topo1 topology and ensure it fits on 2 racks Topologies topologies = new Topologies(td1); - Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config); + Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), + supMap, new HashMap<>(), topologies, config); cluster.setNetworkTopography(testDNSToSwitchMapping.getRackToHosts()); scheduler.schedule(topologies, cluster); Set assignedRacks = cluster.getAssignedRacks(td1.getId()); - assertEquals(numRacks, assignedRacks.size(), "Racks for topology=" + td1.getId() + " is " + assignedRacks); + assertEquals(numRacks, assignedRacks.size(), "Racks for topology=" + td1.getId() + " is " + + assignedRacks); } /** - * Racks are equally likely to be selected, rather than those with low resources already running components + * Racks are equally likely to be selected, rather than those with low resources already running + * components * for the same topology. * . *
    • Schedule topo1 on one rack
    • @@ -661,7 +726,8 @@ public void testDistributeAcrossRacks() { final double numaResourceMultiplier = 1.0; int rackStartNum = 0; int supStartNum = 0; - long compPerRack = (topo1NumSpouts * topo1SpoutParallelism + topo1NumBolts * topo1BoltParallelism + long compPerRack = (topo1NumSpouts * topo1SpoutParallelism + + topo1NumBolts * topo1BoltParallelism + topo2NumSpouts * topo2SpoutParallelism); // enough for topo1 but not topo1+topo2 long compPerSuper = compPerRack / numSupersPerRack; double cpuPerSuper = compPcore * compPerSuper; @@ -677,7 +743,8 @@ public void testDistributeAcrossRacks() { TestDNSToSwitchMapping testDNSToSwitchMapping = new TestDNSToSwitchMapping(supMap.values()); Config config = new Config(); - config.putAll(createRoundRobinClusterConfig(compPcore, compOnHeap, compOffHeap, null, null)); + config.putAll(createRoundRobinClusterConfig(compPcore, compOnHeap, compOffHeap, null, + null)); IScheduler scheduler = new ResourceAwareScheduler(); scheduler.prepare(config, new StormMetricsRegistry()); @@ -685,38 +752,47 @@ public void testDistributeAcrossRacks() { TopologyDetails td1 = genTopology(topoName1, config, topo1NumSpouts, topo1NumBolts, topo1SpoutParallelism, topo1BoltParallelism, 0, 0, "user", topo1MaxHeapSize); - //Schedule the topo1 topology and ensure it fits on 1 rack + // Schedule the topo1 topology and ensure it fits on 1 rack Topologies topologies = new Topologies(td1); - Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config); + Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), + supMap, new HashMap<>(), topologies, config); cluster.setNetworkTopography(testDNSToSwitchMapping.getRackToHosts()); scheduler.schedule(topologies, cluster); Set assignedRacks = cluster.getAssignedRacks(td1.getId()); - assertEquals(numRacks, assignedRacks.size(), "Racks for topology=" + td1.getId() + " is " + assignedRacks); + assertEquals(numRacks, assignedRacks.size(), "Racks for topology=" + td1.getId() + " is " + + assignedRacks); - TopologyBuilder builder = topologyBuilder(topo2NumSpouts, topo2NumBolts, topo2SpoutParallelism, topo2BoltParallelism); - TopologyDetails td2 = topoToTopologyDetails(topoName2, config, builder.createTopology(), 0, 0,"user", topo2MaxHeapSize); + TopologyBuilder builder = topologyBuilder(topo2NumSpouts, topo2NumBolts, + topo2SpoutParallelism, topo2BoltParallelism); + TopologyDetails td2 = topoToTopologyDetails(topoName2, config, builder.createTopology(), 0, + 0, "user", topo2MaxHeapSize); - //Now schedule GPU but with the simple topology in place. + // Now schedule GPU but with the simple topology in place. topologies = new Topologies(td1, td2); cluster = new Cluster(cluster, topologies); scheduler.schedule(topologies, cluster); assignedRacks = cluster.getAssignedRacks(td1.getId(), td2.getId()); - assertEquals(numRacks, assignedRacks.size(), "Racks for topologies=" + td1.getId() + "/" + td2.getId() + " is " + assignedRacks); + assertEquals(numRacks, assignedRacks.size(), "Racks for topologies=" + td1.getId() + "/" + + td2.getId() + " is " + assignedRacks); // topo2 will not get scheduled as topo1 will occupy all racks assignedRacks = cluster.getAssignedRacks(td2.getId()); - assertEquals(0, assignedRacks.size(), "Racks for topologies=" + td2.getId() + " is " + assignedRacks); + assertEquals(0, assignedRacks.size(), "Racks for topologies=" + td2.getId() + " is " + + assignedRacks); - // now unassign topo2, expect all racks to be in use; free some slots and reschedule topo1 some topo1 executors + // now unassign topo2, expect all racks to be in use; free some slots and reschedule topo1 + // some topo1 executors cluster.unassign(td2.getId()); assignedRacks = cluster.getAssignedRacks(td2.getId()); assertEquals(0, assignedRacks.size(), - "After unassigning topology " + td2.getId() + ", racks for topology=" + td2.getId() + " is " + assignedRacks); + "After unassigning topology " + td2.getId() + ", racks for topology=" + td2.getId() + + " is " + assignedRacks); assignedRacks = cluster.getAssignedRacks(td1.getId()); assertEquals(numRacks, assignedRacks.size(), - "After unassigning topology " + td2.getId() + ", racks for topology=" + td1.getId() + " is " + assignedRacks); + "After unassigning topology " + td2.getId() + ", racks for topology=" + td1.getId() + + " is " + assignedRacks); assertFalse(cluster.needsSchedulingRas(td1), "Topology " + td1.getId() + " should be fully assigned before freeing slots"); freeSomeWorkerSlots(cluster); @@ -729,18 +805,22 @@ public void testDistributeAcrossRacks() { // all racks should be in use by topology1 assignedRacks = cluster.getAssignedRacks(td1.getId()); assertEquals(numRacks, assignedRacks.size(), - "After reassigning topology " + td2.getId() + ", racks for topology=" + td1.getId() + " is " + assignedRacks); + "After reassigning topology " + td2.getId() + ", racks for topology=" + td1.getId() + + " is " + assignedRacks); } /** * Assign and then clear out a rack to host list mapping in cluster.networkTopography. * Expected behavior is that: - *
    • the rack without hosts does not show up in {@link NodeSorterHostProximity#getSortedRacks()}
    • - *
    • all the supervisor nodes still get returned in {@link NodeSorterHostProximity#sortAllNodes()} ()}
    • + *
    • the rack without hosts does not show up in {@link + * NodeSorterHostProximity#getSortedRacks()}
    • + *
    • all the supervisor nodes still get returned in {@link + * NodeSorterHostProximity#sortAllNodes()} ()}
    • *
    • supervisors on cleared rack show up under {@link DNSToSwitchMapping#DEFAULT_RACK}
    • * - *

      - * Force an usual condition, where one of the racks is still passed to LazyNodeSortingIterator with + *

      Force an usual condition, where one of the racks is still passed to + * LazyNodeSortingIterator + * with * an empty list and then ensure that code is resilient. *

      */ @@ -756,7 +836,8 @@ void testWithImpairedClusterNetworkTopography() { int topo1BoltParallelism = 200; final int numSupersPerRack = 10; final int numPortsPerSuper = 66; - long compPerRack = (topo1NumSpouts * topo1SpoutParallelism + topo1NumBolts * topo1BoltParallelism + 10); + long compPerRack = (topo1NumSpouts * topo1SpoutParallelism + + topo1NumBolts * topo1BoltParallelism + 10); long compPerSuper = compPerRack / numSupersPerRack; double cpuPerSuper = compPcore * compPerSuper; double memPerSuper = (compOnHeap + compOffHeap) * compPerSuper; @@ -764,12 +845,14 @@ void testWithImpairedClusterNetworkTopography() { final String topoName1 = "topology1"; int numRacks = 3; - Map supMap = genSupervisorsWithRacks(numRacks, numSupersPerRack, numPortsPerSuper, + Map supMap = genSupervisorsWithRacks(numRacks, numSupersPerRack, + numPortsPerSuper, 0, 0, cpuPerSuper, memPerSuper, new HashMap<>()); TestDNSToSwitchMapping testDNSToSwitchMapping = new TestDNSToSwitchMapping(supMap.values()); Config config = new Config(); - config.putAll(createRoundRobinClusterConfig(compPcore, compOnHeap, compOffHeap, null, null)); + config.putAll(createRoundRobinClusterConfig(compPcore, compOnHeap, compOffHeap, null, + null)); IScheduler scheduler = new ResourceAwareScheduler(); scheduler.prepare(config, new StormMetricsRegistry()); @@ -778,51 +861,63 @@ void testWithImpairedClusterNetworkTopography() { topo1NumBolts, topo1SpoutParallelism, topo1BoltParallelism, 0, 0, "user", topo1MaxHeapSize); Topologies topologies = new Topologies(td1); - Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config); + Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), + supMap, new HashMap<>(), topologies, config); cluster.setNetworkTopography(testDNSToSwitchMapping.getRackToHosts()); Map> networkTopography = cluster.getNetworkTopography(); - assertEquals(numRacks, networkTopography.size(), "Expecting " + numRacks + " racks found " + networkTopography.size()); - assertTrue(networkTopography.size() >= 3, "Expecting racks count to be >= 3, found " + networkTopography.size()); - - // Impair cluster.networkTopography and set one rack to have zero hosts, getSortedRacks should exclude this rack. - // Keep, the supervisorDetails unchanged - confirm that these nodes are not lost even with incomplete networkTopography + assertEquals(numRacks, networkTopography.size(), "Expecting " + numRacks + " racks found " + + networkTopography.size()); + assertTrue(networkTopography.size() >= 3, "Expecting racks count to be >= 3, found " + + networkTopography.size()); + + // Impair cluster.networkTopography and set one rack to have zero hosts, getSortedRacks + // should exclude this rack. + // Keep, the supervisorDetails unchanged - confirm that these nodes are not lost even with + // incomplete networkTopography String rackIdToZero = networkTopography.keySet().stream().findFirst().get(); impairClusterRack(cluster, rackIdToZero, true, false); NodeSorterHostProximity nodeSorterHostProximity = new NodeSorterHostProximity(cluster, td1); nodeSorterHostProximity.getSortedRacks().forEach(x -> assertNotEquals(x.id, rackIdToZero)); - // confirm that the above action has not lost the hosts and that they appear under the DEFAULT rack + // confirm that the above action has not lost the hosts and that they appear under the + // DEFAULT rack { Set seenRacks = new HashSet<>(); nodeSorterHostProximity.getSortedRacks().forEach(x -> seenRacks.add(x.id)); assertEquals(numRacks, seenRacks.size(), "Expecting rack cnt to be still " + numRacks); assertTrue(seenRacks.contains(DNSToSwitchMapping.DEFAULT_RACK), - "Expecting to see default-rack=" + DNSToSwitchMapping.DEFAULT_RACK + " in sortedRacks"); + "Expecting to see default-rack=" + DNSToSwitchMapping.DEFAULT_RACK + + " in sortedRacks"); } // now check if node/supervisor is missing when sorting all nodes Set expectedNodes = supMap.keySet(); Set seenNodes = new HashSet<>(); nodeSorterHostProximity.prepare(null); - nodeSorterHostProximity.sortAllNodes().forEach( n -> seenNodes.add(n)); + nodeSorterHostProximity.sortAllNodes().forEach(n -> seenNodes.add(n)); assertEquals(expectedNodes, seenNodes, "Expecting see all supervisors "); // Now fully impair the cluster - confirm no default rack { - cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config); - cluster.setNetworkTopography(new TestDNSToSwitchMapping(supMap.values()).getRackToHosts()); + cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, + new HashMap<>(), topologies, config); + cluster.setNetworkTopography(new TestDNSToSwitchMapping(supMap.values()) + .getRackToHosts()); impairClusterRack(cluster, rackIdToZero, true, true); Set seenRacks = new HashSet<>(); - NodeSorterHostProximity nodeSorterHostProximity2 = new NodeSorterHostProximity(cluster, td1); + NodeSorterHostProximity nodeSorterHostProximity2 = new NodeSorterHostProximity(cluster, + td1); nodeSorterHostProximity2.getSortedRacks().forEach(x -> seenRacks.add(x.id)); Map> rackIdToHosts = nodeSorterHostProximity2.getRackIdToHosts(); String dumpOfRacks = rackIdToHosts.entrySet().stream() - .map(x -> String.format("rack %s -> hosts [%s]", x.getKey(), String.join(",", x.getValue()))) + .map(x -> String.format("rack %s -> hosts [%s]", x.getKey(), String.join(",", x + .getValue()))) .collect(Collectors.joining("\n\t")); assertEquals(numRacks - 1, seenRacks.size(), - "Expecting rack cnt to be " + (numRacks - 1) + " but found " + seenRacks.size() + "\n\t" + dumpOfRacks); + "Expecting rack cnt to be " + (numRacks - 1) + " but found " + seenRacks.size() + + "\n\t" + dumpOfRacks); assertFalse(seenRacks.contains(DNSToSwitchMapping.DEFAULT_RACK), "Found default-rack=" + DNSToSwitchMapping.DEFAULT_RACK + " in \n\t" + dumpOfRacks); } @@ -845,7 +940,8 @@ void testWithBlackListedHosts() { int topo1BoltParallelism = 200; final int numSupersPerRack = 10; final int numPortsPerSuper = 66; - long compPerRack = (topo1NumSpouts * topo1SpoutParallelism + topo1NumBolts * topo1BoltParallelism + 10); + long compPerRack = (topo1NumSpouts * topo1SpoutParallelism + + topo1NumBolts * topo1BoltParallelism + 10); long compPerSuper = compPerRack / numSupersPerRack; double cpuPerSuper = compPcore * compPerSuper; double memPerSuper = (compOnHeap + compOffHeap) * compPerSuper; @@ -853,12 +949,14 @@ void testWithBlackListedHosts() { final String topoName1 = "topology1"; int numRacks = 3; - Map supMap = genSupervisorsWithRacks(numRacks, numSupersPerRack, numPortsPerSuper, + Map supMap = genSupervisorsWithRacks(numRacks, numSupersPerRack, + numPortsPerSuper, 0, 0, cpuPerSuper, memPerSuper, new HashMap<>()); TestDNSToSwitchMapping testDNSToSwitchMapping = new TestDNSToSwitchMapping(supMap.values()); Config config = new Config(); - config.putAll(createRoundRobinClusterConfig(compPcore, compOnHeap, compOffHeap, null, null)); + config.putAll(createRoundRobinClusterConfig(compPcore, compOnHeap, compOffHeap, null, + null)); IScheduler scheduler = new ResourceAwareScheduler(); scheduler.prepare(config, new StormMetricsRegistry()); @@ -867,16 +965,19 @@ void testWithBlackListedHosts() { topo1NumBolts, topo1SpoutParallelism, topo1BoltParallelism, 0, 0, "user", topo1MaxHeapSize); Topologies topologies = new Topologies(td1); - Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config); + Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), + supMap, new HashMap<>(), topologies, config); cluster.setNetworkTopography(testDNSToSwitchMapping.getRackToHosts()); Map> networkTopography = cluster.getNetworkTopography(); - assertEquals(numRacks, networkTopography.size(), "Expecting " + numRacks + " racks found " + networkTopography.size()); - assertTrue(networkTopography.size() >= 3, "Expecting racks count to be >= 3, found " + networkTopography.size()); + assertEquals(numRacks, networkTopography.size(), "Expecting " + numRacks + " racks found " + + networkTopography.size()); + assertTrue(networkTopography.size() >= 3, "Expecting racks count to be >= 3, found " + + networkTopography.size()); Set blackListedHosts = new HashSet<>(); List supArray = new ArrayList<>(supMap.values()); - for (int i = 0 ; i < numSupersPerRack ; i++) { + for (int i = 0; i < numSupersPerRack; i++) { blackListedHosts.add(supArray.get(i).getHost()); } blacklistHostsAndSortNodes(blackListedHosts, supMap.values(), cluster, td1); @@ -899,7 +1000,7 @@ private void blacklistHostsAndSortNodes( Set seenRacks = new HashSet<>(); nodeSorterHostProximity.getSortedRacks().forEach(x -> seenRacks.add(x.id)); Set seenHosts = new HashSet<>(); - nodeSorterHostProximity.getRackIdToHosts().forEach((k,v) -> seenHosts.addAll(v)); + nodeSorterHostProximity.getRackIdToHosts().forEach((k, v) -> seenHosts.addAll(v)); allHosts.removeAll(seenHosts); assertEquals(allHosts, blackListedHosts, "Expecting only blacklisted hosts removed"); } @@ -907,11 +1008,11 @@ private void blacklistHostsAndSortNodes( // now check if sortAllNodes still works Set expectedNodes = sups.stream() .filter(x -> !blackListedHosts.contains(x.getHost())) - .map(x ->x.getId()) + .map(x -> x.getId()) .collect(Collectors.toSet()); Set seenNodes = new HashSet<>(); - nodeSorterHostProximity.prepare(null); - nodeSorterHostProximity.sortAllNodes().forEach( n -> seenNodes.add(n)); + nodeSorterHostProximity.prepare(null); + nodeSorterHostProximity.sortAllNodes().forEach(n -> seenNodes.add(n)); assertEquals(expectedNodes, seenNodes, "Expecting see all supervisors "); } @@ -922,23 +1023,26 @@ private void blacklistHostsAndSortNodes( * * @param cluster cluster to impair * @param rackId rackId to clear - * @param clearNetworkTopography if true, then clear (but not remove) the hosts in list for the rack. + * @param clearNetworkTopography if true, then clear (but not remove) the hosts in list for the + * rack. * @param clearSupervisorMap if true, then remove supervisors for the rack. */ - private void impairClusterRack(Cluster cluster, String rackId, boolean clearNetworkTopography, boolean clearSupervisorMap) { - Set hostIds = new HashSet<>(cluster.getNetworkTopography().computeIfAbsent(rackId, k -> new ArrayList<>())); + private void impairClusterRack(Cluster cluster, String rackId, boolean clearNetworkTopography, + boolean clearSupervisorMap) { + Set hostIds = new HashSet<>(cluster.getNetworkTopography().computeIfAbsent(rackId, + k -> new ArrayList<>())); if (clearNetworkTopography) { cluster.getNetworkTopography().computeIfAbsent(rackId, k -> new ArrayList<>()).clear(); } if (clearSupervisorMap) { Set supToRemove = new HashSet<>(); - for (String hostId: hostIds) { + for (String hostId : hostIds) { cluster.getSupervisorsByHost(hostId).forEach(s -> supToRemove.add(s.getId())); } Map supervisorDetailsMap = cluster.getSupervisors(); - for (String supId: supToRemove) { + for (String supId : supToRemove) { supervisorDetailsMap.remove(supId); } } } -} \ No newline at end of file +} diff --git a/storm-server/src/test/java/org/apache/storm/scheduler/utils/ArtifactoryConfigLoaderTest.java b/storm-server/src/test/java/org/apache/storm/scheduler/utils/ArtifactoryConfigLoaderTest.java index 933cd81cb64..d9d577e8b38 100644 --- a/storm-server/src/test/java/org/apache/storm/scheduler/utils/ArtifactoryConfigLoaderTest.java +++ b/storm-server/src/test/java/org/apache/storm/scheduler/utils/ArtifactoryConfigLoaderTest.java @@ -1,17 +1,26 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

      * http://www.apache.org/licenses/LICENSE-2.0 *

      - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.scheduler.utils; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + import java.io.File; import java.nio.file.Files; import java.nio.file.Path; @@ -20,16 +29,12 @@ import org.apache.commons.io.FileUtils; import org.apache.storm.Config; import org.apache.storm.DaemonConfig; -import org.apache.storm.utils.Time; import org.apache.storm.utils.Time.SimulatedTime; +import org.apache.storm.utils.Time; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; - public class ArtifactoryConfigLoaderTest { private static final String ARTIFACTORY_HTTP_SCHEME_PREFIX = "artifactory+http://"; @@ -68,9 +73,11 @@ public void testPointingAtDirectory() { ArtifactoryConfigLoaderMock loaderMock = new ArtifactoryConfigLoaderMock(conf); loaderMock.setData("Anything", "/location/of/this/dir", - "{\"children\" : [ { \"uri\" : \"/20160621204337.yaml\", \"folder\" : false }]}"); + "{\"children\" : [ { \"uri\" : \"/20160621204337.yaml\", \"folder\" : " + + "false }]}"); loaderMock - .setData(null, null, "{ \"" + DaemonConfig.MULTITENANT_SCHEDULER_USER_POOLS + "\": {one: 1, two: 2, three: 3, four : 4}}"); + .setData(null, null, "{ \"" + DaemonConfig.MULTITENANT_SCHEDULER_USER_POOLS + + "\": {one: 1, two: 2, three: 3, four : 4}}"); Map ret = loaderMock.load(DaemonConfig.MULTITENANT_SCHEDULER_USER_POOLS); assertNotNull(ret, "Unexpectedly returned null"); @@ -102,9 +109,12 @@ public void testArtifactUpdate() { ArtifactoryConfigLoaderMock loaderMock = new ArtifactoryConfigLoaderMock(conf); loaderMock.setData("Anything", "/location/of/test/dir", - "{\"children\" : [ { \"uri\" : \"/20160621204337.yaml\", \"folder\" : false }]}"); - loaderMock.setData(null, null, "{ \"" + DaemonConfig.MULTITENANT_SCHEDULER_USER_POOLS + "\": {one: 1, two: 2, three: 3}}"); - Map ret = loaderMock.load(DaemonConfig.MULTITENANT_SCHEDULER_USER_POOLS); + "{\"children\" : [ { \"uri\" : \"/20160621204337.yaml\", " + + "\"folder\" : false }]}"); + loaderMock.setData(null, null, "{ \"" + DaemonConfig.MULTITENANT_SCHEDULER_USER_POOLS + + "\": {one: 1, two: 2, three: 3}}"); + Map ret = loaderMock + .load(DaemonConfig.MULTITENANT_SCHEDULER_USER_POOLS); assertNotNull(ret, "Unexpectedly returned null"); assertEquals(1, ret.get("one")); @@ -123,15 +133,18 @@ public void testArtifactUpdate() { // Now let's update it, but not advance time. Should get old map again. loaderMock.setData("Anything", "/location/of/test/dir", - "{\"children\" : [ { \"uri\" : \"/20160621204999.yaml\", \"folder\" : false }]}"); + "{\"children\" : [ { \"uri\" : \"/20160621204999.yaml\", " + + "\"folder\" : false }]}"); loaderMock - .setData(null, null, "{ \"" + DaemonConfig.MULTITENANT_SCHEDULER_USER_POOLS + "\": {one: 1, two: 2, three: 3, four : 4}}"); + .setData(null, null, "{ \"" + DaemonConfig.MULTITENANT_SCHEDULER_USER_POOLS + + "\": {one: 1, two: 2, three: 3, four : 4}}"); ret = loaderMock.load(DaemonConfig.MULTITENANT_SCHEDULER_USER_POOLS); assertNotNull(ret, "Unexpectedly returned null"); assertEquals(1, ret.get("one")); assertEquals(2, ret.get("two")); assertEquals(3, ret.get("three")); - assertNull(ret.get("four"), "Unexpectedly did not return null, not enough time passed!"); + assertNull(ret.get("four"), + "Unexpectedly did not return null, not enough time passed!"); // Re-load from cached' file. ret2 = tc2.load(DaemonConfig.MULTITENANT_SCHEDULER_USER_POOLS); @@ -139,7 +152,8 @@ public void testArtifactUpdate() { assertEquals(1, ret2.get("one")); assertEquals(2, ret2.get("two")); assertEquals(3, ret2.get("three")); - assertNull(ret2.get("four"), "Unexpectedly did not return null, last cached result should not have \"four\""); + assertNull(ret2.get("four"), + "Unexpectedly did not return null, last cached result should not have \"four\""); // Now, let's advance time. Time.advanceTime(11 * 60 * 1000); @@ -166,13 +180,16 @@ public void testPointingAtSpecificArtifact() { // This is a test where we are configured to point right at a single artifact Config conf = new Config(); conf.put(DaemonConfig.SCHEDULER_CONFIG_LOADER_URI, - ARTIFACTORY_HTTP_SCHEME_PREFIX + "bogushost.yahoo.com:9999/location/of/this/artifact"); + ARTIFACTORY_HTTP_SCHEME_PREFIX + + "bogushost.yahoo.com:9999/location/of/this/artifact"); conf.put(Config.STORM_LOCAL_DIR, tmpDirPath.toString()); ArtifactoryConfigLoaderMock loaderMock = new ArtifactoryConfigLoaderMock(conf); - loaderMock.setData("Anything", "/location/of/this/artifact", "{ \"downloadUri\": \"anything\"}"); - loaderMock.setData(null, null, "{ \"" + DaemonConfig.MULTITENANT_SCHEDULER_USER_POOLS + "\": {one: 1, two: 2, three: 3}}"); + loaderMock.setData("Anything", "/location/of/this/artifact", + "{ \"downloadUri\": \"anything\"}"); + loaderMock.setData(null, null, "{ \"" + DaemonConfig.MULTITENANT_SCHEDULER_USER_POOLS + + "\": {one: 1, two: 2, three: 3}}"); Map ret = loaderMock.load(DaemonConfig.MULTITENANT_SCHEDULER_USER_POOLS); assertNotNull(ret, "Unexpectedly returned null"); @@ -194,11 +211,13 @@ public void testMalformedYaml() { // This is a test where we are configured to point right at a single artifact Config conf = new Config(); conf.put(DaemonConfig.SCHEDULER_CONFIG_LOADER_URI, - ARTIFACTORY_HTTP_SCHEME_PREFIX + "bogushost.yahoo.com:9999/location/of/this/artifact"); + ARTIFACTORY_HTTP_SCHEME_PREFIX + + "bogushost.yahoo.com:9999/location/of/this/artifact"); conf.put(Config.STORM_LOCAL_DIR, tmpDirPath.toString()); ArtifactoryConfigLoaderMock loaderMock = new ArtifactoryConfigLoaderMock(conf); - loaderMock.setData("Anything", "/location/of/this/artifact", "{ \"downloadUri\": \"anything\"}"); + loaderMock.setData("Anything", "/location/of/this/artifact", + "{ \"downloadUri\": \"anything\"}"); loaderMock.setData(null, null, "ThisIsNotValidYaml"); Map ret = loaderMock.load(DaemonConfig.MULTITENANT_SCHEDULER_USER_POOLS); @@ -229,4 +248,4 @@ protected String doGet(String api, String artifact, String host, Integer port) { return getDataMap.get(artifact); } } -} \ No newline at end of file +} diff --git a/storm-server/src/test/java/org/apache/storm/scheduler/utils/FileConfigLoaderTest.java b/storm-server/src/test/java/org/apache/storm/scheduler/utils/FileConfigLoaderTest.java index c89875248be..716f61a7aa4 100644 --- a/storm-server/src/test/java/org/apache/storm/scheduler/utils/FileConfigLoaderTest.java +++ b/storm-server/src/test/java/org/apache/storm/scheduler/utils/FileConfigLoaderTest.java @@ -1,17 +1,27 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

      - * http://www.apache.org/licenses/LICENSE-2.0 - *

      - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

      http://www.apache.org/licenses/LICENSE-2.0 + * + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.scheduler.utils; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + import java.io.File; import java.io.FileWriter; import java.nio.file.Files; @@ -22,10 +32,6 @@ import org.junit.jupiter.api.Test; import org.yaml.snakeyaml.Yaml; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; - public class FileConfigLoaderTest { private static final String FILE_SCHEME_PREFIX = "file://"; @@ -60,7 +66,8 @@ public void testMalformedYaml() throws Exception { fw.close(); Config conf = new Config(); - conf.put(DaemonConfig.SCHEDULER_CONFIG_LOADER_URI, FILE_SCHEME_PREFIX + temp.getCanonicalPath()); + conf.put(DaemonConfig.SCHEDULER_CONFIG_LOADER_URI, FILE_SCHEME_PREFIX + temp + .getCanonicalPath()); FileConfigLoader testLoader = new FileConfigLoader(conf); Map result = testLoader.load(DaemonConfig.MULTITENANT_SCHEDULER_USER_POOLS); @@ -90,7 +97,8 @@ public void testValidFile() throws Exception { fw.close(); Config conf = new Config(); - conf.put(DaemonConfig.SCHEDULER_CONFIG_LOADER_URI, FILE_SCHEME_PREFIX + temp.getCanonicalPath()); + conf.put(DaemonConfig.SCHEDULER_CONFIG_LOADER_URI, FILE_SCHEME_PREFIX + temp + .getCanonicalPath()); FileConfigLoader loader = new FileConfigLoader(conf); Map result = loader.load(DaemonConfig.MULTITENANT_SCHEDULER_USER_POOLS); @@ -105,4 +113,4 @@ public void testValidFile() throws Exception { assertEquals(expectedValue, returnedValue, "Bad value for key=" + key); } } -} \ No newline at end of file +} diff --git a/storm-server/src/test/java/org/apache/storm/security/auth/AuthTest.java b/storm-server/src/test/java/org/apache/storm/security/auth/AuthTest.java index cc2e4acaee7..92ae6b0f430 100644 --- a/storm-server/src/test/java/org/apache/storm/security/auth/AuthTest.java +++ b/storm-server/src/test/java/org/apache/storm/security/auth/AuthTest.java @@ -1,17 +1,36 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.security.auth; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; +import static org.mockito.Mockito.anyString; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + import java.io.File; import java.io.IOException; import java.net.InetAddress; @@ -39,34 +58,30 @@ import org.apache.storm.security.auth.workertoken.WorkerTokenManager; import org.apache.storm.testing.InProcessZookeeper; import org.apache.storm.thrift.transport.TTransportException; -import org.awaitility.Awaitility; import org.apache.storm.utils.ConfigUtils; import org.apache.storm.utils.NimbusClient; import org.apache.storm.utils.Time; import org.apache.storm.utils.Utils; +import org.awaitility.Awaitility; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.condition.DisabledOnOs; import org.junit.jupiter.api.condition.OS; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; -import static org.mockito.Mockito.*; - public class AuthTest { - //3 seconds in milliseconds + // 3 seconds in milliseconds public static final int NIMBUS_TIMEOUT = 3_000; private static final Logger LOG = LoggerFactory.getLogger(AuthTest.class); private static final File BASE = new File("./src/test/resources/"); - private static final String DIGEST_JAAS_CONF = new File(BASE, "jaas_digest.conf").getAbsolutePath(); - private static final String BAD_PASSWORD_CONF = new File(BASE, "jaas_digest_bad_password.conf").getAbsolutePath(); - private static final String WRONG_USER_CONF = new File(BASE, "jaas_digest_unknown_user.conf").getAbsolutePath(); - private static final String MISSING_CLIENT = new File(BASE, "jaas_digest_missing_client.conf").getAbsolutePath(); + private static final String DIGEST_JAAS_CONF = new File(BASE, "jaas_digest.conf") + .getAbsolutePath(); + private static final String BAD_PASSWORD_CONF = new File(BASE, "jaas_digest_bad_password.conf") + .getAbsolutePath(); + private static final String WRONG_USER_CONF = new File(BASE, "jaas_digest_unknown_user.conf") + .getAbsolutePath(); + private static final String MISSING_CLIENT = new File(BASE, "jaas_digest_missing_client.conf") + .getAbsolutePath(); public static Principal mkPrincipal(final String name) { return new Principal() { @@ -158,24 +173,28 @@ public static void withServer(String loginCfg, } } - public static void verifyIncorrectJaasConf(ThriftServer server, Map conf, String jaas, + public static void verifyIncorrectJaasConf(ThriftServer server, Map conf, + String jaas, Class expectedException) { Map badConf = new HashMap<>(conf); badConf.put("java.security.auth.login.config", jaas); - try (NimbusClient client = NimbusClient.Builder.withConf(badConf).withTimeout(NIMBUS_TIMEOUT) + try (NimbusClient client = NimbusClient.Builder.withConf(badConf) + .withTimeout(NIMBUS_TIMEOUT) .buildWithNimbusHostPort("localhost", server.getPort())) { client.getClient().activate("bad_auth_test_topology"); fail("An exception should have been thrown trying to connect."); } catch (Exception e) { LOG.info("Got Exception...", e); if (!Utils.exceptionCauseIsInstanceOf(expectedException, e)) { - throw new AssertionError("Expecting " + expectedException.getClass().getName() + " but got " + e.getClass().getName(), e); + throw new AssertionError("Expecting " + expectedException.getClass().getName() + + " but got " + e.getClass().getName(), e); } } } public static Subject createSubjectWith(WorkerToken wt) { - //This is a bit ugly, but it shows how this would happen in a worker, so we will use the same APIs + // This is a bit ugly, but it shows how this would happen in a worker, so we will use the + // same APIs Map creds = new HashMap<>(); ClientAuthUtils.setWorkerToken(creds, wt); Subject subject = new Subject(); @@ -183,18 +202,22 @@ public static Subject createSubjectWith(WorkerToken wt) { return subject; } - public static void tryConnectAs(Map conf, ThriftServer server, Subject subject, String topoId) + public static void tryConnectAs(Map conf, ThriftServer server, Subject subject, + String topoId) throws Exception { SubjectCompat.doAs(subject, () -> { - try (NimbusClient client = NimbusClient.Builder.withConf(conf).withTimeout(NIMBUS_TIMEOUT) + try (NimbusClient client = NimbusClient.Builder.withConf(conf) + .withTimeout(NIMBUS_TIMEOUT) .buildWithNimbusHostPort("localhost", server.getPort())) { - client.getClient().activate(topoId); //Yes this should be a topo name, but it makes this simpler... + client.getClient() + .activate(topoId); // Yes this should be a topo name, but it makes this simpler... } return null; }); } - public static Subject testConnectWithTokenFor(WorkerTokenManager wtMan, Map conf, ThriftServer server, + public static Subject testConnectWithTokenFor(WorkerTokenManager wtMan, Map conf, ThriftServer server, String user, String topoId) throws Exception { WorkerToken wt = wtMan.createOrUpdateTokenFor(WorkerTokenServiceType.NIMBUS, user, topoId); Subject subject = createSubjectWith(wt); @@ -203,7 +226,7 @@ public static Subject testConnectWithTokenFor(WorkerTokenManager wtMan, Map user, String userName) { - //The user from the token is bob, so verify that the name was set correctly... + // The user from the token is bob, so verify that the name was set correctly... ReqContext found = user.get(); assertNotNull(found); assertEquals(userName, found.principal().getName()); @@ -211,7 +234,8 @@ public static void verifyUserIs(AtomicReference user, String userNam user.set(null); } - public static ReqContext mkImpersonatingReqContext(String impersonatingUser, String userBeingImpersonated, InetAddress remoteAddress) { + public static ReqContext mkImpersonatingReqContext(String impersonatingUser, + String userBeingImpersonated, InetAddress remoteAddress) { ReqContext ret = new ReqContext(mkSubject(userBeingImpersonated)); ret.setRemoteAddress(remoteAddress); ret.setRealPrincipal(mkPrincipal(impersonatingUser)); @@ -233,24 +257,28 @@ public void simpleAuthTest() throws Exception { withServer(SimpleTransportPlugin.class, impl, (ThriftServer server, Map conf) -> { - try (NimbusClient client = NimbusClient.Builder.withConf(conf).withTimeout(NIMBUS_TIMEOUT) + try (NimbusClient client = NimbusClient.Builder.withConf(conf) + .withTimeout(NIMBUS_TIMEOUT) .buildWithNimbusHostPort("localhost", server.getPort())) { client.getClient().activate("security_auth_test_topology"); } - //Verify digest is rejected... + // Verify digest is rejected... Map badConf = new HashMap<>(conf); - badConf.put(Config.STORM_THRIFT_TRANSPORT_PLUGIN, DigestSaslTransportPlugin.class.getName()); + badConf.put(Config.STORM_THRIFT_TRANSPORT_PLUGIN, + DigestSaslTransportPlugin.class.getName()); badConf.put("java.security.auth.login.config", DIGEST_JAAS_CONF); badConf.put(Config.STORM_NIMBUS_RETRY_TIMES, 0); - try (NimbusClient client = NimbusClient.Builder.withConf(badConf).withTimeout(NIMBUS_TIMEOUT) + try (NimbusClient client = NimbusClient.Builder.withConf(badConf) + .withTimeout(NIMBUS_TIMEOUT) .buildWithNimbusHostPort("localhost", server.getPort())) { client.getClient().activate("bad_security_auth_test_topology"); fail("An exception should have been thrown trying to connect."); } catch (Exception e) { LOG.info("Got Exception...", e); if (!Utils.exceptionCauseIsInstanceOf(TTransportException.class, e)) { - throw new AssertionError("Expecting TTransportException but got " + e.getClass().getName(), e); + throw new AssertionError("Expecting TTransportException but got " + e + .getClass().getName(), e); } } }); @@ -271,36 +299,44 @@ public void digestAuthTest() throws Exception { DigestSaslTransportPlugin.class, impl, (ThriftServer server, Map conf) -> { - try (NimbusClient client = NimbusClient.Builder.withConf(conf).withTimeout(NIMBUS_TIMEOUT) + try (NimbusClient client = NimbusClient.Builder.withConf(conf) + .withTimeout(NIMBUS_TIMEOUT) .buildWithNimbusHostPort("localhost", server.getPort())) { client.getClient().activate("security_auth_test_topology"); } conf.put(Config.STORM_NIMBUS_RETRY_TIMES, 0); - //Verify simple is rejected... + // Verify simple is rejected... Map badTransport = new HashMap<>(conf); - badTransport.put(Config.STORM_THRIFT_TRANSPORT_PLUGIN, SimpleTransportPlugin.class.getName()); - try (NimbusClient client = NimbusClient.Builder.withConf(badTransport).withTimeout(NIMBUS_TIMEOUT) + badTransport.put(Config.STORM_THRIFT_TRANSPORT_PLUGIN, + SimpleTransportPlugin.class.getName()); + try (NimbusClient client = NimbusClient.Builder.withConf(badTransport) + .withTimeout(NIMBUS_TIMEOUT) .buildWithNimbusHostPort("localhost", server.getPort())) { client.getClient().activate("bad_security_auth_test_topology"); fail("An exception should have been thrown trying to connect."); } catch (Exception e) { LOG.info("Got Exception...", e); if (!Utils.exceptionCauseIsInstanceOf(TTransportException.class, e)) { - throw new AssertionError("Expecting TTransportException but got " + e.getClass().getName(), e); + throw new AssertionError("Expecting TTransportException but got " + e + .getClass().getName(), e); } } - //The user here from the jaas conf is bob. No impersonation is done, so verify that + // The user here from the jaas conf is bob. No impersonation is done, so + // verify that ReqContext found = user.get(); assertNotNull(found); assertEquals("bob", found.principal().getName()); assertFalse(found.isImpersonating()); user.set(null); - verifyIncorrectJaasConf(server, conf, BAD_PASSWORD_CONF, TTransportException.class); - verifyIncorrectJaasConf(server, conf, WRONG_USER_CONF, TTransportException.class); - verifyIncorrectJaasConf(server, conf, "./nonexistent.conf", RuntimeException.class); + verifyIncorrectJaasConf(server, conf, BAD_PASSWORD_CONF, + TTransportException.class); + verifyIncorrectJaasConf(server, conf, WRONG_USER_CONF, + TTransportException.class); + verifyIncorrectJaasConf(server, conf, "./nonexistent.conf", + RuntimeException.class); verifyIncorrectJaasConf(server, conf, MISSING_CLIENT, IOException.class); }); verify(impl).activate("security_auth_test_topology"); @@ -318,7 +354,7 @@ public void workerTokenDigestAuthTest() throws Exception { }).when(impl).activate(anyString()); Map extraConfs = new HashMap<>(); - //Let worker tokens work on insecure ZK... + // Let worker tokens work on insecure ZK... extraConfs.put("TESTING.ONLY.ENABLE.INSECURE.WORKER.TOKENS", true); try (InProcessZookeeper zk = new InProcessZookeeper()) { @@ -330,46 +366,55 @@ public void workerTokenDigestAuthTest() throws Exception { (ThriftServer server, Map conf) -> { try (Time.SimulatedTime ignored = new Time.SimulatedTime()) { conf.put(Config.STORM_NIMBUS_RETRY_TIMES, 0); - //We cannot connect if there is no client section in the jaas conf... - try (NimbusClient client = NimbusClient.Builder.withConf(conf).withTimeout(NIMBUS_TIMEOUT) + // We cannot connect if there is no client section in the jaas + // conf... + try (NimbusClient client = NimbusClient.Builder.withConf(conf) + .withTimeout(NIMBUS_TIMEOUT) .buildWithNimbusHostPort("localhost", server.getPort())) { client.getClient().activate("bad_auth_test_topology"); fail("We should not be able to connect without a token..."); } catch (Exception e) { if (!Utils.exceptionCauseIsInstanceOf(IOException.class, e)) { - throw new AssertionError("Expecting IOException but got " + e.getClass().getName(), e); + throw new AssertionError("Expecting IOException but got " + e + .getClass().getName(), e); } } - //Now let's create a token and verify that we can connect... + // Now let's create a token and verify that we can connect... IStormClusterState state = - ClusterUtils.mkStormClusterState(conf, new ClusterStateContext(DaemonType.NIMBUS, conf)); + ClusterUtils.mkStormClusterState(conf, + new ClusterStateContext(DaemonType.NIMBUS, conf)); WorkerTokenManager wtMan = new WorkerTokenManager(conf, state); - Subject bob = testConnectWithTokenFor(wtMan, conf, server, "bob", "topo-bob"); + Subject bob = testConnectWithTokenFor(wtMan, conf, server, "bob", + "topo-bob"); verifyUserIs(user, "bob"); Time.advanceTimeSecs(TimeUnit.HOURS.toSeconds(12)); - //Alice has no digest jaas section at all... - Subject alice = testConnectWithTokenFor(wtMan, conf, server, "alice", "topo-alice"); + // Alice has no digest jaas section at all... + Subject alice = testConnectWithTokenFor(wtMan, conf, server, "alice", + "topo-alice"); verifyUserIs(user, "alice"); Time.advanceTimeSecs(TimeUnit.HOURS.toSeconds(13)); - //Verify that bob's token has expired + // Verify that bob's token has expired try { tryConnectAs(conf, server, bob, "bad_auth_test_topology"); fail("We should not be able to connect with bad auth"); } catch (Exception e) { - if (!Utils.exceptionCauseIsInstanceOf(TTransportException.class, e)) { - throw new AssertionError("Expecting TTransportException but got " + e.getClass().getName(), e); + if (!Utils.exceptionCauseIsInstanceOf(TTransportException.class, + e)) { + throw new AssertionError("Expecting TTransportException " + + "but got " + e.getClass().getName(), e); } } tryConnectAs(conf, server, alice, "topo-alice"); verifyUserIs(user, "alice"); - //Now see if we can create a new token for bob and try again. - bob = testConnectWithTokenFor(wtMan, conf, server, "bob", "topo-bob"); + // Now see if we can create a new token for bob and try again. + bob = testConnectWithTokenFor(wtMan, conf, server, "bob", + "topo-bob"); verifyUserIs(user, "bob"); tryConnectAs(conf, server, alice, "topo-alice"); @@ -520,7 +565,8 @@ public void simpleAclNimbusGroupsAuthTest() { clusterConf.put(Config.NIMBUS_ADMINS_GROUPS, Collections.singletonList("admin-group")); clusterConf.put(Config.NIMBUS_SUPERVISOR_USERS, Collections.singletonList("supervisor")); clusterConf.put(Config.NIMBUS_USERS, Collections.singletonList("user-a")); - clusterConf.put(Config.STORM_GROUP_MAPPING_SERVICE_PROVIDER_PLUGIN, FixedGroupsMapping.class.getName()); + clusterConf.put(Config.STORM_GROUP_MAPPING_SERVICE_PROVIDER_PLUGIN, FixedGroupsMapping.class + .getName()); Map groups = new HashMap<>(); groups.put("admin", Collections.singleton("admin-group")); groups.put("not-admin", Collections.singleton("not-admin-group")); @@ -597,7 +643,8 @@ public void shellBaseGroupsMappingTest() throws Exception { public void getTransportPluginThrowsRunimeTest() { Map conf = ConfigUtils.readStormConfig(); conf.put(Config.STORM_THRIFT_TRANSPORT_PLUGIN, "null.invalid"); - assertThrows(RuntimeException.class, () -> ClientAuthUtils.getTransportPlugin(ThriftConnectionType.NIMBUS, conf)); + assertThrows(RuntimeException.class, () -> ClientAuthUtils + .getTransportPlugin(ThriftConnectionType.NIMBUS, conf)); } @Test @@ -624,27 +671,31 @@ public void impersonationAuthorizerTest() throws Exception { ImpersonationAuthorizer authorizer = new ImpersonationAuthorizer(); authorizer.prepare(clusterConf); - //non impersonating request, should be permitted. + // non impersonating request, should be permitted. assertTrue(authorizer.permit(new ReqContext(mkSubject("anyuser")), "fileUplaod", null)); - //user with no impersonation acl should be rejected - assertFalse(authorizer.permit(mkImpersonatingReqContext("user-with-no-acl", userBeingImpersonated, localHost), + // user with no impersonation acl should be rejected + assertFalse(authorizer.permit(mkImpersonatingReqContext("user-with-no-acl", + userBeingImpersonated, localHost), "someOperation", null)); - //request from hosts that are not authorized should be rejected - assertFalse(authorizer.permit(mkImpersonatingReqContext(impersonatingUser, userBeingImpersonated, unauthorizedHost), + // request from hosts that are not authorized should be rejected + assertFalse(authorizer.permit(mkImpersonatingReqContext(impersonatingUser, + userBeingImpersonated, unauthorizedHost), "someOperation", null)); - //request to impersonate users from unauthorized groups should be rejected. - assertFalse(authorizer.permit(mkImpersonatingReqContext(impersonatingUser, "unauthorized-user", localHost), + // request to impersonate users from unauthorized groups should be rejected. + assertFalse(authorizer.permit(mkImpersonatingReqContext(impersonatingUser, + "unauthorized-user", localHost), "someOperation", null)); - //request from authorized hosts and group should be allowed. - assertTrue(authorizer.permit(mkImpersonatingReqContext(impersonatingUser, userBeingImpersonated, localHost), + // request from authorized hosts and group should be allowed. + assertTrue(authorizer.permit(mkImpersonatingReqContext(impersonatingUser, + userBeingImpersonated, localHost), "someOperation", null)); } public interface MyBiConsumer { void accept(T t, U u) throws Exception; } -} \ No newline at end of file +} diff --git a/storm-server/src/test/java/org/apache/storm/security/auth/DefaultHttpCredentialsPluginTest.java b/storm-server/src/test/java/org/apache/storm/security/auth/DefaultHttpCredentialsPluginTest.java index 93d43b73e02..dc541a1b4ec 100644 --- a/storm-server/src/test/java/org/apache/storm/security/auth/DefaultHttpCredentialsPluginTest.java +++ b/storm-server/src/test/java/org/apache/storm/security/auth/DefaultHttpCredentialsPluginTest.java @@ -1,30 +1,36 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

      - * http://www.apache.org/licenses/LICENSE-2.0 - *

      - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + *

      http://www.apache.org/licenses/LICENSE-2.0 + * + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.security.auth; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import jakarta.servlet.http.HttpServletRequest; import java.security.Principal; import java.util.HashMap; import java.util.HashSet; import javax.security.auth.Subject; -import jakarta.servlet.http.HttpServletRequest; import org.apache.storm.shade.com.google.common.collect.ImmutableSet; import org.junit.jupiter.api.Test; import org.mockito.Mockito; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; - public class DefaultHttpCredentialsPluginTest { @Test @@ -32,9 +38,11 @@ public void test_getUserName() { DefaultHttpCredentialsPlugin handler = new DefaultHttpCredentialsPlugin(); handler.prepare(new HashMap<>()); - assertNull(handler.getUserName((HttpServletRequest) null), "Should return null when request is null"); + assertNull(handler.getUserName((HttpServletRequest) null), + "Should return null when request is null"); - assertNull(handler.getUserName(Mockito.mock(HttpServletRequest.class)), "Should return null when user principal is null"); + assertNull(handler.getUserName(Mockito.mock(HttpServletRequest.class)), + "Should return null when user principal is null"); HttpServletRequest mockRequest = Mockito.mock(HttpServletRequest.class); Mockito.when(mockRequest.getUserPrincipal()).thenReturn(new SingleUserPrincipal("")); @@ -43,12 +51,14 @@ public void test_getUserName() { String expName = "Alice"; mockRequest = Mockito.mock(HttpServletRequest.class); Mockito.when(mockRequest.getUserPrincipal()).thenReturn(new SingleUserPrincipal(expName)); - assertEquals(expName, handler.getUserName(mockRequest), "Should return correct user from requests principal"); + assertEquals(expName, handler.getUserName(mockRequest), + "Should return correct user from requests principal"); try { String doAsUserName = "Bob"; mockRequest = Mockito.mock(HttpServletRequest.class); - Mockito.when(mockRequest.getUserPrincipal()).thenReturn(new SingleUserPrincipal(expName)); + Mockito.when(mockRequest.getUserPrincipal()) + .thenReturn(new SingleUserPrincipal(expName)); Mockito.when(mockRequest.getHeader("doAsUser")).thenReturn(doAsUserName); ReqContext context = handler.populateContext(ReqContext.context(), mockRequest); @@ -66,7 +76,8 @@ public void test_populate_req_context_on_null_user() { DefaultHttpCredentialsPlugin handler = new DefaultHttpCredentialsPlugin(); handler.prepare(new HashMap<>()); Subject subject = - new Subject(false, ImmutableSet.of(new SingleUserPrincipal("test")), new HashSet<>(), new HashSet<>()); + new Subject(false, ImmutableSet.of(new SingleUserPrincipal("test")), + new HashSet<>(), new HashSet<>()); ReqContext context = new ReqContext(subject); diff --git a/storm-server/src/test/java/org/apache/storm/security/auth/NimbusAuthTest.java b/storm-server/src/test/java/org/apache/storm/security/auth/NimbusAuthTest.java index f8826c69480..545f2100b97 100644 --- a/storm-server/src/test/java/org/apache/storm/security/auth/NimbusAuthTest.java +++ b/storm-server/src/test/java/org/apache/storm/security/auth/NimbusAuthTest.java @@ -18,10 +18,12 @@ package org.apache.storm.security.auth; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + import java.util.HashMap; import java.util.Map; import java.util.Optional; - import org.apache.storm.Config; import org.apache.storm.DaemonConfig; import org.apache.storm.LocalCluster; @@ -40,12 +42,10 @@ import org.mockito.ArgumentMatchers; import org.mockito.Mockito; -import static org.junit.jupiter.api.Assertions.*; - /** * Tests for Nimbus authentication and authorization with various transport plugins. * - * Ported from storm-core/test/clj/org/apache/storm/security/auth/nimbus_auth_test.clj + *

      Ported from storm-core/test/clj/org/apache/storm/security/auth/nimbus_auth_test.clj */ public class NimbusAuthTest { @@ -53,7 +53,8 @@ public class NimbusAuthTest { private static final String JAAS_CONF = jaasConfPath(); private static String jaasConfPath() { - java.net.URL url = NimbusAuthTest.class.getResource("/org/apache/storm/security/auth/jaas_digest.conf"); + java.net.URL url = NimbusAuthTest.class + .getResource("/org/apache/storm/security/auth/jaas_digest.conf"); if (url == null) { throw new RuntimeException("jaas_digest.conf not found on classpath"); } @@ -83,7 +84,8 @@ private static void assertThrowsCause(Class expectedCause, ThrowingRunnable action) { try { action.run(); - fail("Expected exception with cause " + expectedCause.getSimpleName() + " but no exception was thrown"); + fail("Expected exception with cause " + expectedCause.getSimpleName() + + " but no exception was thrown"); } catch (Exception e) { assertTrue(hasCause(e, expectedCause), "Expected cause " + expectedCause.getSimpleName() + " but got: " + e); @@ -115,7 +117,8 @@ public void testSimpleAuthentication() throws Exception { try (NimbusClient client = new NimbusClient(clientConf, "localhost", cluster.getThriftServerPort(), NIMBUS_TIMEOUT)) { Nimbus.Iface nimbusClient = client.getClient(); - assertThrowsCause(NotAliveException.class, () -> nimbusClient.activate("topo-name")); + assertThrowsCause(NotAliveException.class, () -> nimbusClient + .activate("topo-name")); } } } @@ -197,12 +200,14 @@ public void testDenyAuthorizationWithSimpleTransport() throws Exception { try (NimbusClient client = new NimbusClient(clientConf, "localhost", cluster.getThriftServerPort(), NIMBUS_TIMEOUT)) { Nimbus.Iface nimbusClient = client.getClient(); - SubmitOptions submitOptions = new SubmitOptions(TopologyInitialStatus.findByValue(2)); + SubmitOptions submitOptions = new SubmitOptions(TopologyInitialStatus + .findByValue(2)); assertThrowsCause(AuthorizationException.class, () -> nimbusClient.submitTopology(topoName, null, null, null)); assertThrowsCause(AuthorizationException.class, - () -> nimbusClient.submitTopologyWithOpts(topoName, null, null, null, submitOptions)); + () -> nimbusClient.submitTopologyWithOpts(topoName, null, null, null, + submitOptions)); assertThrowsCause(AuthorizationException.class, () -> nimbusClient.beginFileUpload()); assertThrowsCause(AuthorizationException.class, @@ -268,7 +273,8 @@ public void testNoopAuthorizationWithSaslDigest() throws Exception { try (NimbusClient client = new NimbusClient(clientConf, "localhost", cluster.getThriftServerPort(), NIMBUS_TIMEOUT)) { Nimbus.Iface nimbusClient = client.getClient(); - assertThrowsCause(NotAliveException.class, () -> nimbusClient.activate("topo-name")); + assertThrowsCause(NotAliveException.class, () -> nimbusClient + .activate("topo-name")); } } } @@ -315,12 +321,14 @@ public void testDenyAuthorizationWithSaslDigest() throws Exception { try (NimbusClient client = new NimbusClient(clientConf, "localhost", cluster.getThriftServerPort(), NIMBUS_TIMEOUT)) { Nimbus.Iface nimbusClient = client.getClient(); - SubmitOptions submitOptions = new SubmitOptions(TopologyInitialStatus.findByValue(2)); + SubmitOptions submitOptions = new SubmitOptions(TopologyInitialStatus + .findByValue(2)); assertThrowsCause(AuthorizationException.class, () -> nimbusClient.submitTopology(topoName, null, null, null)); assertThrowsCause(AuthorizationException.class, - () -> nimbusClient.submitTopologyWithOpts(topoName, null, null, null, submitOptions)); + () -> nimbusClient.submitTopologyWithOpts(topoName, null, null, null, + submitOptions)); assertThrowsCause(AuthorizationException.class, () -> nimbusClient.beginFileUpload()); assertThrowsCause(AuthorizationException.class, diff --git a/storm-server/src/test/java/org/apache/storm/security/auth/ServerAuthUtilsTest.java b/storm-server/src/test/java/org/apache/storm/security/auth/ServerAuthUtilsTest.java index 960d005059f..6f9097159da 100644 --- a/storm-server/src/test/java/org/apache/storm/security/auth/ServerAuthUtilsTest.java +++ b/storm-server/src/test/java/org/apache/storm/security/auth/ServerAuthUtilsTest.java @@ -18,13 +18,12 @@ package org.apache.storm.security.auth; -import org.apache.storm.DaemonConfig; -import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertSame; import java.util.HashMap; import java.util.Map; - -import static org.junit.jupiter.api.Assertions.assertSame; +import org.apache.storm.DaemonConfig; +import org.junit.jupiter.api.Test; public class ServerAuthUtilsTest { @@ -32,7 +31,7 @@ public static class AuthUtilsTestMock implements IHttpCredentialsPlugin { @Override public void prepare(Map topoConf) { - //NO OP + // NO OP } @Override @@ -41,7 +40,8 @@ public String getUserName(jakarta.servlet.http.HttpServletRequest req) { } @Override - public ReqContext populateContext(ReqContext context, jakarta.servlet.http.HttpServletRequest req) { + public ReqContext populateContext(ReqContext context, + jakarta.servlet.http.HttpServletRequest req) { return null; } } @@ -54,7 +54,9 @@ public void uiHttpCredentialsPluginTest() { conf.put( DaemonConfig.DRPC_HTTP_CREDS_PLUGIN, AuthUtilsTestMock.class.getName()); - assertSame(ServerAuthUtils.getUiHttpCredentialsPlugin(conf).getClass(), AuthUtilsTestMock.class); - assertSame(ServerAuthUtils.getDrpcHttpCredentialsPlugin(conf).getClass(), AuthUtilsTestMock.class); + assertSame(ServerAuthUtils.getUiHttpCredentialsPlugin(conf).getClass(), + AuthUtilsTestMock.class); + assertSame(ServerAuthUtils.getDrpcHttpCredentialsPlugin(conf).getClass(), + AuthUtilsTestMock.class); } -} \ No newline at end of file +} diff --git a/storm-server/src/test/java/org/apache/storm/security/auth/workertoken/WorkerTokenTest.java b/storm-server/src/test/java/org/apache/storm/security/auth/workertoken/WorkerTokenTest.java index ec38c8e79a6..b59ef0d2328 100644 --- a/storm-server/src/test/java/org/apache/storm/security/auth/workertoken/WorkerTokenTest.java +++ b/storm-server/src/test/java/org/apache/storm/security/auth/workertoken/WorkerTokenTest.java @@ -18,6 +18,18 @@ package org.apache.storm.security.auth.workertoken; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + import java.util.HashMap; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -31,13 +43,6 @@ import org.apache.storm.utils.Time; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertArrayEquals; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; -import static org.mockito.Mockito.*; - public class WorkerTokenTest { public static final long ONE_DAY_MILLIS = TimeUnit.HOURS.toMillis(24); @@ -48,7 +53,7 @@ public void testBasicGenerateAndAuthorize() { final String userName = "user"; final WorkerTokenServiceType type = WorkerTokenServiceType.NIMBUS; final long versionNumber = 0L; - //Simulate time starts out at 0, so we are going to just leave it here. + // Simulate time starts out at 0, so we are going to just leave it here. try (Time.SimulatedTime ignored = new Time.SimulatedTime()) { IStormClusterState mockState = mock(IStormClusterState.class); Map conf = new HashMap<>(); @@ -56,15 +61,19 @@ public void testBasicGenerateAndAuthorize() { when(mockState.getNextPrivateWorkerKeyVersion(type, topoId)).thenReturn(versionNumber); doAnswer((invocation) -> { - //Save the private worker key away so we can test it too. + // Save the private worker key away so we can test it too. privateKey.set(invocation.getArgument(3)); return null; - }).when(mockState).addPrivateWorkerKey(eq(type), eq(topoId), eq(versionNumber), any(PrivateWorkerKey.class)); - //Answer when we ask for a private key... - when(mockState.getPrivateWorkerKey(type, topoId, versionNumber)).thenAnswer((invocation) -> privateKey.get()); + }).when(mockState) + .addPrivateWorkerKey(eq(type), eq(topoId), eq(versionNumber), + any(PrivateWorkerKey.class)); + // Answer when we ask for a private key... + when(mockState.getPrivateWorkerKey(type, topoId, versionNumber)) + .thenAnswer((invocation) -> privateKey.get()); WorkerToken wt = wtm.createOrUpdateTokenFor(type, userName, topoId); - verify(mockState).addPrivateWorkerKey(eq(type), eq(topoId), eq(versionNumber), any(PrivateWorkerKey.class)); + verify(mockState).addPrivateWorkerKey(eq(type), eq(topoId), eq(versionNumber), + any(PrivateWorkerKey.class)); assertTrue(wt.is_set_serviceType()); assertEquals(type, wt.get_serviceType()); assertTrue(wt.is_set_info()); @@ -86,7 +95,7 @@ public void testBasicGenerateAndAuthorize() { assertEquals(versionNumber, info.get_secretVersion()); try (WorkerTokenAuthorizer wta = new WorkerTokenAuthorizer(type, mockState)) { - //Verify the signature... + // Verify the signature... byte[] signature = wta.getSignedPasswordFor(wt.get_info(), info); assertArrayEquals(wt.get_signature(), signature); } @@ -100,7 +109,7 @@ public void testExpiration() { final String userName = "user"; final WorkerTokenServiceType type = WorkerTokenServiceType.NIMBUS; final long versionNumber = 5L; - //Simulate time starts out at 0, so we are going to just leave it here. + // Simulate time starts out at 0, so we are going to just leave it here. try (Time.SimulatedTime ignored = new Time.SimulatedTime()) { IStormClusterState mockState = mock(IStormClusterState.class); Map conf = new HashMap<>(); @@ -108,15 +117,19 @@ public void testExpiration() { when(mockState.getNextPrivateWorkerKeyVersion(type, topoId)).thenReturn(versionNumber); doAnswer((invocation) -> { - //Save the private worker key away so we can test it too. + // Save the private worker key away so we can test it too. privateKey.set(invocation.getArgument(3)); return null; - }).when(mockState).addPrivateWorkerKey(eq(type), eq(topoId), eq(versionNumber), any(PrivateWorkerKey.class)); - //Answer when we ask for a private key... - when(mockState.getPrivateWorkerKey(type, topoId, versionNumber)).thenAnswer((invocation) -> privateKey.get()); + }).when(mockState) + .addPrivateWorkerKey(eq(type), eq(topoId), eq(versionNumber), + any(PrivateWorkerKey.class)); + // Answer when we ask for a private key... + when(mockState.getPrivateWorkerKey(type, topoId, versionNumber)) + .thenAnswer((invocation) -> privateKey.get()); WorkerToken wt = wtm.createOrUpdateTokenFor(type, userName, topoId); - verify(mockState).addPrivateWorkerKey(eq(type), eq(topoId), eq(versionNumber), any(PrivateWorkerKey.class)); + verify(mockState).addPrivateWorkerKey(eq(type), eq(topoId), eq(versionNumber), + any(PrivateWorkerKey.class)); assertTrue(wt.is_set_serviceType()); assertEquals(type, wt.get_serviceType()); assertTrue(wt.is_set_info()); @@ -137,23 +150,24 @@ public void testExpiration() { assertEquals(ONE_DAY_MILLIS, info.get_expirationTimeMillis()); assertEquals(versionNumber, info.get_secretVersion()); - //Expire the token + // Expire the token Time.advanceTime(ONE_DAY_MILLIS + 1); try (WorkerTokenAuthorizer wta = new WorkerTokenAuthorizer(type, mockState)) { try { - //Verify the signature... + // Verify the signature... wta.getSignedPasswordFor(wt.get_info(), info); fail("Expected an expired token to not be signed!!!"); } catch (IllegalArgumentException ia) { - //What we want... + // What we want... } } - //Verify if WorkerTokenManager recognizes the expired WorkerToken. + // Verify if WorkerTokenManager recognizes the expired WorkerToken. Map creds = new HashMap<>(); ClientAuthUtils.setWorkerToken(creds, wt); - assertTrue(wtm.shouldRenewWorkerToken(creds, type), "Expired WorkerToken should be eligible for renewal"); + assertTrue(wtm.shouldRenewWorkerToken(creds, type), + "Expired WorkerToken should be eligible for renewal"); } } } diff --git a/storm-server/src/test/java/org/apache/storm/trident/TridentIntegrationTest.java b/storm-server/src/test/java/org/apache/storm/trident/TridentIntegrationTest.java index e39d8ce59ef..f7414566555 100644 --- a/storm-server/src/test/java/org/apache/storm/trident/TridentIntegrationTest.java +++ b/storm-server/src/test/java/org/apache/storm/trident/TridentIntegrationTest.java @@ -18,16 +18,23 @@ package org.apache.storm.trident; +import static org.awaitility.Awaitility.await; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; - +import net.minidev.json.JSONValue; +import net.minidev.json.parser.JSONParser; +import net.minidev.json.parser.ParseException; import org.apache.storm.Config; -import org.apache.storm.LocalCluster; import org.apache.storm.LocalCluster.LocalTopology; +import org.apache.storm.LocalCluster; import org.apache.storm.LocalDRPC; import org.apache.storm.Testing; import org.apache.storm.generated.Bolt; @@ -50,18 +57,10 @@ import org.apache.storm.trident.tuple.TridentTuple; import org.apache.storm.tuple.Fields; import org.apache.storm.tuple.Values; -import net.minidev.json.JSONValue; -import net.minidev.json.parser.JSONParser; -import net.minidev.json.parser.ParseException; import org.junit.jupiter.api.Test; -import static org.awaitility.Awaitility.await; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; - /** - * Ported from storm-core/test/clj/integration/org/apache/storm/trident/integration_test.clj + * Ported from storm-core/test/clj/integration/org/apache/storm/trident/integration_test.clj. */ public class TridentIntegrationTest { @@ -75,7 +74,8 @@ public void execute(TridentTuple tuple, TridentCollector collector) { } @Override - public void prepare(Map conf, org.apache.storm.trident.operation.TridentOperationContext context) { + public void prepare(Map conf, + org.apache.storm.trident.operation.TridentOperationContext context) { } @Override @@ -84,7 +84,8 @@ public void cleanup() { } @SuppressWarnings("unchecked") - private static List> execDrpc(LocalDRPC drpc, String functionName, String args) throws Exception { + private static List> execDrpc(LocalDRPC drpc, String functionName, + String args) throws Exception { String res = drpc.execute(functionName, args); if (res == null) { return null; @@ -107,7 +108,8 @@ private static List> execDrpc(LocalDRPC drpc, String functionName, return result; } - private static Set> execDrpcAsSet(LocalDRPC drpc, String functionName, String args) throws Exception { + private static Set> execDrpcAsSet(LocalDRPC drpc, String functionName, + String args) throws Exception { List> result = execDrpc(drpc, functionName, args); return new HashSet<>(result); } @@ -123,15 +125,18 @@ public void testMemoryMapGetTuples() throws Exception { .newStream("tester", feeder) .each(new Fields("sentence"), new Split(), new Fields("word")) .groupBy(new Fields("word")) - .persistentAggregate(new MemoryMapState.Factory(), new Count(), new Fields("count")) + .persistentAggregate(new MemoryMapState.Factory(), new Count(), + new Fields("count")) .parallelismHint(6); topo.newDRPCStream("all-tuples", drpc) .broadcast() - .stateQuery(wordCounts, new Fields("args"), new TupleCollectionGet(), new Fields("word", "count")) + .stateQuery(wordCounts, new Fields("args"), new TupleCollectionGet(), + new Fields("word", "count")) .project(new Fields("word", "count")); - try (LocalTopology stormTopo = cluster.submitTopology("testing", Map.of(), topo.build())) { + try (LocalTopology stormTopo = cluster.submitTopology("testing", Map.of(), topo + .build())) { feeder.feed(Arrays.asList(new Values("hello the man said"), new Values("the"))); Set> expected1 = Set.of( @@ -168,7 +173,8 @@ public void testWordCount() throws Exception { .newStream("tester", feeder) .each(new Fields("sentence"), new Split(), new Fields("word")) .groupBy(new Fields("word")) - .persistentAggregate(new MemoryMapState.Factory(), new Count(), new Fields("count")) + .persistentAggregate(new MemoryMapState.Factory(), new Count(), + new Fields("count")) .parallelismHint(6); topo.newDRPCStream("words", drpc) @@ -178,17 +184,20 @@ public void testWordCount() throws Exception { .aggregate(new Fields("count"), new Sum(), new Fields("sum")) .project(new Fields("sum")); - try (LocalTopology stormTopo = cluster.submitTopology("testing", Map.of(), topo.build())) { + try (LocalTopology stormTopo = cluster.submitTopology("testing", Map.of(), topo + .build())) { feeder.feed(Arrays.asList(new Values("hello the man said"), new Values("the"))); assertEquals(List.of(List.of(2L)), execDrpc(drpc, "words", "the")); assertEquals(List.of(List.of(1L)), execDrpc(drpc, "words", "hello")); - feeder.feed(Arrays.asList(new Values("the man on the moon"), new Values("where are you"))); + feeder.feed(Arrays.asList(new Values("the man on the moon"), + new Values("where are you"))); assertEquals(List.of(List.of(4L)), execDrpc(drpc, "words", "the")); assertEquals(List.of(List.of(2L)), execDrpc(drpc, "words", "man")); - assertEquals(List.of(List.of(8L)), execDrpc(drpc, "words", "man where you the")); + assertEquals(List.of(List.of(8L)), execDrpc(drpc, "words", + "man where you the")); } } } @@ -203,7 +212,8 @@ public void testWordCountCommitterSpout() throws Exception { try (LocalCluster cluster = new LocalCluster()) { try (LocalDRPC drpc = new LocalDRPC(cluster.getMetricRegistry())) { TridentTopology topo = new TridentTopology(); - FeederCommitterBatchSpout feeder = new FeederCommitterBatchSpout(Arrays.asList("sentence")); + FeederCommitterBatchSpout feeder = new FeederCommitterBatchSpout(Arrays + .asList("sentence")); feeder.setWaitToEmit(false); // this causes lots of empty batches TridentState wordCounts = topo @@ -211,7 +221,8 @@ public void testWordCountCommitterSpout() throws Exception { .parallelismHint(2) .each(new Fields("sentence"), new Split(), new Fields("word")) .groupBy(new Fields("word")) - .persistentAggregate(new MemoryMapState.Factory(), new Count(), new Fields("count")) + .persistentAggregate(new MemoryMapState.Factory(), new Count(), + new Fields("count")) .parallelismHint(6); topo.newDRPCStream("words", drpc) @@ -221,7 +232,8 @@ public void testWordCountCommitterSpout() throws Exception { .aggregate(new Fields("count"), new Sum(), new Fields("sum")) .project(new Fields("sum")); - try (LocalTopology stormTopo = cluster.submitTopology("testing", Map.of(), topo.build())) { + try (LocalTopology stormTopo = cluster.submitTopology("testing", Map.of(), topo + .build())) { feeder.feed(Arrays.asList(new Values("hello the man said"), new Values("the"))); assertEquals(List.of(List.of(2L)), execDrpc(drpc, "words", "the")); @@ -237,11 +249,13 @@ public void testWordCountCommitterSpout() throws Exception { assertEquals(List.of(List.of(2L)), result); }); - feeder.feed(Arrays.asList(new Values("the man on the moon"), new Values("where are you"))); + feeder.feed(Arrays.asList(new Values("the man on the moon"), + new Values("where are you"))); assertEquals(List.of(List.of(4L)), execDrpc(drpc, "words", "the")); assertEquals(List.of(List.of(2L)), execDrpc(drpc, "words", "man")); - assertEquals(List.of(List.of(8L)), execDrpc(drpc, "words", "man where you the")); + assertEquals(List.of(List.of(8L)), execDrpc(drpc, "words", + "man where you the")); feeder.feed(Arrays.asList(new Values("the the"))); assertEquals(List.of(List.of(6L)), execDrpc(drpc, "words", "the")); @@ -265,12 +279,14 @@ public void testCountAgg() throws Exception { .parallelismHint(2) // this makes sure batchGlobal is working correctly .project(new Fields("count")); - try (LocalTopology stormTopo = cluster.submitTopology("testing", Map.of(), topo.build())) { + try (LocalTopology stormTopo = cluster.submitTopology("testing", Map.of(), topo + .build())) { for (int i = 0; i < 100; i++) { assertEquals(List.of(List.of(1L)), execDrpc(drpc, "numwords", "the")); } assertEquals(List.of(List.of(0L)), execDrpc(drpc, "numwords", "")); - assertEquals(List.of(List.of(8L)), execDrpc(drpc, "numwords", "1 2 3 4 5 6 7 8")); + assertEquals(List.of(List.of(8L)), execDrpc(drpc, "numwords", + "1 2 3 4 5 6 7 8")); } } } @@ -293,9 +309,11 @@ public void testSplitMerge() throws Exception { topo.merge(s1, s2); - try (LocalTopology stormTopo = cluster.submitTopology("testing", Map.of(), topo.build())) { + try (LocalTopology stormTopo = cluster.submitTopology("testing", Map.of(), topo + .build())) { assertTrue(Testing.multiseteq( - Arrays.asList(Arrays.asList(7L), Arrays.asList("the"), Arrays.asList("man")), + Arrays.asList(Arrays.asList(7L), Arrays.asList("the"), Arrays + .asList("man")), execDrpc(drpc, "splitter", "the man"))); assertTrue(Testing.multiseteq( Arrays.asList(Arrays.asList(5L), Arrays.asList("hello")), @@ -323,7 +341,8 @@ public void testMultipleGroupingsSameStream() throws Exception { topo.merge(s1, s2); - try (LocalTopology stormTopo = cluster.submitTopology("testing", Map.of(), topo.build())) { + try (LocalTopology stormTopo = cluster.submitTopology("testing", Map.of(), topo + .build())) { assertTrue(Testing.multiseteq( Arrays.asList(Arrays.asList("the", 1L), Arrays.asList("the", 1L)), execDrpc(drpc, "tester", "the"))); @@ -346,7 +365,8 @@ public void testMultiRepartition() throws Exception { .shuffle() .aggregate(new CountAsAggregator(), new Fields("count")); - try (LocalTopology stormTopo = cluster.submitTopology("testing", Map.of(), topo.build())) { + try (LocalTopology stormTopo = cluster.submitTopology("testing", Map.of(), topo + .build())) { assertTrue(Testing.multiseteq( List.of(List.of(2L)), execDrpc(drpc, "tester", "the man"))); @@ -361,7 +381,8 @@ public void testMultiRepartition() throws Exception { @Test public void testStreamProjectionValidation() throws Exception { try (LocalCluster cluster = new LocalCluster()) { - FeederCommitterBatchSpout feeder = new FeederCommitterBatchSpout(Arrays.asList("sentence")); + FeederCommitterBatchSpout feeder = new FeederCommitterBatchSpout(Arrays + .asList("sentence")); TridentTopology topo = new TridentTopology(); // valid projection fields will not throw exceptions @@ -423,7 +444,8 @@ public void testStreamProjectionValidation() throws Exception { topo.newDRPCStream("words", drpc) .each(new Fields("args"), new Split(), new Fields("word")) .groupBy(new Fields("word")) - .stateQuery(wordCounts, new Fields("word1"), new MapGet(), new Fields("count"))); + .stateQuery(wordCounts, new Fields("word1"), new MapGet(), + new Fields("count"))); } } } @@ -455,7 +477,8 @@ public void testSetComponentResources() throws Exception { .setCPULoad(50) .setMemoryLoad(1024) .groupBy(new Fields("word!")) - .persistentAggregate(new MemoryMapState.Factory(), new Count(), new Fields("count")) + .persistentAggregate(new MemoryMapState.Factory(), new Count(), + new Fields("count")) .setCPULoad(100) .setMemoryLoad(2048); @@ -463,42 +486,53 @@ public void testSetComponentResources() throws Exception { Config.TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB, 4096.0 ); - try (LocalTopology stormTopo = cluster.submitTopology("testing", conf, topo.build())) { + try (LocalTopology stormTopo = cluster.submitTopology("testing", conf, topo + .build())) { Map bolts = stormTopo.get_bolts(); JSONParser parser = new JSONParser(); // Helper to get JSON conf for a bolt - java.util.function.Function> getJsonConf = (boltId) -> { - try { - Bolt bolt = bolts.get(boltId); - return (Map) parser.parse(bolt.get_common().get_json_conf()); - } catch (ParseException e) { - throw new RuntimeException(e); - } - }; + java.util.function.Function> getJsonConf = + (boltId) -> { + try { + Bolt bolt = bolts.get(boltId); + return (Map) parser.parse(bolt.get_common() + .get_json_conf()); + } catch (ParseException e) { + throw new RuntimeException(e); + } + }; // spout memory Map spoutConf = getJsonConf.apply("spout-words"); - assertEquals(512.0, spoutConf.get(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB)); - assertEquals(256.0, spoutConf.get(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB)); - - Map spoutCoordConf = getJsonConf.apply("$spoutcoord-spout-words"); - assertEquals(512.0, spoutCoordConf.get(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB)); - assertEquals(256.0, spoutCoordConf.get(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB)); + assertEquals(512.0, spoutConf + .get(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB)); + assertEquals(256.0, spoutConf + .get(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB)); + + Map spoutCoordConf = getJsonConf + .apply("$spoutcoord-spout-words"); + assertEquals(512.0, spoutCoordConf + .get(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB)); + assertEquals(256.0, spoutCoordConf + .get(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB)); // spout CPU assertEquals(20.0, spoutConf.get(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT)); - assertEquals(20.0, spoutCoordConf.get(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT)); + assertEquals(20.0, spoutCoordConf + .get(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT)); // bolt combinations (b-1 combines Split + addBang: 10+50 CPU, 512+1024 memory) Map b1Conf = getJsonConf.apply("b-1"); - assertEquals(1024.0 + 512.0, b1Conf.get(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB)); + assertEquals(1024.0 + 512.0, b1Conf + .get(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB)); assertEquals(60.0, b1Conf.get(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT)); // aggregations after partition (b-0 = persistentAggregate) Map b0Conf = getJsonConf.apply("b-0"); - assertEquals(2048.0, b0Conf.get(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB)); + assertEquals(2048.0, b0Conf + .get(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB)); assertEquals(100.0, b0Conf.get(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT)); } } diff --git a/storm-server/src/test/java/org/apache/storm/utils/EquivalenceUtilsTest.java b/storm-server/src/test/java/org/apache/storm/utils/EquivalenceUtilsTest.java index 6bcf85e3a2b..a8f731d8bbc 100644 --- a/storm-server/src/test/java/org/apache/storm/utils/EquivalenceUtilsTest.java +++ b/storm-server/src/test/java/org/apache/storm/utils/EquivalenceUtilsTest.java @@ -18,8 +18,10 @@ package org.apache.storm.utils; -import com.google.common.collect.Maps; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import com.google.common.collect.Maps; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -28,12 +30,11 @@ import org.apache.storm.generated.LocalAssignment; import org.apache.storm.generated.WorkerResources; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; public class EquivalenceUtilsTest { - static WorkerResources mkWorkerResources(Double cpu, Double mem_on_heap, Double mem_off_heap, Map resources) { + static WorkerResources mkWorkerResources(Double cpu, Double mem_on_heap, Double mem_off_heap, + Map resources) { WorkerResources workerResources = mkWorkerResources(cpu, mem_on_heap, mem_off_heap); if (resources != null) { workerResources.set_resources(resources); @@ -57,7 +58,8 @@ static WorkerResources mkWorkerResources(Double cpu, Double mem_on_heap, Double return resources; } - static LocalAssignment mkLocalAssignment(String id, List exec, WorkerResources resources) { + static LocalAssignment mkLocalAssignment(String id, List exec, + WorkerResources resources) { LocalAssignment ret = new LocalAssignment(); ret.set_topology_id(id); ret.set_executors(exec); @@ -82,62 +84,80 @@ static List mkExecutorInfoList(int... executors) { public void testWorkerResourceEquality() { WorkerResources resourcesRNull = mkWorkerResources(100.0, 100.0, 100.0, null); WorkerResources resourcesREmpty = mkWorkerResources(100.0, 100.0, 100.0, Maps.newHashMap()); - assertTrue(EquivalenceUtils.customWorkerResourcesEquality(resourcesRNull,resourcesREmpty)); + assertTrue(EquivalenceUtils.customWorkerResourcesEquality(resourcesRNull, resourcesREmpty)); Map resources = new HashMap<>(); resources.put("network.resource.units", 0.0); - WorkerResources resourcesRNetwork = mkWorkerResources(100.0, 100.0, 100.0,resources); - assertTrue(EquivalenceUtils.customWorkerResourcesEquality(resourcesREmpty, resourcesRNetwork)); + WorkerResources resourcesRNetwork = mkWorkerResources(100.0, 100.0, 100.0, resources); + assertTrue(EquivalenceUtils.customWorkerResourcesEquality(resourcesREmpty, + resourcesRNetwork)); Map resourcesNetwork = new HashMap<>(); resourcesNetwork.put("network.resource.units", 50.0); - WorkerResources resourcesRNetworkNonZero = mkWorkerResources(100.0, 100.0, 100.0,resourcesNetwork); - assertFalse(EquivalenceUtils.customWorkerResourcesEquality(resourcesREmpty, resourcesRNetworkNonZero)); + WorkerResources resourcesRNetworkNonZero = mkWorkerResources(100.0, 100.0, 100.0, + resourcesNetwork); + assertFalse(EquivalenceUtils.customWorkerResourcesEquality(resourcesREmpty, + resourcesRNetworkNonZero)); Map resourcesNetworkOne = new HashMap<>(); resourcesNetworkOne.put("network.resource.units", 50.0); - WorkerResources resourcesRNetworkOne = mkWorkerResources(100.0, 100.0, 100.0,resourcesNetworkOne); - assertTrue(EquivalenceUtils.customWorkerResourcesEquality(resourcesRNetworkOne, resourcesRNetworkNonZero)); + WorkerResources resourcesRNetworkOne = mkWorkerResources(100.0, 100.0, 100.0, + resourcesNetworkOne); + assertTrue(EquivalenceUtils.customWorkerResourcesEquality(resourcesRNetworkOne, + resourcesRNetworkNonZero)); Map resourcesNetworkTwo = new HashMap<>(); resourcesNetworkTwo.put("network.resource.units", 100.0); - WorkerResources resourcesRNetworkTwo = mkWorkerResources(100.0, 100.0, 100.0,resourcesNetworkTwo); - assertFalse(EquivalenceUtils.customWorkerResourcesEquality(resourcesRNetworkOne, resourcesRNetworkTwo)); - - WorkerResources resourcesCpuNull = mkWorkerResources(null, 100.0,100.0); - WorkerResources resourcesCPUZero = mkWorkerResources(0.0, 100.0,100.0); - assertTrue(EquivalenceUtils.customWorkerResourcesEquality(resourcesCpuNull, resourcesCPUZero)); - - WorkerResources resourcesOnHeapMemNull = mkWorkerResources(100.0, null,100.0); - WorkerResources resourcesOnHeapMemZero = mkWorkerResources(100.0, 0.0,100.0); - assertTrue(EquivalenceUtils.customWorkerResourcesEquality(resourcesOnHeapMemNull, resourcesOnHeapMemZero)); - - WorkerResources resourcesOffHeapMemNull = mkWorkerResources(100.0, 100.0,null); - WorkerResources resourcesOffHeapMemZero = mkWorkerResources(100.0, 100.0,0.0); - assertTrue(EquivalenceUtils.customWorkerResourcesEquality(resourcesOffHeapMemNull, resourcesOffHeapMemZero)); + WorkerResources resourcesRNetworkTwo = mkWorkerResources(100.0, 100.0, 100.0, + resourcesNetworkTwo); + assertFalse(EquivalenceUtils.customWorkerResourcesEquality(resourcesRNetworkOne, + resourcesRNetworkTwo)); + + WorkerResources resourcesCpuNull = mkWorkerResources(null, 100.0, 100.0); + WorkerResources resourcesCPUZero = mkWorkerResources(0.0, 100.0, 100.0); + assertTrue(EquivalenceUtils.customWorkerResourcesEquality(resourcesCpuNull, + resourcesCPUZero)); + + WorkerResources resourcesOnHeapMemNull = mkWorkerResources(100.0, null, 100.0); + WorkerResources resourcesOnHeapMemZero = mkWorkerResources(100.0, 0.0, 100.0); + assertTrue(EquivalenceUtils.customWorkerResourcesEquality(resourcesOnHeapMemNull, + resourcesOnHeapMemZero)); + + WorkerResources resourcesOffHeapMemNull = mkWorkerResources(100.0, 100.0, null); + WorkerResources resourcesOffHeapMemZero = mkWorkerResources(100.0, 100.0, 0.0); + assertTrue(EquivalenceUtils.customWorkerResourcesEquality(resourcesOffHeapMemNull, + resourcesOffHeapMemZero)); assertFalse(EquivalenceUtils.customWorkerResourcesEquality(resourcesOffHeapMemNull, null)); } @Test public void testEquivalent() { - LocalAssignment a = mkLocalAssignment("A", mkExecutorInfoList(1, 2, 3, 4, 5), mkWorkerResources(100.0, 100.0, 100.0)); - LocalAssignment aResized = mkLocalAssignment("A", mkExecutorInfoList(1, 2, 3, 4, 5), mkWorkerResources(100.0, 200.0, 100.0)); - LocalAssignment b = mkLocalAssignment("B", mkExecutorInfoList(1, 2, 3, 4, 5, 6), mkWorkerResources(100.0, 100.0, 100.0)); - LocalAssignment bReordered = mkLocalAssignment("B", mkExecutorInfoList(6, 5, 4, 3, 2, 1), mkWorkerResources(100.0, 100.0, 100.0)); - - LocalAssignment c = mkLocalAssignment("C", mkExecutorInfoList(188, 261),mkWorkerResources(400.0,10000.0,0.0)); + LocalAssignment a = mkLocalAssignment("A", mkExecutorInfoList(1, 2, 3, 4, 5), + mkWorkerResources(100.0, 100.0, 100.0)); + LocalAssignment aResized = mkLocalAssignment("A", mkExecutorInfoList(1, 2, 3, 4, 5), + mkWorkerResources(100.0, 200.0, 100.0)); + LocalAssignment b = mkLocalAssignment("B", mkExecutorInfoList(1, 2, 3, 4, 5, 6), + mkWorkerResources(100.0, 100.0, 100.0)); + LocalAssignment bReordered = mkLocalAssignment("B", mkExecutorInfoList(6, 5, 4, 3, 2, 1), + mkWorkerResources(100.0, 100.0, 100.0)); + + LocalAssignment c = mkLocalAssignment("C", mkExecutorInfoList(188, 261), + mkWorkerResources(400.0, 10000.0, 0.0)); WorkerResources workerResources = mkWorkerResources(400.0, 10000.0, 0.0); Map additionalResources = workerResources.get_resources(); - if( additionalResources == null) additionalResources = new HashMap<>(); + if (additionalResources == null) { + additionalResources = new HashMap<>(); + } additionalResources.put("network.resource.units", 0.0); workerResources.set_resources(additionalResources); - LocalAssignment cReordered = mkLocalAssignment("C", mkExecutorInfoList(188, 261), workerResources); + LocalAssignment cReordered = mkLocalAssignment("C", mkExecutorInfoList(188, 261), + workerResources); - assertTrue(EquivalenceUtils.areLocalAssignmentsEquivalent(c,cReordered)); + assertTrue(EquivalenceUtils.areLocalAssignmentsEquivalent(c, cReordered)); assertTrue(EquivalenceUtils.areLocalAssignmentsEquivalent(null, null)); assertTrue(EquivalenceUtils.areLocalAssignmentsEquivalent(a, a)); assertTrue(EquivalenceUtils.areLocalAssignmentsEquivalent(b, bReordered)); diff --git a/storm-server/src/test/java/org/apache/storm/utils/ServerUtilsTest.java b/storm-server/src/test/java/org/apache/storm/utils/ServerUtilsTest.java index 10ed0890d78..2e4062358d1 100644 --- a/storm-server/src/test/java/org/apache/storm/utils/ServerUtilsTest.java +++ b/storm-server/src/test/java/org/apache/storm/utils/ServerUtilsTest.java @@ -64,9 +64,11 @@ public void testExtractZipFileDisallowsPathTraversal() throws Exception { Files.createDirectories(extractionDest); /* - * Contains good.txt and ../evil.txt. Evil.txt will path outside the target dir, and should not be extracted. + * Contains good.txt and ../evil.txt. Evil.txt will path outside the target dir, and + * should not be extracted. */ - try (ZipFile zip = new ZipFile(Paths.get("src/test/resources/evil-path-traversal.jar").toFile())) { + try (ZipFile zip = new ZipFile(Paths.get("src/test/resources/evil-path-traversal.jar") + .toFile())) { ServerUtils.extractZipFile(zip, extractionDest.toFile(), null); } @@ -84,10 +86,12 @@ public void testExtractZipFileDisallowsPathTraversalWhenUsingPrefix() throws Exc Files.createDirectories(extractionDest); /* - * Contains resources/good.txt and resources/../evil.txt. Evil.txt should not be extracted as it would end + * Contains resources/good.txt and resources/../evil.txt. Evil.txt should not be + * extracted as it would end * up outside the extraction dest. */ - try (ZipFile zip = new ZipFile(Paths.get("src/test/resources/evil-path-traversal-resources.jar").toFile())) { + try (ZipFile zip = new ZipFile(Paths + .get("src/test/resources/evil-path-traversal-resources.jar").toFile())) { ServerUtils.extractZipFile(zip, extractionDest.toFile(), "resources"); } @@ -129,7 +133,8 @@ private Collection getRunningProcessIds(String user) throws IOException { continue; // header line } if (!StringUtils.isNumeric(pidStr)) { - LOG.debug("Ignoring line \"{}\" while looking for PIDs in output of \"{}\"", line, cmd); + LOG.debug("Ignoring line \"{}\" while looking for PIDs in output of \"{}\"", + line, cmd); continue; } pids.add(Long.parseLong(pidStr)); @@ -149,14 +154,15 @@ public void testIsProcessAlive() throws Exception { // get list of few running processes Collection pids = getRunningProcessIds(null); assertFalse(pids.isEmpty()); - for (long pid: pids) { + for (long pid : pids) { boolean status = ServerUtils.isProcessAlive(pid, randomUser); - assertFalse(status, "Random user " + randomUser + " is not expected to own any process"); + assertFalse(status, "Random user " + randomUser + + " is not expected to own any process"); } boolean status = false; String currentUser = System.getProperty("user.name"); - for (long pid: pids) { + for (long pid : pids) { // at least one pid will be owned by the current user (doing the testing) if (ServerUtils.isProcessAlive(pid, currentUser)) { status = true; @@ -185,7 +191,8 @@ public void testIsAnyProcessAlive() throws Exception { // userid test is valid only on Posix platforms int inValidUserId = -1; status = ServerUtils.isAnyProcessAlive(pids, inValidUserId); - assertFalse(status, "Invalid userId " + randomUser + " is not expected to own any process"); + assertFalse(status, "Invalid userId " + randomUser + + " is not expected to own any process"); int currentUid = ServerUtils.getUserId(null); status = ServerUtils.isAnyProcessAlive(pids, currentUid); @@ -221,7 +228,8 @@ public void testIsAnyProcessPosixProcessPidDirAlive() throws IOException { if (!parentDir.toFile().exists()) { LOG.info("{}: test cannot be run on system without process directory {}, os.name={}", testName, parentDir, System.getProperty("os.name")); - // check if we can get process id on this Posix system - testing test code, useful on Mac + // check if we can get process id on this Posix system - testing test code, useful on + // Mac String cmd = "/bin/sleep 10"; if (getPidOfPosixProcess(Runtime.getRuntime().exec(cmd), errors) < 0) { fail(String.format("%s: Cannot obtain process id for executed command \"%s\"\n%s", @@ -232,13 +240,15 @@ public void testIsAnyProcessPosixProcessPidDirAlive() throws IOException { // Create processes and wait for their termination Set observables = new HashSet<>(); - for (int i = 0 ; i < maxPidCnt ; i++) { + for (int i = 0; i < maxPidCnt; i++) { String cmd = "sleep 20000"; Process process = Runtime.getRuntime().exec(cmd); long pid = getPidOfPosixProcess(process, errors); LOG.info("{}: ({}) ran process \"{}\" with pid={}", testName, i, cmd, pid); if (pid < 0) { - String e = String.format("%s: (%d) Cannot obtain process id for executed command \"%s\"", testName, i, cmd); + String e = String + .format("%s: (%d) Cannot obtain process id for executed command \"%s\"", + testName, i, cmd); errors.add(e); LOG.error(e); continue; @@ -251,43 +261,55 @@ public void testIsAnyProcessPosixProcessPidDirAlive() throws IOException { final long processKillIntervalMs = 2000; for (int i = 0; i < pidList.size(); i++) { long pid = pidList.get(i); - LOG.info("{}: ({}) Sleeping for {} milliseconds before kill", testName, i, processKillIntervalMs); + LOG.info("{}: ({}) Sleeping for {} milliseconds before kill", testName, i, + processKillIntervalMs); if (sleepInterrupted(processKillIntervalMs)) { return; } Runtime.getRuntime().exec("kill -9 " + pid); - LOG.info("{}: ({}) Sleeping for {} milliseconds after kill", testName, i, processKillIntervalMs); + LOG.info("{}: ({}) Sleeping for {} milliseconds after kill", testName, i, + processKillIntervalMs); if (sleepInterrupted(processKillIntervalMs)) { return; } - boolean pidDirsAvailable = ServerUtils.isAnyPosixProcessPidDirAlive(observables, userName); + boolean pidDirsAvailable = ServerUtils.isAnyPosixProcessPidDirAlive(observables, + userName); if (i < pidList.size() - 1) { if (pidDirsAvailable) { - LOG.info("{}: ({}) Found existing process directories before killing last process", testName, i); + LOG.info("{}: ({}) Found existing process directories before killing last " + + "process", testName, i); } else { - String e = String.format("%s: (%d) Found no existing process directories before killing last process", testName, i); + String e = String + .format("%s: (%d) Found no existing process directories before " + + "killing last process", testName, i); errors.add(e); LOG.error(e); } } else { if (pidDirsAvailable) { - String e = String.format("%s: (%d) Found existing process directories after killing last process", testName, i); + String e = String + .format("%s: (%d) Found existing process directories after killing " + + "last process", testName, i); errors.add(e); LOG.error(e); } else { - LOG.info("{}: ({}) Found no existing process directories after killing last process", testName, i); + LOG.info("{}: ({}) Found no existing process directories after killing last " + + "process", testName, i); } } } if (!errors.isEmpty()) { - fail(String.format("There are %d failures in test:\n\t%s", errors.size(), String.join("\n\t", errors))); + fail(String.format("There are %d failures in test:\n\t%s", errors.size(), String + .join("\n\t", errors))); } } /** - * Simulate the production scenario where the owner of the process directory is sometimes returned as the + * Simulate the production scenario where the owner of the process directory is sometimes + * returned as the * UID instead of user. This scenario is simulated by calling - * {@link ServerUtils#isAnyPosixProcessPidDirAlive(Collection, String, boolean)} with the last parameter + * {@link ServerUtils#isAnyPosixProcessPidDirAlive(Collection, String, boolean)} with the last + * parameter * set to true as well as false. * * @throws Exception on I/O exception @@ -296,7 +318,8 @@ public void testIsAnyProcessPosixProcessPidDirAlive() throws IOException { public void testIsAnyPosixProcessPidDirAliveMockingFileOwnerUid() throws Exception { File procDir = new File("/proc"); if (!procDir.exists()) { - LOG.info("Test testIsAnyPosixProcessPidDirAlive is designed to run on systems with /proc directory only, marking as success"); + LOG.info("Test testIsAnyPosixProcessPidDirAlive is designed to run on systems with " + + "/proc directory only, marking as success"); return; } Collection allPids = getRunningProcessIds(null); @@ -306,25 +329,30 @@ public void testIsAnyPosixProcessPidDirAliveMockingFileOwnerUid() throws Excepti String currentUser = System.getProperty("user.name"); - for (boolean mockFileOwnerToUid: Arrays.asList(true, false)) { + for (boolean mockFileOwnerToUid : Arrays.asList(true, false)) { // at least one pid will be owned by the current user (doing the testing) - boolean status = ServerUtils.isAnyPosixProcessPidDirAlive(allPids, currentUser, mockFileOwnerToUid); - String err = String.format("(mockFileOwnerToUid=%s) Expecting user %s to own at least one process", + boolean status = ServerUtils.isAnyPosixProcessPidDirAlive(allPids, currentUser, + mockFileOwnerToUid); + String err = String + .format("(mockFileOwnerToUid=%s) Expecting user %s to own at least one process", mockFileOwnerToUid, currentUser); assertTrue(status, err); } // simulate reassignment of all process id to a different user (root) - for (boolean mockFileOwnerToUid: Arrays.asList(true, false)) { - boolean status = ServerUtils.isAnyPosixProcessPidDirAlive(rootPids, currentUser, mockFileOwnerToUid); - String err = String.format("(mockFileOwnerToUid=%s) Expecting user %s to own no process", + for (boolean mockFileOwnerToUid : Arrays.asList(true, false)) { + boolean status = ServerUtils.isAnyPosixProcessPidDirAlive(rootPids, currentUser, + mockFileOwnerToUid); + String err = String + .format("(mockFileOwnerToUid=%s) Expecting user %s to own no process", mockFileOwnerToUid, currentUser); assertFalse(status, err); } } /** - * Make the best effort to obtain the Process ID from the Process object. Thus staying entirely with the JVM. + * Make the best effort to obtain the Process ID from the Process object. Thus staying entirely + * with the JVM. * * @param p Process instance returned upon executing {@link Runtime#exec(String)}. * @param errors Populate errors when PID is a negative number. @@ -341,7 +369,8 @@ private synchronized long getPidOfPosixProcess(Process p, List errors) { long pid = f.getLong(p); f.setAccessible(false); if (pid < 0) { - errors.add("\t \"pid\" attribute in Process class " + pclassName + " returned -1, process=" + pObjStr); + errors.add("\t \"pid\" attribute in Process class " + pclassName + + " returned -1, process=" + pObjStr); } return pid; } @@ -349,34 +378,42 @@ private synchronized long getPidOfPosixProcess(Process p, List errors) { if (!f.getName().equalsIgnoreCase("pid")) { continue; } - LOG.info("ServerUtilsTest.getPidOfPosixProcess(): found attribute {}#{}", pclassName, f.getName()); + LOG.info("ServerUtilsTest.getPidOfPosixProcess(): found attribute {}#{}", + pclassName, f.getName()); f.setAccessible(true); long pid = f.getLong(p); f.setAccessible(false); if (pid < 0) { - errors.add("\t \"pid\" attribute in Process class " + pclassName + " returned -1, process=" + pObjStr); + errors.add("\t \"pid\" attribute in Process class " + pclassName + + " returned -1, process=" + pObjStr); } return pid; } - // post JDK 9 there should be getPid() - future JDK-11 compatibility only for the sake of Travis test in community + // post JDK 9 there should be getPid() - future JDK-11 compatibility only for the sake + // of Travis test in community try { Method m = pClass.getDeclaredMethod("getPid"); - LOG.info("ServerUtilsTest.getPidOfPosixProcess(): found method {}#getPid()\n", pclassName); - long pid = (Long)m.invoke(p); + LOG.info("ServerUtilsTest.getPidOfPosixProcess(): found method {}#getPid()\n", + pclassName); + long pid = (Long) m.invoke(p); if (pid < 0) { - errors.add("\t \"getPid()\" method in Process class " + pclassName + " returned -1, process=" + pObjStr); + errors.add("\t \"getPid()\" method in Process class " + pclassName + + " returned -1, process=" + pObjStr); } return pid; } catch (SecurityException e) { - errors.add("\t getPid() method in Process class " + pclassName + " cannot be called: " + e.getMessage() + ", process=" + pObjStr); + errors.add("\t getPid() method in Process class " + pclassName + + " cannot be called: " + e.getMessage() + ", process=" + pObjStr); return -1; } catch (NoSuchMethodException e) { // ignore and try something else } - errors.add("\t Process class " + pclassName + " missing field \"pid\" and missing method \"getPid()\", process=" + pObjStr); + errors.add("\t Process class " + pclassName + + " missing field \"pid\" and missing method \"getPid()\", process=" + pObjStr); return -1; } catch (Exception e) { - errors.add("\t Exception in Process class " + pclassName + ": " + e.getMessage() + ", process=" + pObjStr); + errors.add("\t Exception in Process class " + pclassName + ": " + e.getMessage() + + ", process=" + pObjStr); e.printStackTrace(); return -1; } @@ -403,19 +440,30 @@ private boolean sleepInterrupted(long milliSeconds) { public void testResolveTopologyConfSuppliedName() throws Exception { File baseDir = new File(System.getProperty("java.io.tmpdir"), "stormdist/topo-1-1"); - assertEquals(new File(baseDir, "myblob"), ServerUtils.resolveTopologyConfSuppliedName(baseDir, "myblob")); - assertEquals(new File(baseDir, "resources/myblob"), ServerUtils.resolveTopologyConfSuppliedName(baseDir, "resources/myblob")); + assertEquals(new File(baseDir, "myblob"), ServerUtils + .resolveTopologyConfSuppliedName(baseDir, "myblob")); + assertEquals(new File(baseDir, "resources/myblob"), ServerUtils + .resolveTopologyConfSuppliedName(baseDir, "resources/myblob")); - assertThrows(IOException.class, () -> ServerUtils.resolveTopologyConfSuppliedName(baseDir, "..")); - assertThrows(IOException.class, () -> ServerUtils.resolveTopologyConfSuppliedName(baseDir, "../other-topo/stormjar.jar")); - assertThrows(IOException.class, () -> ServerUtils.resolveTopologyConfSuppliedName(baseDir, "a/../../../etc/passwd")); - assertThrows(IOException.class, () -> ServerUtils.resolveTopologyConfSuppliedName(baseDir, "a/../b/../..")); - //a sibling directory whose name only starts with the base directory name is not inside it - assertThrows(IOException.class, () -> ServerUtils.resolveTopologyConfSuppliedName(baseDir, "../topo-1-1-evil/stormjar.jar")); - assertThrows(IOException.class, () -> ServerUtils.resolveTopologyConfSuppliedName(baseDir, ".")); + assertThrows(IOException.class, () -> ServerUtils.resolveTopologyConfSuppliedName(baseDir, + "..")); + assertThrows(IOException.class, () -> ServerUtils.resolveTopologyConfSuppliedName(baseDir, + "../other-topo/stormjar.jar")); + assertThrows(IOException.class, () -> ServerUtils.resolveTopologyConfSuppliedName(baseDir, + "a/../../../etc/passwd")); + assertThrows(IOException.class, () -> ServerUtils.resolveTopologyConfSuppliedName(baseDir, + "a/../b/../..")); + // a sibling directory whose name only starts with the base directory name is not inside it + assertThrows(IOException.class, () -> ServerUtils.resolveTopologyConfSuppliedName(baseDir, + "../topo-1-1-evil/stormjar.jar")); + assertThrows(IOException.class, () -> ServerUtils.resolveTopologyConfSuppliedName(baseDir, + ".")); assertThrows(IOException.class, - () -> ServerUtils.resolveTopologyConfSuppliedName(baseDir, new File("etc", "passwd").getAbsolutePath())); - assertThrows(IOException.class, () -> ServerUtils.resolveTopologyConfSuppliedName(baseDir, "")); - assertThrows(IOException.class, () -> ServerUtils.resolveTopologyConfSuppliedName(baseDir, null)); + () -> ServerUtils.resolveTopologyConfSuppliedName(baseDir, new File("etc", "passwd") + .getAbsolutePath())); + assertThrows(IOException.class, () -> ServerUtils.resolveTopologyConfSuppliedName(baseDir, + "")); + assertThrows(IOException.class, () -> ServerUtils.resolveTopologyConfSuppliedName(baseDir, + null)); } } diff --git a/storm-server/src/test/java/org/apache/storm/utils/ZookeeperServerCnxnFactoryTest.java b/storm-server/src/test/java/org/apache/storm/utils/ZookeeperServerCnxnFactoryTest.java index d41145f26c8..4531a3a2535 100644 --- a/storm-server/src/test/java/org/apache/storm/utils/ZookeeperServerCnxnFactoryTest.java +++ b/storm-server/src/test/java/org/apache/storm/utils/ZookeeperServerCnxnFactoryTest.java @@ -1,22 +1,27 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ package org.apache.storm.utils; -import org.junit.jupiter.api.Test; - import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import org.junit.jupiter.api.Test; + public class ZookeeperServerCnxnFactoryTest { @Test @@ -27,9 +32,11 @@ public void test_Exception_In_Constructor_If_Port_Too_Large() { @Test public void testFactory() { int arbitraryTestClients = 42; - ZookeeperServerCnxnFactory zkcfNegative = new ZookeeperServerCnxnFactory(-42, arbitraryTestClients); + ZookeeperServerCnxnFactory zkcfNegative = new ZookeeperServerCnxnFactory(-42, + arbitraryTestClients); int nextPort = zkcfNegative.port() + 1; - ZookeeperServerCnxnFactory zkcfNext = new ZookeeperServerCnxnFactory(nextPort, arbitraryTestClients); + ZookeeperServerCnxnFactory zkcfNext = new ZookeeperServerCnxnFactory(nextPort, + arbitraryTestClients); assertEquals(zkcfNext.factory().getMaxClientCnxnsPerHost(), arbitraryTestClients); } } diff --git a/storm-submit-tools/src/main/java/org/apache/storm/submit/command/DependencyResolverMain.java b/storm-submit-tools/src/main/java/org/apache/storm/submit/command/DependencyResolverMain.java index ccfaa276662..1531e0c13c0 100644 --- a/storm-submit-tools/src/main/java/org/apache/storm/submit/command/DependencyResolverMain.java +++ b/storm-submit-tools/src/main/java/org/apache/storm/submit/command/DependencyResolverMain.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

      Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -19,7 +19,6 @@ package org.apache.storm.submit.command; import com.google.common.base.Preconditions; - import java.io.File; import java.net.MalformedURLException; import java.net.URI; @@ -33,9 +32,7 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; - import net.minidev.json.JSONValue; - import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; import org.apache.commons.cli.DefaultParser; @@ -57,7 +54,8 @@ public class DependencyResolverMain { private static final String OPTION_ARTIFACTS_LONG = "artifacts"; private static final String OPTION_ARTIFACT_REPOSITORIES_LONG = "artifactRepositories"; - private static final String OPTION_MAVEN_LOCAL_REPOSITORY_DIRECTORY_LONG = "mavenLocalRepositoryDirectory"; + private static final String OPTION_MAVEN_LOCAL_REPOSITORY_DIRECTORY_LONG = + "mavenLocalRepositoryDirectory"; private static final String OPTION_PROXY_URL_LONG = "proxyUrl"; private static final String OPTION_PROXY_USERNAME_LONG = "proxyUsername"; private static final String OPTION_PROXY_PASSWORD_LONG = "proxyPassword"; @@ -87,7 +85,8 @@ public static void main(String[] args) throws ParseException, MalformedURLExcept List repositories; if (commandLine.hasOption(OPTION_ARTIFACT_REPOSITORIES_LONG)) { - String remoteRepositoryArg = commandLine.getOptionValue(OPTION_ARTIFACT_REPOSITORIES_LONG); + String remoteRepositoryArg = commandLine + .getOptionValue(OPTION_ARTIFACT_REPOSITORIES_LONG); // DO NOT CHANGE THIS TO SYSOUT System.err.println("DependencyResolver input - repositories: " + remoteRepositoryArg); @@ -125,7 +124,8 @@ public static void main(String[] args) throws ParseException, MalformedURLExcept throw new RuntimeException("Some artifacts are not resolved"); } - System.out.println(JSONValue.toJSONString(transformArtifactResultToArtifactToPaths(artifactResults))); + System.out.println(JSONValue + .toJSONString(transformArtifactResultToArtifactToPaths(artifactResults))); System.out.flush(); } catch (Exception e) { throw new RuntimeException(e); @@ -134,7 +134,8 @@ public static void main(String[] args) throws ParseException, MalformedURLExcept private static void printMissingArtifactsToSysErr(Iterable missingArtifacts) { for (ArtifactResult artifactResult : missingArtifacts) { - System.err.println("ArtifactResult : " + artifactResult + " / Errors : " + artifactResult.getExceptions()); + System.err.println("ArtifactResult : " + artifactResult + " / Errors : " + + artifactResult.getExceptions()); } } @@ -182,12 +183,14 @@ private static Map transformArtifactResultToArtifactToPaths(List Map artifactToPath = new LinkedHashMap<>(); for (ArtifactResult artifactResult : artifactResults) { Artifact artifact = artifactResult.getArtifact(); - artifactToPath.put(AetherUtils.artifactToString(artifact), artifact.getFile().getAbsolutePath()); + artifactToPath.put(AetherUtils.artifactToString(artifact), artifact.getFile() + .getAbsolutePath()); } return artifactToPath; } - private static String getOrDefaultLocalMavenRepositoryPath(String customLocalMavenPath, String defaultPath) { + private static String getOrDefaultLocalMavenRepositoryPath(String customLocalMavenPath, + String defaultPath) { if (customLocalMavenPath != null) { Path customPath = new File(customLocalMavenPath).toPath(); Preconditions.checkArgument(!Files.exists(customPath) || Files.isDirectory(customPath), diff --git a/storm-submit-tools/src/main/java/org/apache/storm/submit/dependency/AetherUtils.java b/storm-submit-tools/src/main/java/org/apache/storm/submit/dependency/AetherUtils.java index ad9a1a3f353..7acfdf2d15c 100644 --- a/storm-submit-tools/src/main/java/org/apache/storm/submit/dependency/AetherUtils.java +++ b/storm-submit-tools/src/main/java/org/apache/storm/submit/dependency/AetherUtils.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

      Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -22,7 +22,6 @@ import java.util.Arrays; import java.util.Collection; import java.util.List; - import org.eclipse.aether.artifact.Artifact; import org.eclipse.aether.artifact.DefaultArtifact; import org.eclipse.aether.graph.Dependency; diff --git a/storm-submit-tools/src/main/java/org/apache/storm/submit/dependency/Booter.java b/storm-submit-tools/src/main/java/org/apache/storm/submit/dependency/Booter.java index 94c82d395ac..30aa9c7a9b9 100644 --- a/storm-submit-tools/src/main/java/org/apache/storm/submit/dependency/Booter.java +++ b/storm-submit-tools/src/main/java/org/apache/storm/submit/dependency/Booter.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

      Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -19,7 +19,6 @@ package org.apache.storm.submit.dependency; import java.io.File; - import org.apache.maven.repository.internal.MavenRepositorySystemUtils; import org.eclipse.aether.DefaultRepositorySystemSession; import org.eclipse.aether.RepositorySystem; diff --git a/storm-submit-tools/src/main/java/org/apache/storm/submit/dependency/DependencyResolver.java b/storm-submit-tools/src/main/java/org/apache/storm/submit/dependency/DependencyResolver.java index e99b7237798..48b7e8b420e 100644 --- a/storm-submit-tools/src/main/java/org/apache/storm/submit/dependency/DependencyResolver.java +++ b/storm-submit-tools/src/main/java/org/apache/storm/submit/dependency/DependencyResolver.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

      Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -22,7 +22,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; - import org.eclipse.aether.RepositorySystem; import org.eclipse.aether.RepositorySystemSession; import org.eclipse.aether.collection.CollectRequest; @@ -101,7 +100,8 @@ private String handleRelativePath(String localRepoPath) { * * @param dependencies the list of dependency * @return downloaded information of artifacts - * @throws DependencyResolutionException If the dependency tree could not be built or any dependency + * @throws DependencyResolutionException If the dependency tree could not be built or any + * dependency * artifact could not be resolved. * @throws ArtifactResolutionException If the artifact could not be resolved. */ @@ -122,14 +122,16 @@ public List resolve(List dependencies) throws DependencyFilter classpathFilter = DependencyFilterUtils .classpathFilter(JavaScopes.COMPILE, JavaScopes.RUNTIME); - DependencyRequest dependencyRequest = new DependencyRequest(collectRequest, classpathFilter); + DependencyRequest dependencyRequest = new DependencyRequest(collectRequest, + classpathFilter); return system.resolveDependencies(session, dependencyRequest).getArtifactResults(); } private void applyProxy() { List appliedRepositories = new ArrayList<>(remoteRepositories.size()); for (RemoteRepository repository : remoteRepositories) { - appliedRepositories.add(new RemoteRepository.Builder(repository).setProxy(proxy).build()); + appliedRepositories.add(new RemoteRepository.Builder(repository).setProxy(proxy) + .build()); } this.remoteRepositories = appliedRepositories; diff --git a/storm-submit-tools/src/main/java/org/apache/storm/submit/dependency/RepositorySystemFactory.java b/storm-submit-tools/src/main/java/org/apache/storm/submit/dependency/RepositorySystemFactory.java index f9a04c32ad8..a66cd4ef0f1 100644 --- a/storm-submit-tools/src/main/java/org/apache/storm/submit/dependency/RepositorySystemFactory.java +++ b/storm-submit-tools/src/main/java/org/apache/storm/submit/dependency/RepositorySystemFactory.java @@ -6,10 +6,10 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - *

      - * http://www.apache.org/licenses/LICENSE-2.0 - *

      - * Unless required by applicable law or agreed to in writing, software + * + *

      http://www.apache.org/licenses/LICENSE-2.0 + * + *

      Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -19,7 +19,6 @@ package org.apache.storm.submit.dependency; import org.apache.maven.repository.internal.MavenRepositorySystemUtils; - import org.eclipse.aether.RepositorySystem; import org.eclipse.aether.connector.basic.BasicRepositoryConnectorFactory; import org.eclipse.aether.impl.DefaultServiceLocator; diff --git a/storm-submit-tools/src/test/java/org/apache/storm/submit/dependency/AetherUtilsTest.java b/storm-submit-tools/src/test/java/org/apache/storm/submit/dependency/AetherUtilsTest.java index 39e10b17548..126b1ab9d2d 100644 --- a/storm-submit-tools/src/test/java/org/apache/storm/submit/dependency/AetherUtilsTest.java +++ b/storm-submit-tools/src/test/java/org/apache/storm/submit/dependency/AetherUtilsTest.java @@ -17,22 +17,22 @@ */ package org.apache.storm.submit.dependency; +import static org.junit.jupiter.api.Assertions.assertEquals; + import com.google.common.collect.Lists; -import org.junit.jupiter.api.Test; +import java.util.List; import org.eclipse.aether.artifact.Artifact; +import org.eclipse.aether.artifact.DefaultArtifact; import org.eclipse.aether.graph.Dependency; import org.eclipse.aether.graph.Exclusion; -import org.eclipse.aether.artifact.DefaultArtifact; import org.eclipse.aether.util.artifact.JavaScopes; - -import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertEquals; +import org.junit.jupiter.api.Test; public class AetherUtilsTest { @Test public void parseDependency() { - String testDependency = "testgroup:testartifact:1.0.0^testgroup:testexcartifact^testgroup:*"; + String testDependency = + "testgroup:testartifact:1.0.0^testgroup:testexcartifact^testgroup:*"; Dependency dependency = AetherUtils.parseDependency(testDependency); diff --git a/storm-submit-tools/src/test/java/org/apache/storm/submit/dependency/DependencyResolverTest.java b/storm-submit-tools/src/test/java/org/apache/storm/submit/dependency/DependencyResolverTest.java index 2644562d2af..d49ac76410e 100644 --- a/storm-submit-tools/src/test/java/org/apache/storm/submit/dependency/DependencyResolverTest.java +++ b/storm-submit-tools/src/test/java/org/apache/storm/submit/dependency/DependencyResolverTest.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

      Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -17,22 +17,21 @@ */ package org.apache.storm.submit.dependency; +import static org.junit.jupiter.api.Assertions.assertTrue; + import com.google.common.collect.Lists; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; import org.apache.commons.io.FileUtils; +import org.eclipse.aether.artifact.DefaultArtifact; +import org.eclipse.aether.graph.Dependency; +import org.eclipse.aether.resolution.ArtifactResult; +import org.eclipse.aether.util.artifact.JavaScopes; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.eclipse.aether.graph.Dependency; -import org.eclipse.aether.resolution.ArtifactResult; -import org.eclipse.aether.artifact.DefaultArtifact; -import org.eclipse.aether.util.artifact.JavaScopes; - -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertTrue; public class DependencyResolverTest { private static Path tempDirForTest; @@ -57,8 +56,11 @@ public void setUp() { @Test public void resolveValid() throws Exception { // please pick small artifact which has small transitive dependency - // and let's mark as Ignore if we want to run test even without internet or maven central is often not stable - Dependency dependency = new Dependency(new DefaultArtifact("org.apache.storm:flux-core:1.0.0"), JavaScopes.COMPILE); + // and let's mark as Ignore if we want to run test even without internet or maven central is + // often not stable + Dependency dependency = + new Dependency(new DefaultArtifact("org.apache.storm:flux-core:1.0.0"), + JavaScopes.COMPILE); List results = sut.resolve(Lists.newArrayList(dependency)); assertTrue(results.size() > 0); @@ -67,17 +69,19 @@ public void resolveValid() throws Exception { assertContains(results, "commons-cli", "commons-cli", "1.2"); } - private void assertContains(List results, String groupId, String artifactId, String version) { + private void assertContains(List results, String groupId, String artifactId, + String version) { for (ArtifactResult result : results) { - if (result.getArtifact().getGroupId().equals(groupId) && - result.getArtifact().getArtifactId().equals(artifactId) && - result.getArtifact().getVersion().equals(version) && - result.isResolved()) { + if (result.getArtifact().getGroupId().equals(groupId) + && result.getArtifact().getArtifactId().equals(artifactId) + && result.getArtifact().getVersion().equals(version) + && result.isResolved()) { return; } } - throw new AssertionError("Result doesn't contain expected artifact > " + groupId + ":" + artifactId + ":" + version); + throw new AssertionError("Result doesn't contain expected artifact > " + groupId + ":" + + artifactId + ":" + version); } } diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/common/AuthorizationExceptionMapper.java b/storm-webapp/src/main/java/org/apache/storm/daemon/common/AuthorizationExceptionMapper.java index dbfd3ca4c86..2dab402e29c 100644 --- a/storm-webapp/src/main/java/org/apache/storm/daemon/common/AuthorizationExceptionMapper.java +++ b/storm-webapp/src/main/java/org/apache/storm/daemon/common/AuthorizationExceptionMapper.java @@ -25,7 +25,6 @@ import jakarta.ws.rs.core.Response; import jakarta.ws.rs.ext.ExceptionMapper; import jakarta.ws.rs.ext.Provider; - import org.apache.storm.generated.AuthorizationException; @Provider diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/common/JsonResponseBuilder.java b/storm-webapp/src/main/java/org/apache/storm/daemon/common/JsonResponseBuilder.java index 96afaff4ddb..59cad95c4cf 100644 --- a/storm-webapp/src/main/java/org/apache/storm/daemon/common/JsonResponseBuilder.java +++ b/storm-webapp/src/main/java/org/apache/storm/daemon/common/JsonResponseBuilder.java @@ -19,10 +19,8 @@ package org.apache.storm.daemon.common; import jakarta.ws.rs.core.Response; - import java.util.Collections; import java.util.Map; - import org.apache.storm.daemon.ui.UIHelpers; /** diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/common/ReloadableSslContextFactory.java b/storm-webapp/src/main/java/org/apache/storm/daemon/common/ReloadableSslContextFactory.java index b3980ec55f7..d3a0588cadf 100644 --- a/storm-webapp/src/main/java/org/apache/storm/daemon/common/ReloadableSslContextFactory.java +++ b/storm-webapp/src/main/java/org/apache/storm/daemon/common/ReloadableSslContextFactory.java @@ -51,7 +51,8 @@ protected void doStart() throws Exception { if (keyStorePathStr != null) { Path keyStorePath = Paths.get(URI.create(keyStorePathStr).getPath()); FileWatcher.Callback keyStoreWatcherCallback = () -> - ReloadableSslContextFactory.this.reload((scf) -> LOG.info("Reloading SslContextFactory due to keystore change")); + ReloadableSslContextFactory.this.reload((scf) -> LOG + .info("Reloading SslContextFactory due to keystore change")); keyStoreWatcher = new FileWatcher(keyStorePath, keyStoreWatcherCallback); keyStoreWatcher.start(); } else { @@ -62,7 +63,8 @@ protected void doStart() throws Exception { if (trustStorePathStr != null) { Path trustStorePath = Paths.get(URI.create(trustStorePathStr).getPath()); FileWatcher.Callback trustStoreWatcherCallback = () -> - ReloadableSslContextFactory.this.reload((scf) -> LOG.info("Reloading SslContextFactory due to truststore change")); + ReloadableSslContextFactory.this.reload((scf) -> LOG + .info("Reloading SslContextFactory due to truststore change")); trustStoreWatcher = new FileWatcher(trustStorePath, trustStoreWatcherCallback); trustStoreWatcher.start(); } else { diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/drpc/DRPCServer.java b/storm-webapp/src/main/java/org/apache/storm/daemon/drpc/DRPCServer.java index 837dee00ed2..25c235cac83 100644 --- a/storm-webapp/src/main/java/org/apache/storm/daemon/drpc/DRPCServer.java +++ b/storm-webapp/src/main/java/org/apache/storm/daemon/drpc/DRPCServer.java @@ -55,21 +55,25 @@ public class DRPCServer implements AutoCloseable { private static final Logger LOG = LoggerFactory.getLogger(DRPCServer.class); private final Meter meterShutdownCalls; - //TODO: in the future this might be better in a common webapp location + // TODO: in the future this might be better in a common webapp location /** * Add a request context filter to the Servlet Context Handler. + * * @param context The Servlet Context handler * @param configName Config name * @param conf Conf to be added in context filter */ - public static void addRequestContextFilter(ServletContextHandler context, String configName, Map conf) { - IHttpCredentialsPlugin auth = ServerAuthUtils.getHttpCredentialsPlugin(conf, (String) conf.get(configName)); + public static void addRequestContextFilter(ServletContextHandler context, String configName, + Map conf) { + IHttpCredentialsPlugin auth = ServerAuthUtils.getHttpCredentialsPlugin(conf, (String) conf + .get(configName)); ReqContextFilter filter = new ReqContextFilter(auth); context.addFilter(new FilterHolder(filter), "/*", EnumSet.allOf(DispatcherType.class)); } - private static ThriftServer mkHandlerServer(final DistributedRPC.Iface service, Integer port, Map conf) { + private static ThriftServer mkHandlerServer(final DistributedRPC.Iface service, Integer port, + Map conf) { ThriftServer ret = null; if (port != null && port >= 0) { ret = new ThriftServer(conf, new DistributedRPC.Processor<>(service), @@ -78,48 +82,64 @@ private static ThriftServer mkHandlerServer(final DistributedRPC.Iface service, return ret; } - private static ThriftServer mkInvokeServer(final DistributedRPCInvocations.Iface service, int port, Map conf) { + private static ThriftServer mkInvokeServer(final DistributedRPCInvocations.Iface service, + int port, Map conf) { return new ThriftServer(conf, new DistributedRPCInvocations.Processor<>(service), ThriftConnectionType.DRPC_INVOCATIONS); } - private static Server mkHttpServer(StormMetricsRegistry metricsRegistry, Map conf, DRPC drpc) { + private static Server mkHttpServer(StormMetricsRegistry metricsRegistry, Map conf, DRPC drpc) { Integer drpcHttpPort = (Integer) conf.get(DaemonConfig.DRPC_HTTP_PORT); Server ret = null; if (drpcHttpPort != null && drpcHttpPort >= 0) { LOG.info("Starting RPC HTTP servers..."); String filterClass = (String) (conf.get(DaemonConfig.DRPC_HTTP_FILTER)); @SuppressWarnings("unchecked") - Map filterParams = (Map) (conf.get(DaemonConfig.DRPC_HTTP_FILTER_PARAMS)); - FilterConfiguration filterConfiguration = new FilterConfiguration(filterClass, filterParams); - final List filterConfigurations = Arrays.asList(filterConfiguration); - final Integer httpsPort = ObjectReader.getInt(conf.get(DaemonConfig.DRPC_HTTPS_PORT), 0); + Map filterParams = (Map) (conf + .get(DaemonConfig.DRPC_HTTP_FILTER_PARAMS)); + FilterConfiguration filterConfiguration = new FilterConfiguration(filterClass, + filterParams); + final List filterConfigurations = Arrays + .asList(filterConfiguration); + final Integer httpsPort = ObjectReader.getInt(conf.get(DaemonConfig.DRPC_HTTPS_PORT), + 0); final String httpsKsPath = (String) (conf.get(DaemonConfig.DRPC_HTTPS_KEYSTORE_PATH)); - final String httpsKsPassword = (String) (conf.get(DaemonConfig.DRPC_HTTPS_KEYSTORE_PASSWORD)); + final String httpsKsPassword = (String) (conf + .get(DaemonConfig.DRPC_HTTPS_KEYSTORE_PASSWORD)); final String httpsKsType = (String) (conf.get(DaemonConfig.DRPC_HTTPS_KEYSTORE_TYPE)); - final String httpsKeyPassword = (String) (conf.get(DaemonConfig.DRPC_HTTPS_KEY_PASSWORD)); + final String httpsKeyPassword = (String) (conf + .get(DaemonConfig.DRPC_HTTPS_KEY_PASSWORD)); final String httpsTsPath = (String) (conf.get(DaemonConfig.DRPC_HTTPS_TRUSTSTORE_PATH)); - final String httpsTsPassword = (String) (conf.get(DaemonConfig.DRPC_HTTPS_TRUSTSTORE_PASSWORD)); + final String httpsTsPassword = (String) (conf + .get(DaemonConfig.DRPC_HTTPS_TRUSTSTORE_PASSWORD)); final String httpsTsType = (String) (conf.get(DaemonConfig.DRPC_HTTPS_TRUSTSTORE_TYPE)); - final Boolean httpsWantClientAuth = (Boolean) (conf.get(DaemonConfig.DRPC_HTTPS_WANT_CLIENT_AUTH)); - final Boolean httpsNeedClientAuth = (Boolean) (conf.get(DaemonConfig.DRPC_HTTPS_NEED_CLIENT_AUTH)); - final Boolean disableHttpBinding = (Boolean) (conf.get(DaemonConfig.DRPC_DISABLE_HTTP_BINDING)); - final boolean enableSslReload = ObjectReader.getBoolean(conf.get(DaemonConfig.DRPC_HTTPS_ENABLE_SSL_RELOAD), false); + final Boolean httpsWantClientAuth = (Boolean) (conf + .get(DaemonConfig.DRPC_HTTPS_WANT_CLIENT_AUTH)); + final Boolean httpsNeedClientAuth = (Boolean) (conf + .get(DaemonConfig.DRPC_HTTPS_NEED_CLIENT_AUTH)); + final Boolean disableHttpBinding = (Boolean) (conf + .get(DaemonConfig.DRPC_DISABLE_HTTP_BINDING)); + final boolean enableSslReload = ObjectReader.getBoolean(conf + .get(DaemonConfig.DRPC_HTTPS_ENABLE_SSL_RELOAD), false); - //TODO: a better way to do this would be great. + // TODO: a better way to do this would be great. DRPCApplication.setup(drpc, metricsRegistry); ret = UIHelpers.jettyCreateServer(drpcHttpPort, null, httpsPort, disableHttpBinding); - UIHelpers.configSsl(ret, httpsPort, httpsKsPath, httpsKsPassword, httpsKsType, httpsKeyPassword, + UIHelpers.configSsl(ret, httpsPort, httpsKsPath, httpsKsPassword, httpsKsType, + httpsKeyPassword, httpsTsPath, httpsTsPassword, httpsTsType, httpsNeedClientAuth, httpsWantClientAuth, enableSslReload); - ServletContextHandler context = new ServletContextHandler(ServletContextHandler.NO_SESSIONS); + ServletContextHandler context = + new ServletContextHandler(ServletContextHandler.NO_SESSIONS); context.setContextPath("/"); ret.setHandler(context); ServletHolder jerseyServlet = context.addServlet(ServletContainer.class, "/*"); jerseyServlet.setInitOrder(1); - jerseyServlet.setInitParameter("jakarta.ws.rs.Application", DRPCApplication.class.getName()); + jerseyServlet.setInitParameter("jakarta.ws.rs.Application", DRPCApplication.class + .getName()); UIHelpers.configFilters(context, filterConfigurations); addRequestContextFilter(context, DaemonConfig.DRPC_HTTP_CREDS_PLUGIN, conf); @@ -136,6 +156,7 @@ private static Server mkHttpServer(StormMetricsRegistry metricsRegistry, Map conf, StormMetricsRegistry metricsRegistry meterShutdownCalls = metricsRegistry.registerMeter("drpc:num-shutdown-calls"); drpc = new DRPC(metricsRegistry, conf); DRPCThrift thrift = new DRPCThrift(drpc); - handlerServer = mkHandlerServer(thrift, ObjectReader.getInt(conf.get(Config.DRPC_PORT), null), conf); - invokeServer = mkInvokeServer(thrift, ObjectReader.getInt(conf.get(Config.DRPC_INVOCATIONS_PORT), 3773), conf); + handlerServer = mkHandlerServer(thrift, ObjectReader.getInt(conf.get(Config.DRPC_PORT), + null), conf); + invokeServer = mkInvokeServer(thrift, ObjectReader.getInt(conf + .get(Config.DRPC_INVOCATIONS_PORT), 3773), conf); httpServer = mkHttpServer(metricsRegistry, conf, drpc); } @@ -182,10 +205,10 @@ public synchronized void close() { invokeServer.stop(); } - //TODO: this is causing issues... - //if (httpServer != null) { + // TODO: this is causing issues... + // if (httpServer != null) { // httpServer.destroy(); - //} + // } drpc.close(); closed = true; @@ -194,6 +217,7 @@ public synchronized void close() { /** * The port the DRPC handler server is listening on. + * * @return The port the DRPC handler server is listening on. */ public int getDrpcPort() { @@ -202,6 +226,7 @@ public int getDrpcPort() { /** * The port the DRPC invoke server is listening on. + * * @return The port the DRPC invoke server is listening on. */ public int getDrpcInvokePort() { @@ -210,7 +235,9 @@ public int getDrpcInvokePort() { /** * The port the HTTP server is listening on. Not available until {@link #start() } has run. - * @return The port the HTTP server is listening on. Not available until {@link #start() } has run. + * + * @return The port the HTTP server is listening on. Not available until {@link #start() } has + * run. */ public int getHttpServerPort() { assert httpServer.getConnectors().length == 1; diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/drpc/webapp/DRPCApplication.java b/storm-webapp/src/main/java/org/apache/storm/daemon/drpc/webapp/DRPCApplication.java index ab735d3e0cf..a5fe30ddab8 100644 --- a/storm-webapp/src/main/java/org/apache/storm/daemon/drpc/webapp/DRPCApplication.java +++ b/storm-webapp/src/main/java/org/apache/storm/daemon/drpc/webapp/DRPCApplication.java @@ -20,10 +20,8 @@ import jakarta.ws.rs.ApplicationPath; import jakarta.ws.rs.core.Application; - import java.util.HashSet; import java.util.Set; - import org.apache.storm.daemon.common.AuthorizationExceptionMapper; import org.apache.storm.daemon.drpc.DRPC; import org.apache.storm.metric.StormMetricsRegistry; diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/drpc/webapp/DRPCExceptionMapper.java b/storm-webapp/src/main/java/org/apache/storm/daemon/drpc/webapp/DRPCExceptionMapper.java index 3de7573ddf9..78e07b712de 100644 --- a/storm-webapp/src/main/java/org/apache/storm/daemon/drpc/webapp/DRPCExceptionMapper.java +++ b/storm-webapp/src/main/java/org/apache/storm/daemon/drpc/webapp/DRPCExceptionMapper.java @@ -18,16 +18,13 @@ package org.apache.storm.daemon.drpc.webapp; -import jakarta.ws.rs.core.Response; import jakarta.ws.rs.core.Response.ResponseBuilder; +import jakarta.ws.rs.core.Response; import jakarta.ws.rs.ext.ExceptionMapper; import jakarta.ws.rs.ext.Provider; - import java.util.HashMap; import java.util.Map; - import net.minidev.json.JSONValue; - import org.apache.storm.generated.DRPCExecutionException; @Provider @@ -42,20 +39,20 @@ public Response toResponse(DRPCExecutionException ex) { builder.status(400); break; case SERVER_SHUTDOWN: - builder.status(503); //Not available + builder.status(503); // Not available break; case SERVER_TIMEOUT: - builder.status(504); //proxy timeout + builder.status(504); // proxy timeout break; case INTERNAL_ERROR: - //fall throw on purpose + // fall throw on purpose default: - //Empty (Still 500) + // Empty (Still 500) break; } Map body = new HashMap<>(); - //TODO: I would love to standardize this... + // TODO: I would love to standardize this... body.put("error", ex.is_set_type() ? ex.get_type().toString() : "Internal Error"); body.put("errorMessage", ex.get_msg()); return builder.entity(JSONValue.toJSONString(body)).type("application/json").build(); diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/drpc/webapp/DRPCResource.java b/storm-webapp/src/main/java/org/apache/storm/daemon/drpc/webapp/DRPCResource.java index 7d2ba133a69..1b315bece32 100644 --- a/storm-webapp/src/main/java/org/apache/storm/daemon/drpc/webapp/DRPCResource.java +++ b/storm-webapp/src/main/java/org/apache/storm/daemon/drpc/webapp/DRPCResource.java @@ -20,14 +20,12 @@ import com.codahale.metrics.Meter; import com.codahale.metrics.Timer; - import jakarta.servlet.http.HttpServletRequest; import jakarta.ws.rs.GET; import jakarta.ws.rs.POST; import jakarta.ws.rs.Path; import jakarta.ws.rs.PathParam; import jakarta.ws.rs.core.Context; - import org.apache.storm.daemon.drpc.DRPC; import org.apache.storm.metric.StormMetricsRegistry; @@ -41,14 +39,16 @@ public class DRPCResource { public DRPCResource(DRPC drpc, StormMetricsRegistry metricsRegistry) { this.drpc = drpc; this.meterHttpRequests = metricsRegistry.registerMeter("drpc:num-execute-http-requests"); - this.responseDuration = metricsRegistry.registerTimer("drpc:HTTP-request-response-duration"); + this.responseDuration = metricsRegistry + .registerTimer("drpc:HTTP-request-response-duration"); } - //TODO: put in some better exception mapping... - //TODO: move populateContext to a filter... + // TODO: put in some better exception mapping... + // TODO: move populateContext to a filter... @POST @Path("/{func}") - public String post(@PathParam("func") String func, String args, @Context HttpServletRequest request) throws Exception { + public String post(@PathParam("func") String func, String args, + @Context HttpServletRequest request) throws Exception { meterHttpRequests.mark(); return responseDuration.time(() -> drpc.executeBlocking(func, args)); } @@ -63,7 +63,8 @@ public String get(@PathParam("func") String func, @PathParam("args") String args @GET @Path("/{func}") - public String get(@PathParam("func") String func, @Context HttpServletRequest request) throws Exception { + public String get(@PathParam("func") String func, + @Context HttpServletRequest request) throws Exception { meterHttpRequests.mark(); return responseDuration.time(() -> drpc.executeBlocking(func, "")); } diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/drpc/webapp/ReqContextFilter.java b/storm-webapp/src/main/java/org/apache/storm/daemon/drpc/webapp/ReqContextFilter.java index 8f9ff9b5ac6..205123a0d6a 100644 --- a/storm-webapp/src/main/java/org/apache/storm/daemon/drpc/webapp/ReqContextFilter.java +++ b/storm-webapp/src/main/java/org/apache/storm/daemon/drpc/webapp/ReqContextFilter.java @@ -28,9 +28,7 @@ import jakarta.servlet.http.HttpServletResponse; import jakarta.ws.rs.container.PreMatching; import jakarta.ws.rs.ext.Provider; - import java.io.IOException; - import org.apache.storm.security.auth.IHttpCredentialsPlugin; import org.apache.storm.security.auth.ReqContext; @@ -44,7 +42,9 @@ public ReqContextFilter(IHttpCredentialsPlugin httpCredsHandler) { } /** - * Populate the Storm RequestContext from an servlet request. This should be called in each handler + * Populate the Storm RequestContext from an servlet request. This should be called in each + * handler + * * @param request the request to populate */ public void populateContext(HttpServletRequest request) { @@ -55,28 +55,34 @@ public void populateContext(HttpServletRequest request) { @Override public void init(FilterConfig config) throws ServletException { - //NOOP - //We could add in configs through the web.xml if we wanted something stand alone here... + // NOOP + // We could add in configs through the web.xml if we wanted something stand alone here... } /** - * A filter which populates the request if it is null and then passes it on to the next entity in the chain. + * A filter which populates the request if it is null and then passes it on to the next entity + * in the chain. + * * @param request the request to populate * @param response the response to populate * @param chain the next chain of entities to pass the object to */ @Override - public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) throws IOException, ServletException { + public void doFilter(ServletRequest request, ServletResponse response, + FilterChain chain) throws IOException, ServletException { handle((HttpServletRequest) request, (HttpServletResponse) response, chain); } /** - * A method used by doFilter which populates the request if it is null and then passes it on to the next entity in the chain. + * A method used by doFilter which populates the request if it is null and then passes it on to + * the next entity in the chain. + * * @param request the request to populate * @param response the response to populate * @param chain the next chain of entities to pass the object to */ - public void handle(HttpServletRequest request, HttpServletResponse response, FilterChain chain) throws IOException, ServletException { + public void handle(HttpServletRequest request, HttpServletResponse response, + FilterChain chain) throws IOException, ServletException { if (request != null) { populateContext(request); } @@ -85,6 +91,6 @@ public void handle(HttpServletRequest request, HttpServletResponse response, Fil @Override public void destroy() { - //NOOP + // NOOP } } diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/LogviewerServer.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/LogviewerServer.java index 1ef42d9e7a7..737aea5b429 100644 --- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/LogviewerServer.java +++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/LogviewerServer.java @@ -20,12 +20,10 @@ import com.codahale.metrics.Meter; import com.google.common.annotations.VisibleForTesting; - import java.io.File; import java.util.Arrays; import java.util.List; import java.util.Map; - import org.apache.commons.lang3.StringUtils; import org.apache.storm.DaemonConfig; import org.apache.storm.daemon.logviewer.utils.DirectoryCleaner; @@ -57,7 +55,8 @@ public class LogviewerServer implements AutoCloseable { public static final String STATIC_RESOURCE_DIRECTORY_PATH = stormHome + "/public"; private final Meter meterShutdownCalls; - private static Server mkHttpServer(StormMetricsRegistry metricsRegistry, Map conf) { + private static Server mkHttpServer(StormMetricsRegistry metricsRegistry, Map conf) { Integer logviewerHttpPort = (Integer) conf.get(DaemonConfig.LOGVIEWER_PORT); Server ret = null; if (logviewerHttpPort != null && logviewerHttpPort >= 0) { @@ -70,31 +69,49 @@ private static Server mkHttpServer(StormMetricsRegistry metricsRegistry, Map filterParams = (Map) (conf.get(filterParamKey)); - FilterConfiguration filterConfiguration = new FilterConfiguration(filterClass, filterParams); - final List filterConfigurations = Arrays.asList(filterConfiguration); - - final Integer httpsPort = ObjectReader.getInt(conf.get(DaemonConfig.LOGVIEWER_HTTPS_PORT), 0); - final String httpsKsPath = (String) (conf.get(DaemonConfig.LOGVIEWER_HTTPS_KEYSTORE_PATH)); - final String httpsKsPassword = (String) (conf.get(DaemonConfig.LOGVIEWER_HTTPS_KEYSTORE_PASSWORD)); - final String httpsKsType = (String) (conf.get(DaemonConfig.LOGVIEWER_HTTPS_KEYSTORE_TYPE)); - final String httpsKeyPassword = (String) (conf.get(DaemonConfig.LOGVIEWER_HTTPS_KEY_PASSWORD)); - final String httpsTsPath = (String) (conf.get(DaemonConfig.LOGVIEWER_HTTPS_TRUSTSTORE_PATH)); - final String httpsTsPassword = (String) (conf.get(DaemonConfig.LOGVIEWER_HTTPS_TRUSTSTORE_PASSWORD)); - final String httpsTsType = (String) (conf.get(DaemonConfig.LOGVIEWER_HTTPS_TRUSTSTORE_TYPE)); - final Boolean httpsWantClientAuth = (Boolean) (conf.get(DaemonConfig.LOGVIEWER_HTTPS_WANT_CLIENT_AUTH)); - final Boolean httpsNeedClientAuth = (Boolean) (conf.get(DaemonConfig.LOGVIEWER_HTTPS_NEED_CLIENT_AUTH)); - final Boolean disableHttpBinding = (Boolean) (conf.get(DaemonConfig.LOGVIEWER_DISABLE_HTTP_BINDING)); - final boolean enableSslReload = ObjectReader.getBoolean(conf.get(DaemonConfig.LOGVIEWER_HTTPS_ENABLE_SSL_RELOAD), false); + FilterConfiguration filterConfiguration = new FilterConfiguration(filterClass, + filterParams); + final List filterConfigurations = Arrays + .asList(filterConfiguration); + + final Integer httpsPort = ObjectReader.getInt(conf + .get(DaemonConfig.LOGVIEWER_HTTPS_PORT), 0); + final String httpsKsPath = (String) (conf + .get(DaemonConfig.LOGVIEWER_HTTPS_KEYSTORE_PATH)); + final String httpsKsPassword = (String) (conf + .get(DaemonConfig.LOGVIEWER_HTTPS_KEYSTORE_PASSWORD)); + final String httpsKsType = (String) (conf + .get(DaemonConfig.LOGVIEWER_HTTPS_KEYSTORE_TYPE)); + final String httpsKeyPassword = (String) (conf + .get(DaemonConfig.LOGVIEWER_HTTPS_KEY_PASSWORD)); + final String httpsTsPath = (String) (conf + .get(DaemonConfig.LOGVIEWER_HTTPS_TRUSTSTORE_PATH)); + final String httpsTsPassword = (String) (conf + .get(DaemonConfig.LOGVIEWER_HTTPS_TRUSTSTORE_PASSWORD)); + final String httpsTsType = (String) (conf + .get(DaemonConfig.LOGVIEWER_HTTPS_TRUSTSTORE_TYPE)); + final Boolean httpsWantClientAuth = (Boolean) (conf + .get(DaemonConfig.LOGVIEWER_HTTPS_WANT_CLIENT_AUTH)); + final Boolean httpsNeedClientAuth = (Boolean) (conf + .get(DaemonConfig.LOGVIEWER_HTTPS_NEED_CLIENT_AUTH)); + final Boolean disableHttpBinding = (Boolean) (conf + .get(DaemonConfig.LOGVIEWER_DISABLE_HTTP_BINDING)); + final boolean enableSslReload = ObjectReader.getBoolean(conf + .get(DaemonConfig.LOGVIEWER_HTTPS_ENABLE_SSL_RELOAD), false); LogviewerApplication.setup(conf, metricsRegistry); - ret = UIHelpers.jettyCreateServer(logviewerHttpPort, null, httpsPort, disableHttpBinding); + ret = UIHelpers.jettyCreateServer(logviewerHttpPort, null, httpsPort, + disableHttpBinding); - UIHelpers.configSsl(ret, httpsPort, httpsKsPath, httpsKsPassword, httpsKsType, httpsKeyPassword, + UIHelpers.configSsl(ret, httpsPort, httpsKsPath, httpsKsPassword, httpsKsType, + httpsKeyPassword, httpsTsPath, httpsTsPassword, httpsTsType, httpsNeedClientAuth, httpsWantClientAuth, enableSslReload); - ServletContextHandler context = new ServletContextHandler(ServletContextHandler.NO_SESSIONS); - context.setBaseResource(new PathResourceFactory().newResource(STATIC_RESOURCE_DIRECTORY_PATH)); + ServletContextHandler context = + new ServletContextHandler(ServletContextHandler.NO_SESSIONS); + context.setBaseResource(new PathResourceFactory() + .newResource(STATIC_RESOURCE_DIRECTORY_PATH)); context.setWelcomeFiles(new String[]{"logviewer.html"}); context.setContextPath("/"); ret.setHandler(context); @@ -105,7 +122,8 @@ private static Server mkHttpServer(StormMetricsRegistry metricsRegistry, Map { server.meterShutdownCalls.mark(); diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogDownloadHandler.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogDownloadHandler.java index 5c6f9137737..7c7bdd3b5fd 100644 --- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogDownloadHandler.java +++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogDownloadHandler.java @@ -19,9 +19,7 @@ package org.apache.storm.daemon.logviewer.handler; import jakarta.ws.rs.core.Response; - import java.io.IOException; - import org.apache.storm.daemon.logviewer.utils.LogFileDownloader; import org.apache.storm.daemon.logviewer.utils.ResourceAuthorizer; import org.apache.storm.daemon.logviewer.utils.WorkerLogs; @@ -42,7 +40,8 @@ public class LogviewerLogDownloadHandler { */ public LogviewerLogDownloadHandler(String logRoot, String daemonLogRoot, WorkerLogs workerLogs, ResourceAuthorizer resourceAuthorizer, StormMetricsRegistry metricsRegistry) { - this.logFileDownloadHelper = new LogFileDownloader(logRoot, daemonLogRoot, workerLogs, resourceAuthorizer, metricsRegistry); + this.logFileDownloadHelper = new LogFileDownloader(logRoot, daemonLogRoot, workerLogs, + resourceAuthorizer, metricsRegistry); } /** @@ -66,7 +65,8 @@ public Response downloadLogFile(String host, String fileName, String user) throw * @param user username * @return a Response which lets browsers download that file. */ - public Response downloadDaemonLogFile(String host, String fileName, String user) throws IOException { + public Response downloadDaemonLogFile(String host, String fileName, + String user) throws IOException { return logFileDownloadHelper.downloadFile(host, fileName, user, true); } diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogPageHandler.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogPageHandler.java index b1144ca653d..615315ea5a4 100644 --- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogPageHandler.java +++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogPageHandler.java @@ -40,9 +40,7 @@ import com.codahale.metrics.Meter; import j2html.attributes.Attr; import j2html.tags.DomContent; - import jakarta.ws.rs.core.Response; - import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileInputStream; @@ -64,7 +62,6 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.zip.GZIPInputStream; - import org.apache.commons.lang3.StringUtils; import org.apache.storm.daemon.logviewer.LogviewerConstant; import org.apache.storm.daemon.logviewer.utils.DirectoryCleaner; @@ -108,13 +105,16 @@ public LogviewerLogPageHandler(String logRoot, String daemonLogRoot, this.workerLogs = workerLogs; this.resourceAuthorizer = resourceAuthorizer; this.numPageRead = metricsRegistry.registerMeter("logviewer:num-page-read"); - this.numFileOpenExceptions = metricsRegistry.registerMeter(ExceptionMeterNames.NUM_FILE_OPEN_EXCEPTIONS); - this.numFileReadExceptions = metricsRegistry.registerMeter(ExceptionMeterNames.NUM_FILE_READ_EXCEPTIONS); + this.numFileOpenExceptions = metricsRegistry + .registerMeter(ExceptionMeterNames.NUM_FILE_OPEN_EXCEPTIONS); + this.numFileReadExceptions = metricsRegistry + .registerMeter(ExceptionMeterNames.NUM_FILE_READ_EXCEPTIONS); this.directoryCleaner = new DirectoryCleaner(metricsRegistry); } /** - * Enumerate worker log files for given criteria. Only the files the user is allowed to access are returned. + * Enumerate worker log files for given criteria. Only the files the user is allowed to access + * are returned. * * @param user username * @param port worker's port, null for all workers @@ -123,7 +123,8 @@ public LogviewerLogPageHandler(String logRoot, String daemonLogRoot, * @param origin origin * @return list of worker logs for given criteria */ - public Response listLogFiles(String user, Integer port, String topologyId, String callback, String origin) throws IOException { + public Response listLogFiles(String user, Integer port, String topologyId, String callback, + String origin) throws IOException { List fileResults = null; if (topologyId == null) { if (port == null) { @@ -138,7 +139,8 @@ public Response listLogFiles(String user, Integer port, String topologyId, Strin if (topoDirFiles != null) { for (File portDir : topoDirFiles) { if (portDir.getName().equals(port.toString())) { - fileResults.addAll(directoryCleaner.getFilesForDir(portDir.toPath())); + fileResults.addAll(directoryCleaner.getFilesForDir(portDir + .toPath())); } } } @@ -151,7 +153,8 @@ public Response listLogFiles(String user, Integer port, String topologyId, Strin Path topoDir = logRoot.resolve(topologyId).toAbsolutePath().normalize(); if (!topoDir.startsWith(logRoot)) { - return LogviewerResponseBuilder.buildSuccessJsonResponse(Collections.emptyList(), callback, origin); + return LogviewerResponseBuilder.buildSuccessJsonResponse(Collections + .emptyList(), callback, origin); } if (topoDir.toFile().exists()) { File[] topoDirFiles = topoDir.toFile().listFiles(); @@ -163,9 +166,11 @@ public Response listLogFiles(String user, Integer port, String topologyId, Strin } } else { - File portDir = ConfigUtils.getWorkerDirFromRoot(logRoot.toString(), topologyId, port).getCanonicalFile(); + File portDir = ConfigUtils.getWorkerDirFromRoot(logRoot.toString(), topologyId, + port).getCanonicalFile(); if (!portDir.getPath().startsWith(logRoot.toString())) { - return LogviewerResponseBuilder.buildSuccessJsonResponse(Collections.emptyList(), callback, origin); + return LogviewerResponseBuilder.buildSuccessJsonResponse(Collections + .emptyList(), callback, origin); } if (portDir.exists()) { fileResults = directoryCleaner.getFilesForDir(portDir.toPath()); @@ -188,10 +193,13 @@ public Response listLogFiles(String user, Integer port, String topologyId, Strin } /** - * Check whether the user may access the given "topologyId/port/fileName" worker log. The authorization only depends on the - * topology and the port, so the answer is cached per port directory to avoid re-reading the log metadata for every file. + * Check whether the user may access the given "topologyId/port/fileName" worker log. The + * authorization only depends on the + * topology and the port, so the answer is cached per port directory to avoid re-reading the log + * metadata for every file. */ - private boolean isUserAllowedToAccessLog(String user, String fileStr, Map authorizedPortDirs) { + private boolean isUserAllowedToAccessLog(String user, String fileStr, Map authorizedPortDirs) { Path portDir = Paths.get(fileStr).getParent(); if (portDir == null) { return resourceAuthorizer.isUserAllowedToAccessFile(user, fileStr); @@ -211,12 +219,14 @@ private boolean isUserAllowedToAccessLog(String user, String fileStr, Map bodyContents = new ArrayList<>(); if (StringUtils.isNotEmpty(grep)) { @@ -265,7 +278,8 @@ public Response logPage(String fileName, Integer start, Integer length, String g } else { DomContent pagerData = null; if (isTxtFile(fileName)) { - pagerData = pagerLinks(fileName, start, length, Long.valueOf(fileLength).intValue(), "log"); + pagerData = pagerLinks(fileName, start, length, Long.valueOf(fileLength) + .intValue(), "log"); } bodyContents.add(searchFileForm(fileName, "no")); @@ -305,11 +319,12 @@ public Response logPage(String fileName, Integer start, Integer length, String g * @param user username * @return HTML view page of daemon log */ - public Response daemonLogPage(String fileName, Integer start, Integer length, String grep, String user) + public Response daemonLogPage(String fileName, Integer start, Integer length, String grep, + String user) throws IOException, InvalidRequestException { Path file = daemonLogRoot.resolve(fileName).toAbsolutePath().normalize(); if (!file.startsWith(daemonLogRoot) || Paths.get(fileName).getNameCount() != 1) { - //Prevent fileName from pathing into worker logs, or outside daemon log root + // Prevent fileName from pathing into worker logs, or outside daemon log root return LogviewerResponseBuilder.buildResponsePageNotFound(); } @@ -329,15 +344,18 @@ public Response daemonLogPage(String fileName, Integer start, Integer length, St .collect(toList()); reorderedFilesStr.add(fileName); - length = length != null ? Math.min(10485760, length) : LogviewerConstant.DEFAULT_BYTES_PER_PAGE; + length = length != null ? Math.min(10485760, + length) : LogviewerConstant.DEFAULT_BYTES_PER_PAGE; final boolean isZipFile = file.getFileName().toString().endsWith(".gz"); long fileLength = getFileLength(file.toFile(), isZipFile); if (start == null) { start = Long.valueOf(fileLength - length).intValue(); } - String logString = isTxtFile(fileName) ? escapeHtml4(pageFile(file.toString(), isZipFile, fileLength, start, length)) : - escapeHtml4("This is a binary file and cannot display! You may download the full file."); + String logString = isTxtFile(fileName) ? escapeHtml4(pageFile(file.toString(), + isZipFile, fileLength, start, length)) : + escapeHtml4("This is a binary file and cannot display! You may download the " + + "full file."); List bodyContents = new ArrayList<>(); if (StringUtils.isNotEmpty(grep)) { @@ -347,7 +365,8 @@ public Response daemonLogPage(String fileName, Integer start, Integer length, St } else { DomContent pagerData = null; if (isTxtFile(fileName)) { - pagerData = pagerLinks(fileName, start, length, Long.valueOf(fileLength).intValue(), "daemonlog"); + pagerData = pagerLinks(fileName, start, length, Long.valueOf(fileLength) + .intValue(), "daemonlog"); } bodyContents.add(searchFileForm(fileName, "yes")); @@ -389,7 +408,9 @@ private DomContent logTemplate(List bodyContents, String fileName, S finalBodyContents.add(div(p("User: " + user)).withClass("ui-user")); } - finalBodyContents.add(div(p("Note: the drop-list shows at most 1024 files for each worker directory.")).withClass("ui-note")); + finalBodyContents + .add(div(p("Note: the drop-list shows at most 1024 files for each worker " + + "directory.")).withClass("ui-note")); finalBodyContents.add(h3(escapeHtml4(fileName))); finalBodyContents.addAll(bodyContents); @@ -397,7 +418,8 @@ private DomContent logTemplate(List bodyContents, String fileName, S head( title(escapeHtml4(fileName) + " - Storm Log Viewer"), link().withRel("stylesheet").withHref("/css/bootstrap-3.3.1.min.css"), - link().withRel("stylesheet").withHref("/css/jquery.dataTables.1.10.4.min.css"), + link().withRel("stylesheet") + .withHref("/css/jquery.dataTables.1.10.4.min.css"), link().withRel("stylesheet").withHref("/css/style.css") ), body( @@ -407,18 +429,21 @@ private DomContent logTemplate(List bodyContents, String fileName, S } private DomContent downloadLink(String fileName) { - return p(linkTo(UIHelpers.urlFormat("/api/v1/download?file=%s", fileName), "Download Full File")); + return p(linkTo(UIHelpers.urlFormat("/api/v1/download?file=%s", fileName), + "Download Full File")); } private DomContent daemonDownloadLink(String fileName) { - return p(linkTo(UIHelpers.urlFormat("/api/v1/daemondownload?file=%s", fileName), "Download Full File")); + return p(linkTo(UIHelpers.urlFormat("/api/v1/daemondownload?file=%s", fileName), + "Download Full File")); } private DomContent linkTo(String url, String content) { return a(content).withHref(url); } - private DomContent logFileSelectionForm(List logFiles, String selectedFile, String type) { + private DomContent logFileSelectionForm(List logFiles, String selectedFile, + String type) { return form( dropDown("file", logFiles, selectedFile), input().withType("submit").withValue("Switch file") @@ -427,9 +452,11 @@ private DomContent logFileSelectionForm(List logFiles, String selectedFi private DomContent dropDown(String name, List logFiles, String selectedFile) { List options = logFiles.stream() - .map(file -> option(file).condAttr(file.equals(selectedFile), "selected", "selected")) + .map(file -> option(file).condAttr(file.equals(selectedFile), "selected", + "selected")) .collect(toList()); - return select(options.toArray(new DomContent[]{})).withName(name).withId(name).attr(Attr.VALUE, selectedFile); + return select(options.toArray(new DomContent[]{})).withName(name).withId(name) + .attr(Attr.VALUE, selectedFile); } private DomContent searchFileForm(String fileName, String isDaemonValue) { @@ -442,7 +469,8 @@ private DomContent searchFileForm(String fileName, String isDaemonValue) { ).withAction("/logviewer_search.html").withId("search-box"); } - private DomContent pagerLinks(String fileName, Integer start, Integer length, Integer fileLength, String type) { + private DomContent pagerLinks(String fileName, Integer start, Integer length, + Integer fileLength, String type) { Map urlQueryParams = new HashMap<>(); urlQueryParams.put("file", fileName); urlQueryParams.put("start", Math.max(0, start - length)); @@ -451,7 +479,8 @@ private DomContent pagerLinks(String fileName, Integer start, Integer length, In List btnLinks = new ArrayList<>(); int prevStart = Math.max(0, start - length); - btnLinks.add(toButtonLink(UrlBuilder.build("/api/v1/" + type, urlQueryParams), "Prev", prevStart < start)); + btnLinks.add(toButtonLink(UrlBuilder.build("/api/v1/" + type, urlQueryParams), "Prev", + prevStart < start)); urlQueryParams.clear(); urlQueryParams.put("file", fileName); @@ -471,8 +500,10 @@ private DomContent pagerLinks(String fileName, Integer start, Integer length, In urlQueryParams.put("start", Math.min(Math.max(0, fileLength - length), start + length)); urlQueryParams.put("length", length); - int nextStart = fileLength > 0 ? Math.min(Math.max(0, fileLength - length), start + length) : start + length; - btnLinks.add(toButtonLink(UrlBuilder.build("/api/v1/" + type, urlQueryParams), "Next", nextStart > start)); + int nextStart = fileLength > 0 ? Math.min(Math.max(0, fileLength - length), start + + length) : start + length; + btnLinks.add(toButtonLink(UrlBuilder.build("/api/v1/" + type, urlQueryParams), "Next", + nextStart > start)); return div(btnLinks.toArray(new DomContent[]{})); } @@ -482,12 +513,15 @@ private DomContent toButtonLink(String url, String text) { } private DomContent toButtonLink(String url, String text, boolean enabled) { - return a(text).withHref(url).withClass("btn btn-default " + (enabled ? "enabled" : "disabled")); + return a(text).withHref(url).withClass("btn btn-default " + (enabled + ? "enabled" : "disabled")); } - private String pageFile(String path, boolean isZipFile, long fileLength, Integer start, Integer readLength) + private String pageFile(String path, boolean isZipFile, long fileLength, Integer start, + Integer readLength) throws IOException, InvalidRequestException { - try (InputStream input = isZipFile ? new GZIPInputStream(new FileInputStream(path)) : new FileInputStream(path); + try (InputStream input = isZipFile + ? new GZIPInputStream(new FileInputStream(path)) : new FileInputStream(path); ByteArrayOutputStream output = new ByteArrayOutputStream()) { if (start >= fileLength) { throw new InvalidRequestException("Cannot start past the end of the file"); diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogSearchHandler.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogSearchHandler.java index 99da13fb18e..e72747dda4c 100644 --- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogSearchHandler.java +++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogSearchHandler.java @@ -31,9 +31,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.annotations.VisibleForTesting; - import jakarta.ws.rs.core.Response; - import java.io.BufferedInputStream; import java.io.IOException; import java.io.InputStream; @@ -55,9 +53,7 @@ import java.util.regex.Pattern; import java.util.stream.Stream; import java.util.zip.GZIPInputStream; - import net.minidev.json.JSONAware; - import org.apache.commons.lang3.BooleanUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; @@ -77,7 +73,6 @@ import org.apache.storm.utils.ObjectReader; import org.apache.storm.utils.ServerUtils; import org.apache.storm.utils.Utils; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -112,7 +107,8 @@ public class LogviewerLogSearchHandler { * @param resourceAuthorizer {@link ResourceAuthorizer} * @param metricsRegistry The logviewer metrics registry */ - public LogviewerLogSearchHandler(Map stormConf, Path logRoot, Path daemonLogRoot, + public LogviewerLogSearchHandler(Map stormConf, Path logRoot, + Path daemonLogRoot, ResourceAuthorizer resourceAuthorizer, StormMetricsRegistry metricsRegistry) { this.stormConf = stormConf; this.logRoot = logRoot.toAbsolutePath().normalize(); @@ -126,11 +122,16 @@ public LogviewerLogSearchHandler(Map stormConf, Path logRoot, Pa this.logviewerPort = ObjectReader.getInt(httpsPort); this.scheme = "https"; } - this.numDeepSearchNoResult = metricsRegistry.registerMeter("logviewer:num-deep-search-no-result"); - this.numFileScanned = metricsRegistry.registerHistogram("logviewer:num-files-scanned-per-deep-search"); - this.numSearchRequestNoResult = metricsRegistry.registerMeter("logviewer:num-search-request-no-result"); - this.numFileOpenExceptions = metricsRegistry.registerMeter(ExceptionMeterNames.NUM_FILE_OPEN_EXCEPTIONS); - this.numFileReadExceptions = metricsRegistry.registerMeter(ExceptionMeterNames.NUM_FILE_READ_EXCEPTIONS); + this.numDeepSearchNoResult = metricsRegistry + .registerMeter("logviewer:num-deep-search-no-result"); + this.numFileScanned = metricsRegistry + .registerHistogram("logviewer:num-files-scanned-per-deep-search"); + this.numSearchRequestNoResult = metricsRegistry + .registerMeter("logviewer:num-search-request-no-result"); + this.numFileOpenExceptions = metricsRegistry + .registerMeter(ExceptionMeterNames.NUM_FILE_OPEN_EXCEPTIONS); + this.numFileReadExceptions = metricsRegistry + .registerMeter(ExceptionMeterNames.NUM_FILE_READ_EXCEPTIONS); this.directoryCleaner = new DirectoryCleaner(metricsRegistry); } @@ -155,32 +156,39 @@ public Response searchLogFile(String fileName, String user, boolean isDaemon, St Path rootDir = isDaemon ? daemonLogRoot : logRoot; Path rawFile = rootDir.resolve(fileName); Path absFile = rawFile.toAbsolutePath().normalize(); - if (!absFile.startsWith(rootDir) || !rawFile.normalize().toString().equals(rawFile.toString())) { - //Ensure filename doesn't contain ../ parts + if (!absFile.startsWith(rootDir) || !rawFile.normalize().toString().equals(rawFile + .toString())) { + // Ensure filename doesn't contain ../ parts return searchLogFileNotFound(callback); } if (isDaemon && Paths.get(fileName).getNameCount() != 1) { - //Don't permit path traversal for calls intended to read from the daemon logs + // Don't permit path traversal for calls intended to read from the daemon logs return searchLogFileNotFound(callback); } Response response; if (absFile.toFile().exists()) { if (isDaemon ? resourceAuthorizer.isUserAllowedToAccessDaemonFile(user) : resourceAuthorizer.isUserAllowedToAccessFile(user, fileName)) { - Integer numMatchesInt = numMatchesStr != null ? tryParseIntParam("num-matches", numMatchesStr) : null; - Integer offsetInt = offsetStr != null ? tryParseIntParam("start-byte-offset", offsetStr) : null; + Integer numMatchesInt = numMatchesStr != null ? tryParseIntParam("num-matches", + numMatchesStr) : null; + Integer offsetInt = offsetStr != null ? tryParseIntParam("start-byte-offset", + offsetStr) : null; try { - if (StringUtils.isNotEmpty(search) && search.getBytes("UTF-8").length <= GREP_MAX_SEARCH_SIZE) { + if (StringUtils.isNotEmpty(search) && search + .getBytes("UTF-8").length <= GREP_MAX_SEARCH_SIZE) { Map entity = new HashMap<>(); entity.put("isDaemon", isDaemon ? "yes" : "no"); - Map res = substringSearch(absFile, search, isDaemon, numMatchesInt, offsetInt); + Map res = substringSearch(absFile, search, isDaemon, + numMatchesInt, offsetInt); entity.putAll(res); noResult = ((List) res.get("matches")).isEmpty(); - response = LogviewerResponseBuilder.buildSuccessJsonResponse(entity, callback, origin); + response = LogviewerResponseBuilder.buildSuccessJsonResponse(entity, + callback, origin); } else { - throw new InvalidRequestException("Search substring must be between 1 and 1024 " + throw new InvalidRequestException("Search substring must be between 1 and " + + "1024 " + "UTF-8 bytes in size (inclusive)"); } } catch (Exception ex) { @@ -188,7 +196,8 @@ public Response searchLogFile(String fileName, String user, boolean isDaemon, St } } else { // unauthorized - response = LogviewerResponseBuilder.buildUnauthorizedUserJsonResponse(user, callback); + response = LogviewerResponseBuilder.buildUnauthorizedUserJsonResponse(user, + callback); } } else { response = searchLogFileNotFound(callback); @@ -205,7 +214,8 @@ private Response searchLogFileNotFound(String callback) { entity.put("error", "Not Found"); entity.put("errorMessage", "The file was not found on this node."); - return new JsonResponseBuilder().setData(entity).setCallback(callback).setStatus(404).build(); + return new JsonResponseBuilder().setData(entity).setCallback(callback).setStatus(404) + .build(); } /** @@ -214,7 +224,8 @@ private Response searchLogFileNotFound(String callback) { * @param topologyId topology ID * @param user username * @param search search string - * @param numMatchesStr the count of maximum matches. Note that this number is with respect to each port, not to each log or each search + * @param numMatchesStr the count of maximum matches. Note that this number is with respect to + * each port, not to each log or each search * request * @param portStr worker port, null or '*' if the request wants to search from all worker logs * @param fileOffsetStr index (offset) of the log files @@ -233,7 +244,8 @@ public Response deepSearchLogsForTopology(String topologyId, String user, String Path rootDir = logRoot; Path absTopoDir = rootDir.resolve(topologyId).toAbsolutePath().normalize(); Object returnValue; - if (StringUtils.isEmpty(search) || !absTopoDir.toFile().exists() || !absTopoDir.startsWith(rootDir)) { + if (StringUtils.isEmpty(search) || !absTopoDir.toFile().exists() || !absTopoDir + .startsWith(rootDir)) { returnValue = new ArrayList<>(); } else { int fileOffset = ObjectReader.getInt(fileOffsetStr, 0); @@ -254,8 +266,10 @@ public Response deepSearchLogsForTopology(String topologyId, String user, String final List matchedList = portsOfLogs .map(logs -> findNMatches(logs, numMatches, 0, 0, search)) .collect(toList()); - numMatchedFiles = matchedList.stream().mapToInt(match -> match.getMatches().size()).sum(); - numScannedFiles = matchedList.stream().mapToInt(match -> match.openedFiles).sum(); + numMatchedFiles = matchedList.stream().mapToInt(match -> match.getMatches() + .size()).sum(); + numScannedFiles = matchedList.stream().mapToInt(match -> match.openedFiles) + .sum(); returnValue = matchedList; } } else { @@ -268,7 +282,8 @@ public Response deepSearchLogsForTopology(String topologyId, String user, String if (!containsPort) { returnValue = new ArrayList<>(); } else { - Path absPortDir = absTopoDir.resolve(Integer.toString(port)).toAbsolutePath().normalize(); + Path absPortDir = absTopoDir.resolve(Integer.toString(port)).toAbsolutePath() + .normalize(); if (!absPortDir.toFile().exists() || !absPortDir.startsWith(absTopoDir)) { @@ -279,7 +294,8 @@ public Response deepSearchLogsForTopology(String topologyId, String user, String filteredLogs = Collections.singletonList(first(filteredLogs)); fileOffset = 0; } - returnValue = findNMatches(filteredLogs, numMatches, fileOffset, offset, search); + returnValue = findNMatches(filteredLogs, numMatches, fileOffset, offset, + search); numMatchedFiles = ((Matched) returnValue).getMatches().size(); numScannedFiles = ((Matched) returnValue).openedFiles; } @@ -294,7 +310,8 @@ public Response deepSearchLogsForTopology(String topologyId, String user, String return LogviewerResponseBuilder.buildSuccessJsonResponse(returnValue, callback, origin); } - private Integer tryParseIntParam(String paramName, String value) throws InvalidRequestException { + private Integer tryParseIntParam(String paramName, + String value) throws InvalidRequestException { try { return Integer.parseInt(value); } catch (NumberFormatException e) { @@ -303,12 +320,14 @@ private Integer tryParseIntParam(String paramName, String value) throws InvalidR } @VisibleForTesting - Map substringSearch(Path file, String searchString) throws InvalidRequestException { + Map substringSearch(Path file, + String searchString) throws InvalidRequestException { return substringSearch(file, searchString, false, 10, 0); } @VisibleForTesting - Map substringSearch(Path file, String searchString, int numMatches) throws InvalidRequestException { + Map substringSearch(Path file, String searchString, + int numMatches) throws InvalidRequestException { return substringSearch(file, searchString, false, numMatches, 0); } @@ -320,13 +339,16 @@ Map substringSearch(Path file, return substringSearch(file, searchString, false, numMatches, startByteOffset); } - private Map substringSearch(Path file, String searchString, boolean isDaemon, Integer numMatches, + private Map substringSearch(Path file, String searchString, boolean isDaemon, + Integer numMatches, Integer startByteOffset) throws InvalidRequestException { if (StringUtils.isEmpty(searchString)) { - throw new IllegalArgumentException("Precondition fails: search string should not be empty."); + throw new IllegalArgumentException("Precondition fails: search string should not be " + + "empty."); } if (searchString.getBytes(StandardCharsets.UTF_8).length > GREP_MAX_SEARCH_SIZE) { - throw new IllegalArgumentException("Precondition fails: the length of search string should be less than " + throw new IllegalArgumentException("Precondition fails: the length of search string " + + "should be less than " + GREP_MAX_SEARCH_SIZE); } @@ -335,8 +357,9 @@ private Map substringSearch(Path file, String searchString, bool try (InputStream gzippedInputStream = isZipFile ? new GZIPInputStream(fis) : fis; BufferedInputStream stream = new BufferedInputStream(gzippedInputStream)) { - //It's more likely to be a file read exception here, so we don't differentiate - int fileLength = isZipFile ? (int) ServerUtils.zipFileSize(file.toFile()) : (int) Files.size(file); + // It's more likely to be a file read exception here, so we don't differentiate + int fileLength = isZipFile ? (int) ServerUtils.zipFileSize(file + .toFile()) : (int) Files.size(file); ByteBuffer buf = ByteBuffer.allocate(GREP_BUF_SIZE); final byte[] bufArray = buf.array(); @@ -345,7 +368,8 @@ private Map substringSearch(Path file, String searchString, bool startByteOffset = startByteOffset != null ? startByteOffset : 0; // Start at the part of the log file we are interested in. - // Allow searching when start-byte-offset == file-len so it doesn't blow up on 0-length files + // Allow searching when start-byte-offset == file-len so it doesn't blow up on + // 0-length files if (startByteOffset > fileLength) { throw new InvalidRequestException("Cannot search past the end of the file"); } @@ -368,22 +392,26 @@ private Map substringSearch(Path file, String searchString, bool Map ret = new HashMap<>(); while (true) { - SubstringSearchResult searchRet = bufferSubstringSearch(isDaemon, file, fileLength, byteOffset, initBufOffset, + SubstringSearchResult searchRet = bufferSubstringSearch(isDaemon, file, + fileLength, byteOffset, initBufOffset, stream, startByteOffset, totalBytesRead, buf, searchBytes, initialMatches, numMatches, beforeBytes); List> matches = searchRet.getMatches(); Integer newByteOffset = searchRet.getNewByteOffset(); byte[] newBeforeBytes = searchRet.getNewBeforeBytes(); - if (matches.size() < numMatches && totalBytesRead + startByteOffset < fileLength) { + if (matches.size() < numMatches && totalBytesRead + + startByteOffset < fileLength) { // The start index is positioned to find any possible // occurrence search string that did not quite fit in the // buffer on the previous read. - final int newBufOffset = Math.min(buf.limit(), GREP_MAX_SEARCH_SIZE) - searchBytes.length; + final int newBufOffset = Math.min(buf.limit(), + GREP_MAX_SEARCH_SIZE) - searchBytes.length; totalBytesRead = rotateGrepBuffer(buf, stream, totalBytesRead, fileLength); if (totalBytesRead < 0) { - throw new InvalidRequestException("Cannot search past the end of the file"); + throw new InvalidRequestException("Cannot search past the end of the " + + "file"); } initialMatches = matches; @@ -394,12 +422,14 @@ private Map substringSearch(Path file, String searchString, bool ret.put("isDaemon", isDaemon ? "yes" : "no"); Integer nextByteOffset = null; if (matches.size() >= numMatches || totalBytesRead < fileLength) { - nextByteOffset = (Integer) last(matches).get("byteOffset") + searchBytes.length; + nextByteOffset = (Integer) last(matches).get("byteOffset") + + searchBytes.length; if (fileLength <= nextByteOffset) { nextByteOffset = null; } } - ret.putAll(mkGrepResponse(searchBytes, startByteOffset, matches, nextByteOffset)); + ret.putAll(mkGrepResponse(searchBytes, startByteOffset, matches, + nextByteOffset)); break; } } @@ -417,7 +447,8 @@ private Map substringSearch(Path file, String searchString, bool } @VisibleForTesting - Map substringSearchDaemonLog(Path file, String searchString) throws InvalidRequestException { + Map substringSearchDaemonLog(Path file, + String searchString) throws InvalidRequestException { return substringSearch(file, searchString, true, 10, 0); } @@ -428,11 +459,13 @@ Map substringSearchDaemonLog(Path file, String searchString) thr List logsForPort(String user, Path portDir) { try { List workerLogs = directoryCleaner.getFilesForDir(portDir).stream() - .filter(file -> WORKER_LOG_FILENAME_PATTERN.asPredicate().test(file.getFileName().toString())) + .filter(file -> WORKER_LOG_FILENAME_PATTERN.asPredicate().test(file.getFileName() + .toString())) .collect(toList()); return workerLogs.stream() - .filter(log -> resourceAuthorizer.isUserAllowedToAccessFile(user, WorkerLogs.getTopologyPortWorkerLog(log))) + .filter(log -> resourceAuthorizer.isUserAllowedToAccessFile(user, WorkerLogs + .getTopologyPortWorkerLog(log))) .map(p -> { try { return Pair.of(p, Files.getLastModifiedTime(p)); @@ -459,7 +492,8 @@ List logsForPort(String user, Path portDir) { * @return all matched results */ @VisibleForTesting - Matched findNMatches(List logs, int numMatches, int fileOffset, int startByteOffset, String targetStr) { + Matched findNMatches(List logs, int numMatches, int fileOffset, int startByteOffset, + String targetStr) { logs = drop(logs, fileOffset); LOG.debug("{} files to scan", logs.size()); @@ -469,7 +503,7 @@ Matched findNMatches(List logs, int numMatches, int fileOffset, int startB while (true) { if (logs.isEmpty()) { - //fileOffset = one past last scanned file + // fileOffset = one past last scanned file break; } @@ -477,7 +511,8 @@ Matched findNMatches(List logs, int numMatches, int fileOffset, int startB Map matchInLog; try { LOG.debug("Looking through {}", firstLog); - matchInLog = substringSearch(firstLog, targetStr, numMatches - matchCount, startByteOffset); + matchInLog = substringSearch(firstLog, targetStr, numMatches - matchCount, + startByteOffset); scannedFiles++; } catch (InvalidRequestException e) { LOG.error("Can't search past end of file.", e); @@ -486,15 +521,18 @@ Matched findNMatches(List logs, int numMatches, int fileOffset, int startB String fileName = WorkerLogs.getTopologyPortWorkerLog(firstLog); - //This section simply put the formatted log filename and corresponding port in the matching. + // This section simply put the formatted log filename and corresponding port in the + // matching. final List> newMatches = new ArrayList<>(matches); Map currentFileMatch = new HashMap<>(matchInLog); currentFileMatch.put("fileName", fileName); Path firstLogAbsPath = firstLog.toAbsolutePath().normalize(); - currentFileMatch.put("port", truncatePathToLastElements(firstLogAbsPath, 2).getName(0).toString()); + currentFileMatch.put("port", truncatePathToLastElements(firstLogAbsPath, 2).getName(0) + .toString()); newMatches.add(currentFileMatch); - int newCount = matchCount + ((List) matchInLog.getOrDefault("matches", Collections.emptyList())).size(); + int newCount = matchCount + ((List) matchInLog.getOrDefault("matches", Collections + .emptyList())).size(); if (newCount == matchCount) { // matches and matchCount is not changed logs = rest(logs); @@ -502,7 +540,7 @@ Matched findNMatches(List logs, int numMatches, int fileOffset, int startB fileOffset = fileOffset + 1; } else if (newCount >= numMatches) { matches = newMatches; - //fileOffset = the index of last scanned file + // fileOffset = the index of last scanned file break; } else { matches = newMatches; @@ -518,10 +556,12 @@ Matched findNMatches(List logs, int numMatches, int fileOffset, int startB } /** - * As the file is read into a buffer, 1/2 the buffer's size at a time, we search the buffer for matches of the substring and return a + * As the file is read into a buffer, 1/2 the buffer's size at a time, we search the buffer for + * matches of the substring and return a * list of zero or more matches. */ - private SubstringSearchResult bufferSubstringSearch(boolean isDaemon, Path file, int fileLength, int offsetToBuf, + private SubstringSearchResult bufferSubstringSearch(boolean isDaemon, Path file, int fileLength, + int offsetToBuf, int initBufOffset, BufferedInputStream stream, Integer bytesSkipped, int bytesRead, ByteBuffer haystack, byte[] needle, List> initialMatches, Integer numMatches, byte[] beforeBytes) @@ -536,7 +576,8 @@ private SubstringSearchResult bufferSubstringSearch(boolean isDaemon, Path file, int offset = offsetOfBytes(haystack.array(), needle, bufOffset); if (matches.size() < numMatches && offset >= 0) { final int fileOffset = offsetToBuf + offset; - final int bytesNeededAfterMatch = haystack.limit() - GREP_CONTEXT_SIZE - needle.length; + final int bytesNeededAfterMatch = haystack + .limit() - GREP_CONTEXT_SIZE - needle.length; byte[] beforeArg = null; byte[] afterArg = null; @@ -554,12 +595,14 @@ private SubstringSearchResult bufferSubstringSearch(boolean isDaemon, Path file, } else { int beforeStrToOffset = Math.min(haystack.limit(), GREP_MAX_SEARCH_SIZE); int beforeStrFromOffset = Math.max(0, beforeStrToOffset - GREP_CONTEXT_SIZE); - newBeforeBytes = Arrays.copyOfRange(haystack.array(), beforeStrFromOffset, beforeStrToOffset); + newBeforeBytes = Arrays.copyOfRange(haystack.array(), beforeStrFromOffset, + beforeStrToOffset); // It's OK if new-byte-offset is negative. // This is normal if we are out of bytes to read from a small file. if (matches.size() >= numMatches) { - newByteOffset = ((Number) last(matches).get("byteOffset")).intValue() + needle.length; + newByteOffset = ((Number) last(matches).get("byteOffset")).intValue() + + needle.length; } else { newByteOffset = bytesSkipped + bytesRead - GREP_MAX_SEARCH_SIZE; } @@ -571,7 +614,8 @@ private SubstringSearchResult bufferSubstringSearch(boolean isDaemon, Path file, return new SubstringSearchResult(matches, newByteOffset, newBeforeBytes); } - private int rotateGrepBuffer(ByteBuffer buf, BufferedInputStream stream, int totalBytesRead, int fileLength) throws IOException { + private int rotateGrepBuffer(ByteBuffer buf, BufferedInputStream stream, int totalBytesRead, + int fileLength) throws IOException { byte[] bufArray = buf.array(); // Copy the 2nd half of the buffer to the first half. @@ -581,17 +625,20 @@ private int rotateGrepBuffer(ByteBuffer buf, BufferedInputStream stream, int tot Arrays.fill(bufArray, GREP_MAX_SEARCH_SIZE, bufArray.length, (byte) 0); // Fill the 2nd half with new bytes from the stream. - int bytesRead = stream.read(bufArray, GREP_MAX_SEARCH_SIZE, Math.min(fileLength, GREP_MAX_SEARCH_SIZE)); + int bytesRead = stream.read(bufArray, GREP_MAX_SEARCH_SIZE, Math.min(fileLength, + GREP_MAX_SEARCH_SIZE)); buf.limit(GREP_MAX_SEARCH_SIZE + bytesRead); return totalBytesRead + bytesRead; } - private Map mkMatchData(byte[] needle, ByteBuffer haystack, int haystackOffset, int fileOffset, Path canonicalPath, + private Map mkMatchData(byte[] needle, ByteBuffer haystack, int haystackOffset, + int fileOffset, Path canonicalPath, boolean isDaemon, byte[] beforeBytes, byte[] afterBytes) throws UnsupportedEncodingException, UnknownHostException { String url; if (isDaemon) { - url = urlToMatchCenteredInLogPageDaemonFile(needle, canonicalPath, fileOffset, logviewerPort); + url = urlToMatchCenteredInLogPageDaemonFile(needle, canonicalPath, fileOffset, + logviewerPort); } else { url = urlToMatchCenteredInLogPage(needle, canonicalPath, fileOffset, logviewerPort); } @@ -601,7 +648,8 @@ private Map mkMatchData(byte[] needle, ByteBuffer haystack, int String afterString; if (haystackOffset >= GREP_CONTEXT_SIZE) { - beforeString = new String(haystackBytes, (haystackOffset - GREP_CONTEXT_SIZE), GREP_CONTEXT_SIZE, "UTF-8"); + beforeString = new String(haystackBytes, (haystackOffset - GREP_CONTEXT_SIZE), + GREP_CONTEXT_SIZE, "UTF-8"); } else { int numDesired = Math.max(0, GREP_CONTEXT_SIZE - haystackOffset); int beforeSize = beforeBytes != null ? beforeBytes.length : 0; @@ -630,11 +678,13 @@ private Map mkMatchData(byte[] needle, ByteBuffer haystack, int if (numExpected > 0) { StringBuilder sb = new StringBuilder(); - sb.append(new String(haystackBytes, afterOffset, (haystackSize - afterOffset), "UTF-8")); + sb.append(new String(haystackBytes, afterOffset, (haystackSize - afterOffset), + "UTF-8")); sb.append(new String(afterBytes, 0, numExpected, "UTF-8")); afterString = sb.toString(); } else { - afterString = new String(haystackBytes, afterOffset, (haystackSize - afterOffset), "UTF-8"); + afterString = new String(haystackBytes, afterOffset, (haystackSize - afterOffset), + "UTF-8"); } } @@ -649,9 +699,11 @@ private Map mkMatchData(byte[] needle, ByteBuffer haystack, int } /** - * Tries once to read ahead in the stream to fill the context and resets the stream to its position before the call. + * Tries once to read ahead in the stream to fill the context and resets the stream to its + * position before the call. */ - private byte[] tryReadAhead(BufferedInputStream stream, ByteBuffer haystack, int offset, int fileLength, int bytesRead) + private byte[] tryReadAhead(BufferedInputStream stream, ByteBuffer haystack, int offset, + int fileLength, int bytesRead) throws IOException { int numExpected = Math.min(fileLength - bytesRead, GREP_CONTEXT_SIZE); byte[] afterBytes = new byte[numExpected]; @@ -663,7 +715,8 @@ private byte[] tryReadAhead(BufferedInputStream stream, ByteBuffer haystack, int } /** - * Searches a given byte array for a match of a sub-array of bytes. Returns the offset to the byte that matches, or -1 if no match was + * Searches a given byte array for a match of a sub-array of bytes. Returns the offset to the + * byte that matches, or -1 if no match was * found. */ private int offsetOfBytes(byte[] buffer, byte[] search, int initOffset) { @@ -714,7 +767,8 @@ private int offsetOfBytes(byte[] buffer, byte[] search, int initOffset) { /** * This response data only includes a next byte offset if there is more of the file to read. */ - private Map mkGrepResponse(byte[] searchBytes, Integer offset, List> matches, + private Map mkGrepResponse(byte[] searchBytes, Integer offset, List> matches, Integer nextByteOffset) throws UnsupportedEncodingException { Map ret = new HashMap<>(); ret.put("searchString", new String(searchBytes, "UTF-8")); @@ -727,29 +781,35 @@ private Map mkGrepResponse(byte[] searchBytes, Integer offset, L } @VisibleForTesting - String urlToMatchCenteredInLogPage(byte[] needle, Path canonicalPath, int offset, Integer port) throws UnknownHostException { + String urlToMatchCenteredInLogPage(byte[] needle, Path canonicalPath, int offset, + Integer port) throws UnknownHostException { final String host = Utils.hostname(); final Path truncatedFilePath = truncatePathToLastElements(canonicalPath, 3); Map parameters = new HashMap<>(); parameters.put("file", truncatedFilePath.toString()); - parameters.put("start", Math.max(0, offset - (LogviewerConstant.DEFAULT_BYTES_PER_PAGE / 2) - (needle.length / -2))); + parameters.put("start", Math.max(0, + offset - (LogviewerConstant.DEFAULT_BYTES_PER_PAGE / 2) - (needle.length / -2))); parameters.put("length", LogviewerConstant.DEFAULT_BYTES_PER_PAGE); - return UrlBuilder.build(String.format(this.scheme + "://%s:%d/api/v1/log", host, port), parameters); + return UrlBuilder.build(String.format(this.scheme + "://%s:%d/api/v1/log", host, port), + parameters); } @VisibleForTesting - String urlToMatchCenteredInLogPageDaemonFile(byte[] needle, Path canonicalPath, int offset, Integer port) throws UnknownHostException { + String urlToMatchCenteredInLogPageDaemonFile(byte[] needle, Path canonicalPath, int offset, + Integer port) throws UnknownHostException { final String host = Utils.hostname(); final Path truncatedFilePath = truncatePathToLastElements(canonicalPath, 1); Map parameters = new HashMap<>(); parameters.put("file", truncatedFilePath.toString()); - parameters.put("start", Math.max(0, offset - (LogviewerConstant.DEFAULT_BYTES_PER_PAGE / 2) - (needle.length / -2))); + parameters.put("start", Math.max(0, + offset - (LogviewerConstant.DEFAULT_BYTES_PER_PAGE / 2) - (needle.length / -2))); parameters.put("length", LogviewerConstant.DEFAULT_BYTES_PER_PAGE); - return UrlBuilder.build(String.format(this.scheme + "://%s:%d/api/v1/daemonlog", host, port), parameters); + return UrlBuilder.build(String.format(this.scheme + "://%s:%d/api/v1/daemonlog", host, + port), parameters); } @VisibleForTesting @@ -771,7 +831,8 @@ public static class Matched implements JSONAware { * @param matches map representing matched search result * @param openedFiles number of files scanned, used for metrics only */ - public Matched(int fileOffset, String searchString, List> matches, int openedFiles) { + public Matched(int fileOffset, String searchString, List> matches, + int openedFiles) { this.fileOffset = fileOffset; this.searchString = searchString; this.matches = matches; @@ -806,7 +867,8 @@ private static class SubstringSearchResult { private Integer newByteOffset; private byte[] newBeforeBytes; - SubstringSearchResult(List> matches, Integer newByteOffset, byte[] newBeforeBytes) { + SubstringSearchResult(List> matches, Integer newByteOffset, + byte[] newBeforeBytes) { this.matches = matches; this.newByteOffset = newByteOffset; this.newBeforeBytes = newBeforeBytes; diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerProfileHandler.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerProfileHandler.java index 566d5dcaded..e899b592426 100644 --- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerProfileHandler.java +++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/handler/LogviewerProfileHandler.java @@ -30,15 +30,12 @@ import com.codahale.metrics.Meter; import j2html.tags.DomContent; - import jakarta.ws.rs.core.Response; - import java.io.File; import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; import java.util.List; - import org.apache.commons.lang3.StringUtils; import org.apache.storm.daemon.logviewer.utils.DirectoryCleaner; import org.apache.storm.daemon.logviewer.utils.ExceptionMeterNames; @@ -63,10 +60,12 @@ public class LogviewerProfileHandler { * @param resourceAuthorizer {@link ResourceAuthorizer} * @param metricsRegistry The logviewer metrisc registry */ - public LogviewerProfileHandler(String logRoot, ResourceAuthorizer resourceAuthorizer, StormMetricsRegistry metricsRegistry) { + public LogviewerProfileHandler(String logRoot, ResourceAuthorizer resourceAuthorizer, + StormMetricsRegistry metricsRegistry) { this.logRoot = Paths.get(logRoot).toAbsolutePath().normalize(); this.resourceAuthorizer = resourceAuthorizer; - this.numFileDownloadExceptions = metricsRegistry.registerMeter(ExceptionMeterNames.NUM_FILE_DOWNLOAD_EXCEPTIONS); + this.numFileDownloadExceptions = metricsRegistry + .registerMeter(ExceptionMeterNames.NUM_FILE_DOWNLOAD_EXCEPTIONS); this.directoryCleaner = new DirectoryCleaner(metricsRegistry); } @@ -78,17 +77,20 @@ public LogviewerProfileHandler(String logRoot, ResourceAuthorizer resourceAuthor * @param user username * @return The HTML page representing list page of dump files */ - public Response listDumpFiles(String topologyId, String hostPort, String user) throws IOException { + public Response listDumpFiles(String topologyId, String hostPort, + String user) throws IOException { String portStr = hostPort.split(":")[1]; Path rawDir = logRoot.resolve(topologyId).resolve(portStr); Path absDir = rawDir.toAbsolutePath().normalize(); - if (!absDir.startsWith(logRoot) || !rawDir.normalize().toString().equals(rawDir.toString())) { - //Ensure filename doesn't contain ../ parts + if (!absDir.startsWith(logRoot) || !rawDir.normalize().toString().equals(rawDir + .toString())) { + // Ensure filename doesn't contain ../ parts return LogviewerResponseBuilder.buildResponsePageNotFound(); } if (absDir.toFile().exists()) { - String workerFileRelativePath = String.join(File.separator, topologyId, portStr, WORKER_LOG_FILENAME); + String workerFileRelativePath = String.join(File.separator, topologyId, portStr, + WORKER_LOG_FILENAME); if (resourceAuthorizer.isUserAllowedToAccessFile(user, workerFileRelativePath)) { String content = buildDumpFileListPage(topologyId, hostPort, absDir.toFile()); return LogviewerResponseBuilder.buildSuccessHtmlResponse(content); @@ -109,22 +111,27 @@ public Response listDumpFiles(String topologyId, String hostPort, String user) t * @param user username * @return a Response which lets browsers download that file. */ - public Response downloadDumpFile(String topologyId, String hostPort, String fileName, String user) throws IOException { + public Response downloadDumpFile(String topologyId, String hostPort, String fileName, + String user) throws IOException { String[] hostPortSplit = hostPort.split(":"); String host = hostPortSplit[0]; String portStr = hostPortSplit[1]; Path rawFile = logRoot.resolve(topologyId).resolve(portStr).resolve(fileName); Path absFile = rawFile.toAbsolutePath().normalize(); - if (!absFile.startsWith(logRoot) || !rawFile.normalize().toString().equals(rawFile.toString())) { - //Ensure filename doesn't contain ../ parts + if (!absFile.startsWith(logRoot) || !rawFile.normalize().toString().equals(rawFile + .toString())) { + // Ensure filename doesn't contain ../ parts return LogviewerResponseBuilder.buildResponsePageNotFound(); } if (absFile.toFile().exists()) { - String workerFileRelativePath = String.join(File.separator, topologyId, portStr, WORKER_LOG_FILENAME); + String workerFileRelativePath = String.join(File.separator, topologyId, portStr, + WORKER_LOG_FILENAME); if (resourceAuthorizer.isUserAllowedToAccessFile(user, workerFileRelativePath)) { - String downloadedFileName = host + "-" + topologyId + "-" + portStr + "-" + absFile.getFileName(); - return LogviewerResponseBuilder.buildDownloadFile(downloadedFileName, absFile.toFile(), numFileDownloadExceptions); + String downloadedFileName = host + "-" + topologyId + "-" + portStr + "-" + absFile + .getFileName(); + return LogviewerResponseBuilder.buildDownloadFile(downloadedFileName, absFile + .toFile(), numFileDownloadExceptions); } else { return LogviewerResponseBuilder.buildResponseUnauthorizedUser(user); } @@ -133,9 +140,11 @@ public Response downloadDumpFile(String topologyId, String hostPort, String file } } - private String buildDumpFileListPage(String topologyId, String hostPort, File dir) throws IOException { + private String buildDumpFileListPage(String topologyId, String hostPort, + File dir) throws IOException { List liTags = getProfilerDumpFiles(dir).stream() - .map(file -> li(a(file).withHref("/api/v1/dumps/" + topologyId + "/" + hostPort + "/" + file))) + .map(file -> li(a(file).withHref("/api/v1/dumps/" + topologyId + "/" + hostPort + "/" + + file))) .collect(toList()); return html( @@ -158,7 +167,8 @@ private List getProfilerDumpFiles(File dir) throws IOException { .filter(file -> { String fileName = file.getName(); return StringUtils.isNotEmpty(fileName) - && (fileName.endsWith(".txt") || fileName.endsWith(".jfr") || fileName.endsWith(".bin")); + && (fileName.endsWith(".txt") || fileName.endsWith(".jfr") || fileName + .endsWith(".bin")); }).map(File::getName).collect(toList()); } diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/DeletionMeta.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/DeletionMeta.java index 9e0afd97e96..d854a66dd4d 100644 --- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/DeletionMeta.java +++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/DeletionMeta.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

      Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/DirectoryCleaner.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/DirectoryCleaner.java index 8d1da3509b6..881019f9f6c 100644 --- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/DirectoryCleaner.java +++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/DirectoryCleaner.java @@ -7,9 +7,9 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software + *

      Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and @@ -34,7 +34,6 @@ import java.util.regex.Pattern; import org.apache.commons.lang3.tuple.Pair; import org.apache.storm.metric.StormMetricsRegistry; - import org.apache.storm.utils.Utils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -47,7 +46,8 @@ public class DirectoryCleaner { private static final Logger LOG = LoggerFactory.getLogger(DirectoryCleaner.class); // used to recognize the pattern of active log files, we may remove the "current" from this list - private static final Pattern ACTIVE_LOG_PATTERN = Pattern.compile(".*\\.(log|err|out|current|yaml|pid|metrics)$"); + private static final Pattern ACTIVE_LOG_PATTERN = Pattern + .compile(".*\\.(log|err|out|current|yaml|pid|metrics)$"); // used to recognize the pattern of some meta files in a worker log directory private static final Pattern META_LOG_PATTERN = Pattern.compile(".*\\.(yaml|pid)$"); @@ -58,7 +58,8 @@ public class DirectoryCleaner { private final Meter numFileOpenExceptions; public DirectoryCleaner(StormMetricsRegistry metricsRegistry) { - this.numFileOpenExceptions = metricsRegistry.registerMeter(ExceptionMeterNames.NUM_FILE_OPEN_EXCEPTIONS); + this.numFileOpenExceptions = metricsRegistry + .registerMeter(ExceptionMeterNames.NUM_FILE_OPEN_EXCEPTIONS); } /** @@ -83,7 +84,8 @@ public DirectoryStream getStreamForDirectory(Path dir) throws IOException * * @param dirs the list of directories to be scanned for deletion * @param quota the per-dir quota or the total quota for the all directories - * @param forPerDir if true, deletion happens for a single dir; otherwise, for all directories globally + * @param forPerDir if true, deletion happens for a single dir; otherwise, for all directories + * globally * @param activeDirs only for global deletion, we want to skip the active logs in activeDirs * @return number of files deleted */ @@ -105,13 +107,15 @@ public DeletionMeta deleteOldestWhileTooLarge(List dirs, int deletedFiles = 0; long deletedSize = 0; - // the oldest pq_size files in this directory will be placed in PQ, with the newest at the root + // the oldest pq_size files in this directory will be placed in PQ, with the newest at the + // root PriorityQueue> pq = new PriorityQueue<>(PQ_SIZE, Comparator.comparing((Pair p) -> p.getRight()).reversed()); int round = 0; final Set excluded = new HashSet<>(); while (toDeleteSize > 0) { - LOG.debug("To delete size is {}, start a new round of deletion, round: {}", toDeleteSize, round); + LOG.debug("To delete size is {}, start a new round of deletion, round: {}", + toDeleteSize, round); for (Path dir : dirs) { try (DirectoryStream stream = getStreamForDirectory(dir)) { for (Path path : stream) { @@ -119,10 +123,12 @@ public DeletionMeta deleteOldestWhileTooLarge(List dirs, if (isFileEligibleToSkipDelete(forPerDir, activeDirs, dir, path)) { excluded.add(path); } else { - Pair p = Pair.of(path, Files.getLastModifiedTime(path)); + Pair p = Pair.of(path, Files + .getLastModifiedTime(path)); if (pq.size() < PQ_SIZE) { pq.offer(p); - } else if (p.getRight().toMillis() < pq.peek().getRight().toMillis()) { + } else if (p.getRight().toMillis() < pq.peek().getRight() + .toMillis()) { pq.poll(); pq.offer(p); } @@ -143,10 +149,11 @@ public DeletionMeta deleteOldestWhileTooLarge(List dirs, final String canonicalPath = file.toAbsolutePath().normalize().toString(); final long fileSize = Files.size(file); final long lastModified = pair.getRight().toMillis(); - //Original implementation doesn't actually check if delete succeeded or not. + // Original implementation doesn't actually check if delete succeeded or not. try { Utils.forceDelete(file.toString()); - LOG.info("Delete file: {}, size: {}, lastModified: {}", canonicalPath, fileSize, lastModified); + LOG.info("Delete file: {}, size: {}, lastModified: {}", canonicalPath, + fileSize, lastModified); toDeleteSize -= fileSize; deletedSize += fileSize; deletedFiles++; @@ -158,11 +165,14 @@ public DeletionMeta deleteOldestWhileTooLarge(List dirs, round++; if (round >= MAX_ROUNDS) { if (forPerDir) { - LOG.warn("Reach the MAX_ROUNDS: {} during per-dir deletion, you may have too many files in " - + "a single directory : {}, will delete the rest files in next interval.", + LOG.warn("Reach the MAX_ROUNDS: {} during per-dir deletion, you may have " + + "too many files in " + + "a single directory : {}, will delete the rest files in next " + + "interval.", MAX_ROUNDS, dirs.get(0).toAbsolutePath().normalize()); } else { - LOG.warn("Reach the MAX_ROUNDS: {} during global deletion, you may have too many files, " + LOG.warn("Reach the MAX_ROUNDS: {} during global deletion, you may have " + + "too many files, " + "will delete the rest files in next interval.", MAX_ROUNDS); } break; @@ -170,8 +180,10 @@ public DeletionMeta deleteOldestWhileTooLarge(List dirs, } else { LOG.warn("No more files able to delete this round, but {} is over quota by {} MB", forPerDir ? "this directory" : "root directory", toDeleteSize * 1e-6); - LOG.warn("No more files eligible to be deleted this round, but {} is over {} quota by {} MB", - forPerDir ? "worker directory: " + dirs.get(0).toAbsolutePath().normalize() : "log root directory", + LOG.warn("No more files eligible to be deleted this round, but {} is over {} " + + "quota by {} MB", + forPerDir ? "worker directory: " + dirs.get(0).toAbsolutePath() + .normalize() : "log root directory", forPerDir ? "per-worker" : "global", toDeleteSize * 1e-6); break; // No entries left to delete } @@ -179,12 +191,14 @@ public DeletionMeta deleteOldestWhileTooLarge(List dirs, return new DeletionMeta(deletedSize, deletedFiles); } - private boolean isFileEligibleToSkipDelete(boolean forPerDir, Set activeDirs, Path dir, Path file) throws IOException { + private boolean isFileEligibleToSkipDelete(boolean forPerDir, Set activeDirs, Path dir, + Path file) throws IOException { if (forPerDir) { return ACTIVE_LOG_PATTERN.matcher(file.getFileName().toString()).matches(); } else { // for global cleanup // for an active worker's dir, make sure for the last "/" - return activeDirs.contains(dir) ? ACTIVE_LOG_PATTERN.matcher(file.getFileName().toString()).matches() : + return activeDirs.contains(dir) ? ACTIVE_LOG_PATTERN.matcher(file.getFileName() + .toString()).matches() : META_LOG_PATTERN.matcher(file.getFileName().toString()).matches(); } } diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/ExceptionMeterNames.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/ExceptionMeterNames.java index 61055ba2612..895871bde3e 100644 --- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/ExceptionMeterNames.java +++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/ExceptionMeterNames.java @@ -1,14 +1,20 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. - * See the NOTICE file distributed with this work for additional information regarding copyright ownership. + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. + * See the NOTICE file distributed with this work for additional information regarding copyright + * ownership. * The ASF licenses this file to you under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

      - * http://www.apache.org/licenses/LICENSE-2.0 - *

      - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * you may not use this file except in compliance with the License. You may obtain a copy of the + * License at + * + *

      http://www.apache.org/licenses/LICENSE-2.0 + * + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -18,26 +24,36 @@ public class ExceptionMeterNames { - //Operation level IO Exceptions + // Operation level IO Exceptions public static final String NUM_FILE_OPEN_EXCEPTIONS = "logviewer:num-file-open-exceptions"; public static final String NUM_FILE_READ_EXCEPTIONS = "logviewer:num-file-read-exceptions"; - public static final String NUM_FILE_REMOVAL_EXCEPTIONS = "logviewer:num-file-removal-exceptions"; - public static final String NUM_FILE_DOWNLOAD_EXCEPTIONS = "logviewer:num-file-download-exceptions"; - public static final String NUM_SET_PERMISSION_EXCEPTIONS = "logviewer:num-set-permission-exceptions"; + public static final String NUM_FILE_REMOVAL_EXCEPTIONS = + "logviewer:num-file-removal-exceptions"; + public static final String NUM_FILE_DOWNLOAD_EXCEPTIONS = + "logviewer:num-file-download-exceptions"; + public static final String NUM_SET_PERMISSION_EXCEPTIONS = + "logviewer:num-set-permission-exceptions"; - //Routine level + // Routine level public static final String NUM_CLEANUP_EXCEPTIONS = "logviewer:num-other-cleanup-exceptions"; public static final String NUM_READ_LOG_EXCEPTIONS = "logviewer:num-read-log-exceptions"; - public static final String NUM_READ_DAEMON_LOG_EXCEPTIONS = "logviewer:num-read-daemon-log-exceptions"; + public static final String NUM_READ_DAEMON_LOG_EXCEPTIONS = + "logviewer:num-read-daemon-log-exceptions"; public static final String NUM_LIST_LOG_EXCEPTIONS = "logviewer:num-search-log-exceptions"; - public static final String NUM_LIST_DUMP_EXCEPTIONS = "logviewer:num-list-dump-files-exceptions"; - public static final String NUM_DOWNLOAD_DUMP_EXCEPTIONS = "logviewer:num-download-dump-exceptions"; - public static final String NUM_DOWNLOAD_LOG_EXCEPTIONS = "logviewer:num-download-log-exceptions"; - public static final String NUM_DOWNLOAD_DAEMON_LOG_EXCEPTIONS = "logviewer:num-download-daemon-log-exceptions"; + public static final String NUM_LIST_DUMP_EXCEPTIONS = + "logviewer:num-list-dump-files-exceptions"; + public static final String NUM_DOWNLOAD_DUMP_EXCEPTIONS = + "logviewer:num-download-dump-exceptions"; + public static final String NUM_DOWNLOAD_LOG_EXCEPTIONS = + "logviewer:num-download-log-exceptions"; + public static final String NUM_DOWNLOAD_DAEMON_LOG_EXCEPTIONS = + "logviewer:num-download-daemon-log-exceptions"; public static final String NUM_SEARCH_EXCEPTIONS = "logviewer:num-search-exceptions"; /** - * It may be helpful to register these meters up front, so they are output even if their values are zero. + * It may be helpful to register these meters up front, so they are output even if their values + * are zero. + * * @param registry The metrics registry. */ public static void registerMeters(StormMetricsRegistry registry) { diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogCleaner.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogCleaner.java index f5740d8241e..dc089c5f130 100644 --- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogCleaner.java +++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogCleaner.java @@ -20,7 +20,6 @@ import static java.util.stream.Collectors.joining; import static java.util.stream.Collectors.toList; - import static org.apache.storm.DaemonConfig.LOGVIEWER_CLEANUP_AGE_MINS; import static org.apache.storm.DaemonConfig.LOGVIEWER_CLEANUP_INTERVAL_SECS; import static org.apache.storm.DaemonConfig.LOGVIEWER_MAX_PER_WORKER_LOGS_SIZE_MB; @@ -30,7 +29,6 @@ import com.codahale.metrics.Meter; import com.codahale.metrics.Timer; import com.google.common.annotations.VisibleForTesting; - import java.io.Closeable; import java.io.IOException; import java.nio.file.DirectoryStream; @@ -50,7 +48,6 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import java.util.stream.StreamSupport; - import org.apache.storm.StormTimer; import org.apache.storm.metric.StormMetricsRegistry; import org.apache.storm.utils.ObjectReader; @@ -90,27 +87,38 @@ public class LogCleaner implements Runnable, Closeable { * @param logRootDir root log directory * @param metricsRegistry The logviewer metrics registry */ - public LogCleaner(Map stormConf, WorkerLogs workerLogs, DirectoryCleaner directoryCleaner, + public LogCleaner(Map stormConf, WorkerLogs workerLogs, + DirectoryCleaner directoryCleaner, Path logRootDir, StormMetricsRegistry metricsRegistry) { this.stormConf = stormConf; - this.intervalSecs = ObjectReader.getInt(stormConf.get(LOGVIEWER_CLEANUP_INTERVAL_SECS), null); + this.intervalSecs = ObjectReader.getInt(stormConf.get(LOGVIEWER_CLEANUP_INTERVAL_SECS), + null); this.logRootDir = logRootDir; this.workerLogs = workerLogs; this.directoryCleaner = directoryCleaner; - maxSumWorkerLogsSizeMb = ObjectReader.getInt(stormConf.get(LOGVIEWER_MAX_SUM_WORKER_LOGS_SIZE_MB)); - maxPerWorkerLogsSizeMb = ObjectReader.getInt(stormConf.get(LOGVIEWER_MAX_PER_WORKER_LOGS_SIZE_MB)); - maxPerWorkerLogsSizeMb = Math.min(maxPerWorkerLogsSizeMb, (long) (maxSumWorkerLogsSizeMb * 0.5)); + maxSumWorkerLogsSizeMb = ObjectReader.getInt(stormConf + .get(LOGVIEWER_MAX_SUM_WORKER_LOGS_SIZE_MB)); + maxPerWorkerLogsSizeMb = ObjectReader.getInt(stormConf + .get(LOGVIEWER_MAX_PER_WORKER_LOGS_SIZE_MB)); + maxPerWorkerLogsSizeMb = Math.min(maxPerWorkerLogsSizeMb, + (long) (maxSumWorkerLogsSizeMb * 0.5)); - LOG.info("configured max total size of worker logs: {} MB, max total size of worker logs per directory: {} MB", + LOG.info("configured max total size of worker logs: {} MB, max total size of worker logs " + + "per directory: {} MB", maxSumWorkerLogsSizeMb, maxPerWorkerLogsSizeMb); - //Switch to CachedGauge if this starts to hurt performance + // Switch to CachedGauge if this starts to hurt performance metricsRegistry.registerGauge("logviewer:worker-log-dir-size", () -> sizeOfDir(logRootDir)); - this.cleanupRoutineDuration = metricsRegistry.registerTimer("logviewer:cleanup-routine-duration-ms"); - this.numFilesCleanedUp = metricsRegistry.registerHistogram("logviewer:num-files-cleaned-up"); - this.diskSpaceFreed = metricsRegistry.registerHistogram("logviewer:disk-space-freed-in-bytes"); - this.numFileRemovalExceptions = metricsRegistry.registerMeter(ExceptionMeterNames.NUM_FILE_REMOVAL_EXCEPTIONS); - this.numCleanupExceptions = metricsRegistry.registerMeter(ExceptionMeterNames.NUM_CLEANUP_EXCEPTIONS); + this.cleanupRoutineDuration = metricsRegistry + .registerTimer("logviewer:cleanup-routine-duration-ms"); + this.numFilesCleanedUp = metricsRegistry + .registerHistogram("logviewer:num-files-cleaned-up"); + this.diskSpaceFreed = metricsRegistry + .registerHistogram("logviewer:disk-space-freed-in-bytes"); + this.numFileRemovalExceptions = metricsRegistry + .registerMeter(ExceptionMeterNames.NUM_FILE_REMOVAL_EXCEPTIONS); + this.numCleanupExceptions = metricsRegistry + .registerMeter(ExceptionMeterNames.NUM_CLEANUP_EXCEPTIONS); } private long sizeOfDir(Path dir) { @@ -120,7 +128,7 @@ private long sizeOfDir(Path dir) { .mapToLong(p -> p.toFile().length()) .sum(); } catch (IOException e) { - //This is only used for logging/metrics. Don't crash the process over it. + // This is only used for logging/metrics. Don't crash the process over it. LOG.debug("Failed to get size of directory {}", dir); return 0; } @@ -171,7 +179,8 @@ public void run() { LOG.debug("log cleanup: now={} old log dirs {} dead worker dirs {}", nowSecs, oldLogDirs.stream().map(p -> p.getFileName().toString()).collect(joining(",")), - deadWorkerDirs.stream().map(p -> p.getFileName().toString()).collect(joining(","))); + deadWorkerDirs.stream().map(p -> p.getFileName().toString()) + .collect(joining(","))); for (Path dir : deadWorkerDirs) { Path path = dir.toAbsolutePath().normalize(); @@ -189,10 +198,14 @@ public void run() { } } - final List perWorkerDirCleanupMeta = perWorkerDirCleanup(maxPerWorkerLogsSizeMb * 1024 * 1024); - numFilesCleaned += perWorkerDirCleanupMeta.stream().mapToInt(meta -> meta.deletedFiles).sum(); - diskSpaceCleaned += perWorkerDirCleanupMeta.stream().mapToLong(meta -> meta.deletedSize).sum(); - final DeletionMeta globalLogCleanupMeta = globalLogCleanup(maxSumWorkerLogsSizeMb * 1024 * 1024); + final List perWorkerDirCleanupMeta = + perWorkerDirCleanup(maxPerWorkerLogsSizeMb * 1024 * 1024); + numFilesCleaned += perWorkerDirCleanupMeta.stream().mapToInt(meta -> meta.deletedFiles) + .sum(); + diskSpaceCleaned += perWorkerDirCleanupMeta.stream().mapToLong(meta -> meta.deletedSize) + .sum(); + final DeletionMeta globalLogCleanupMeta = + globalLogCleanup(maxSumWorkerLogsSizeMb * 1024 * 1024); numFilesCleaned += globalLogCleanupMeta.deletedFiles; diskSpaceCleaned += globalLogCleanupMeta.deletedSize; } catch (Exception ex) { @@ -211,7 +224,8 @@ List perWorkerDirCleanup(long size) { return workerLogs.getAllWorkerDirs().stream() .map(dir -> { try { - return directoryCleaner.deleteOldestWhileTooLarge(Collections.singletonList(dir), size, true, null); + return directoryCleaner.deleteOldestWhileTooLarge(Collections + .singletonList(dir), size, true, null); } catch (IOException e) { throw new RuntimeException(e); } @@ -277,7 +291,7 @@ Set selectDirsForCleanup(long nowMillis) { @VisibleForTesting Predicate mkFileFilterForLogCleanup(long nowMillis) { - //It seems safer not to follow symlinks, since we don't expect them here + // It seems safer not to follow symlinks, since we don't expect them here return file -> Files.isDirectory(file, LinkOption.NOFOLLOW_LINKS) && lastModifiedTimeWorkerLogdir(file) <= cleanupCutoffAgeMillis(nowMillis); } diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogFileDownloader.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogFileDownloader.java index 04e08bfc437..733687e5449 100644 --- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogFileDownloader.java +++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogFileDownloader.java @@ -20,17 +20,13 @@ import com.codahale.metrics.Histogram; import com.codahale.metrics.Meter; - import jakarta.ws.rs.core.Response; - import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; - import org.apache.commons.io.FileUtils; import org.apache.storm.metric.StormMetricsRegistry; - public class LogFileDownloader { private final Histogram fileDownloadSizeDistMb; private final Meter numFileDownloadExceptions; @@ -54,8 +50,10 @@ public LogFileDownloader(String logRoot, String daemonLogRoot, WorkerLogs worker this.daemonLogRoot = Paths.get(daemonLogRoot).toAbsolutePath().normalize(); this.workerLogs = workerLogs; this.resourceAuthorizer = resourceAuthorizer; - this.fileDownloadSizeDistMb = metricsRegistry.registerHistogram("logviewer:download-file-size-rounded-MB"); - this.numFileDownloadExceptions = metricsRegistry.registerMeter(ExceptionMeterNames.NUM_FILE_DOWNLOAD_EXCEPTIONS); + this.fileDownloadSizeDistMb = metricsRegistry + .registerHistogram("logviewer:download-file-size-rounded-MB"); + this.numFileDownloadExceptions = metricsRegistry + .registerMeter(ExceptionMeterNames.NUM_FILE_DOWNLOAD_EXCEPTIONS); } /** @@ -67,16 +65,18 @@ public LogFileDownloader(String logRoot, String daemonLogRoot, WorkerLogs worker * @param isDaemon true if the file is a daemon log, false if the file is an worker log * @return a Response which lets browsers download that file. */ - public Response downloadFile(String host, String fileName, String user, boolean isDaemon) throws IOException { + public Response downloadFile(String host, String fileName, String user, + boolean isDaemon) throws IOException { Path rootDir = isDaemon ? daemonLogRoot : logRoot; Path rawFile = rootDir.resolve(fileName); Path file = rawFile.toAbsolutePath().normalize(); - if (!file.startsWith(rootDir) || !rawFile.normalize().toString().equals(rawFile.toString())) { - //Ensure filename doesn't contain ../ parts + if (!file.startsWith(rootDir) || !rawFile.normalize().toString().equals(rawFile + .toString())) { + // Ensure filename doesn't contain ../ parts return LogviewerResponseBuilder.buildResponsePageNotFound(); } if (isDaemon && Paths.get(fileName).getNameCount() != 1) { - //Prevent daemon log reads from pathing into worker logs + // Prevent daemon log reads from pathing into worker logs return LogviewerResponseBuilder.buildResponsePageNotFound(); } @@ -84,22 +84,25 @@ public Response downloadFile(String host, String fileName, String user, boolean if (isDaemon ? resourceAuthorizer.isUserAllowedToAccessDaemonFile(user) : resourceAuthorizer.isUserAllowedToAccessFile(user, fileName)) { if (!isDaemon) { - //Only widen the permission of a worker log once the request is known to be served + // Only widen the permission of a worker log once the request is known to be + // served workerLogs.setLogFilePermission(fileName); } - fileDownloadSizeDistMb.update(Math.round((double) file.toFile().length() / FileUtils.ONE_MB)); + fileDownloadSizeDistMb.update(Math.round((double) file.toFile() + .length() / FileUtils.ONE_MB)); String downloadedFileName; Path pathRelativeToRootDir = rootDir.relativize(file); if (isDaemon || pathRelativeToRootDir.getNameCount() != 3) { downloadedFileName = host + "-" + rawFile.getFileName(); } else { - //host-topoId-port-fileName + // host-topoId-port-fileName downloadedFileName = host + "-" + pathRelativeToRootDir.getName(0) + "-" + pathRelativeToRootDir.getName(1) + "-" + pathRelativeToRootDir.getName(2); } - return LogviewerResponseBuilder.buildDownloadFile(downloadedFileName, file.toFile(), numFileDownloadExceptions); + return LogviewerResponseBuilder.buildDownloadFile(downloadedFileName, file.toFile(), + numFileDownloadExceptions); } else { return LogviewerResponseBuilder.buildResponseUnauthorizedUser(user); } diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogviewerResponseBuilder.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogviewerResponseBuilder.java index d1ddabd8e79..e4d2a656410 100644 --- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogviewerResponseBuilder.java +++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/LogviewerResponseBuilder.java @@ -24,18 +24,15 @@ import com.codahale.metrics.Meter; import com.google.common.io.ByteStreams; - import jakarta.ws.rs.core.MediaType; import jakarta.ws.rs.core.Response; import jakarta.ws.rs.core.StreamingOutput; - import java.io.BufferedOutputStream; import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.nio.file.Files; - import org.apache.storm.daemon.common.JsonResponseBuilder; import org.apache.storm.daemon.ui.UIHelpers; @@ -82,7 +79,8 @@ public static Response buildDownloadFile(String contentDispositionName, return Response.status(jakarta.ws.rs.core.Response.Status.OK) .entity(wrapWithStreamingOutput(is)) .type(MediaType.APPLICATION_OCTET_STREAM_TYPE) - .header("Content-Disposition", "attachment; filename=\"" + contentDispositionName + "\"") + .header("Content-Disposition", "attachment; filename=\"" + + contentDispositionName + "\"") .build(); } catch (IOException e) { numFileDownloadExceptions.mark(); diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/ResourceAuthorizer.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/ResourceAuthorizer.java index 9f321fc1924..725071ce703 100644 --- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/ResourceAuthorizer.java +++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/ResourceAuthorizer.java @@ -20,7 +20,6 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Sets; - import java.io.File; import java.io.IOException; import java.nio.file.FileSystems; @@ -29,7 +28,6 @@ import java.util.List; import java.util.Map; import java.util.Set; - import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.Validate; import org.apache.storm.Config; @@ -58,14 +56,17 @@ public class ResourceAuthorizer { */ public ResourceAuthorizer(Map stormConf) { this.stormConf = stormConf; - this.groupMappingServiceProvider = ClientAuthUtils.getGroupMappingServiceProviderPlugin(stormConf); + this.groupMappingServiceProvider = ClientAuthUtils + .getGroupMappingServiceProviderPlugin(stormConf); this.principalToLocal = ClientAuthUtils.getPrincipalToLocalPlugin(stormConf); } /** - * Checks whether user is allowed to access a Logviewer file via UI. Always true when the Logviewer filter is not configured. + * Checks whether user is allowed to access a Logviewer file via UI. Always true when the + * Logviewer filter is not configured. * - * @param fileName file name to access. The file name must not contain upward path traversal sequences (e.g. "../"). + * @param fileName file name to access. The file name must not contain upward path traversal + * sequences (e.g. "../"). * @param user username */ public boolean isUserAllowedToAccessFile(String user, String fileName) { @@ -73,7 +74,8 @@ public boolean isUserAllowedToAccessFile(String user, String fileName) { } /** - * Checks whether user is allowed to access a daemon log file via UI. Daemon logs have no owning topology, so only the + * Checks whether user is allowed to access a daemon log file via UI. Daemon logs have no owning + * topology, so only the * cluster level lists are consulted. Always true when the Logviewer filter is not configured. * * @param user username @@ -111,7 +113,8 @@ public boolean isAuthorizedDaemonLogUser(String user) { * Checks whether user is authorized to access file. Checks regardless of UI filter. * * @param user username - * @param fileName file name to access. The file name must not contain upward path traversal sequences (e.g. "../"). + * @param fileName file name to access. The file name must not contain upward path traversal + * sequences (e.g. "../"). */ public boolean isAuthorizedLogUser(String user, String fileName) { Validate.isTrue(!fileName.contains(".." + FileSystems.getDefault().getSeparator())); @@ -149,7 +152,8 @@ public boolean isAuthorizedLogUser(String user, String fileName) { */ public LogUserGroupWhitelist getLogUserGroupWhitelist(String fileName) { File wlFile = ServerConfigUtils.getLogMetaDataFile(fileName); - Map map = (Map) Utils.readYamlFile(wlFile.getAbsolutePath()); + Map map = (Map) Utils.readYamlFile(wlFile + .getAbsolutePath()); if (map == null) { return null; @@ -177,8 +181,10 @@ Set getUserGroups(String user) { } private boolean isLogviewerFilterConfigured() { - return StringUtils.isNotBlank(ObjectReader.getString(stormConf.get(DaemonConfig.LOGVIEWER_FILTER), null)) - || StringUtils.isNotBlank(ObjectReader.getString(stormConf.get(DaemonConfig.UI_FILTER), null)); + return StringUtils.isNotBlank(ObjectReader.getString(stormConf + .get(DaemonConfig.LOGVIEWER_FILTER), null)) + || StringUtils.isNotBlank(ObjectReader.getString(stormConf + .get(DaemonConfig.UI_FILTER), null)); } public static class LogUserGroupWhitelist { diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/WorkerLogs.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/WorkerLogs.java index 7a9af36478d..69666050a60 100644 --- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/WorkerLogs.java +++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/WorkerLogs.java @@ -24,7 +24,6 @@ import com.codahale.metrics.Meter; import com.google.common.collect.Lists; - import java.io.File; import java.io.IOException; import java.nio.file.Files; @@ -40,7 +39,6 @@ import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; - import org.apache.storm.Config; import org.apache.storm.daemon.supervisor.ClientSupervisorUtils; import org.apache.storm.daemon.supervisor.SupervisorUtils; @@ -77,10 +75,12 @@ public class WorkerLogs { * @param logRootDir the log root directory * @param metricsRegistry The logviewer metrics registry */ - public WorkerLogs(Map stormConf, Path logRootDir, StormMetricsRegistry metricsRegistry) { + public WorkerLogs(Map stormConf, Path logRootDir, + StormMetricsRegistry metricsRegistry) { this.stormConf = stormConf; this.logRootDir = logRootDir.toAbsolutePath().normalize(); - this.numSetPermissionsExceptions = metricsRegistry.registerMeter(ExceptionMeterNames.NUM_SET_PERMISSION_EXCEPTIONS); + this.numSetPermissionsExceptions = metricsRegistry + .registerMeter(ExceptionMeterNames.NUM_SET_PERMISSION_EXCEPTIONS); this.directoryCleaner = new DirectoryCleaner(metricsRegistry); this.mapTopologyIdToHeartbeatTimeout = new LruMap<>(200); } @@ -95,18 +95,23 @@ public void setLogFilePermission(String fileName) throws IOException { if (!absFile.startsWith(logRootDir)) { return; } - boolean runAsUser = ObjectReader.getBoolean(stormConf.get(SUPERVISOR_RUN_WORKER_AS_USER), false); + boolean runAsUser = ObjectReader.getBoolean(stormConf.get(SUPERVISOR_RUN_WORKER_AS_USER), + false); Path parent = logRootDir.resolve(fileName).getParent(); - Optional mdFile = (parent == null) ? Optional.empty() : getMetadataFileForWorkerLogDir(parent); + Optional mdFile = (parent == null) ? Optional + .empty() : getMetadataFileForWorkerLogDir(parent); Optional topoOwner = mdFile.isPresent() - ? Optional.of(getTopologyOwnerFromMetadataFile(mdFile.get().toAbsolutePath().normalize())) + ? Optional.of(getTopologyOwnerFromMetadataFile(mdFile.get().toAbsolutePath() + .normalize())) : Optional.empty(); - if (runAsUser && topoOwner.isPresent() && absFile.toFile().exists() && !Files.isReadable(absFile)) { + if (runAsUser && topoOwner.isPresent() && absFile.toFile().exists() && !Files + .isReadable(absFile)) { LOG.debug("Setting permissions on file {} with topo-owner {}", fileName, topoOwner); try { ClientSupervisorUtils.processLauncherAndWait(stormConf, topoOwner.get(), - Lists.newArrayList("blob", absFile.toAbsolutePath().normalize().toString()), null, + Lists.newArrayList("blob", absFile.toAbsolutePath().normalize() + .toString()), null, "setup group read permissions for file: " + fileName); } catch (IOException e) { numSetPermissionsExceptions.mark(); @@ -143,7 +148,7 @@ public Set getAllWorkerDirs() { } catch (IOException e) { throw new RuntimeException(e); } - }) //Worker dirs + }) // Worker dirs .filter(Files::isDirectory) .collect(Collectors.toCollection(TreeSet::new)); } catch (IOException e) { @@ -162,6 +167,7 @@ public SortedSet getAliveWorkerDirs() throws IOException { /** * Return a metadata file (worker.yaml) for given worker log directory. + * * @param logDir worker log directory */ public Optional getMetadataFileForWorkerLogDir(Path logDir) throws IOException { @@ -169,7 +175,8 @@ public Optional getMetadataFileForWorkerLogDir(Path logDir) throws IOExcep if (metaFile.toFile().exists()) { return Optional.of(metaFile); } else { - LOG.warn("Could not find {} to clean up for {}", metaFile.toAbsolutePath().normalize(), logDir); + LOG.warn("Could not find {} to clean up for {}", metaFile.toAbsolutePath().normalize(), + logDir); return Optional.empty(); } } @@ -225,14 +232,16 @@ private int getTopologyTimeout(LSWorkerHeartbeat hb) { } private int getWorkerLogTimeout(Map conf, String topologyId, int port) { - int defaultWorkerLogTimeout = ObjectReader.getInt(conf.get(Config.SUPERVISOR_WORKER_TIMEOUT_SECS)); + int defaultWorkerLogTimeout = ObjectReader.getInt(conf + .get(Config.SUPERVISOR_WORKER_TIMEOUT_SECS)); File file = ServerConfigUtils.getLogMetaDataFile(conf, topologyId, port); Map map = (Map) Utils.readYamlFile(file.getAbsolutePath()); if (map == null) { return defaultWorkerLogTimeout; } - return (Integer) map.getOrDefault(Config.TOPOLOGY_WORKER_TIMEOUT_SECS, defaultWorkerLogTimeout); + return (Integer) map.getOrDefault(Config.TOPOLOGY_WORKER_TIMEOUT_SECS, + defaultWorkerLogTimeout); } /** @@ -250,7 +259,8 @@ public SortedSet getLogDirs(Set logDirs, Predicate predicate try { Optional metaFile = getMetadataFileForWorkerLogDir(logDir); if (metaFile.isPresent()) { - workerId = getWorkerIdFromMetadataFile(metaFile.get().toAbsolutePath().normalize()); + workerId = getWorkerIdFromMetadataFile(metaFile.get().toAbsolutePath() + .normalize()); if (workerId == null) { workerId = ""; } @@ -266,7 +276,7 @@ public SortedSet getLogDirs(Set logDirs, Predicate predicate } /** - * Return the path of the worker log with the format of topoId/port/worker.log.* + * Return the path of the worker log with the format of topoId/port/worker.log.*. * * @param file worker log */ diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/webapp/LogviewerApplication.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/webapp/LogviewerApplication.java index 35c5523d1ec..fbfb0fef70f 100644 --- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/webapp/LogviewerApplication.java +++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/webapp/LogviewerApplication.java @@ -22,13 +22,11 @@ import jakarta.ws.rs.ApplicationPath; import jakarta.ws.rs.core.Application; - import java.io.File; import java.nio.file.Paths; import java.util.HashSet; import java.util.Map; import java.util.Set; - import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.core.Appender; import org.apache.logging.log4j.core.LoggerContext; @@ -57,21 +55,28 @@ public class LogviewerApplication extends Application { */ public LogviewerApplication() { String logRoot = ConfigUtils.workerArtifactsRoot(stormConf); - String daemonLogRoot = logRootDir(ObjectReader.getString(stormConf.get(LOGVIEWER_APPENDER_NAME))); + String daemonLogRoot = logRootDir(ObjectReader.getString(stormConf + .get(LOGVIEWER_APPENDER_NAME))); ResourceAuthorizer resourceAuthorizer = new ResourceAuthorizer(stormConf); WorkerLogs workerLogs = new WorkerLogs(stormConf, Paths.get(logRoot), metricsRegistry); - LogviewerLogPageHandler logviewer = new LogviewerLogPageHandler(logRoot, daemonLogRoot, workerLogs, resourceAuthorizer, + LogviewerLogPageHandler logviewer = new LogviewerLogPageHandler(logRoot, daemonLogRoot, + workerLogs, resourceAuthorizer, metricsRegistry); - LogviewerProfileHandler profileHandler = new LogviewerProfileHandler(logRoot, resourceAuthorizer, metricsRegistry); - LogviewerLogDownloadHandler logDownloadHandler = new LogviewerLogDownloadHandler(logRoot, daemonLogRoot, + LogviewerProfileHandler profileHandler = new LogviewerProfileHandler(logRoot, + resourceAuthorizer, metricsRegistry); + LogviewerLogDownloadHandler logDownloadHandler = new LogviewerLogDownloadHandler(logRoot, + daemonLogRoot, workerLogs, resourceAuthorizer, metricsRegistry); - LogviewerLogSearchHandler logSearchHandler = new LogviewerLogSearchHandler(stormConf, Paths.get(logRoot), Paths.get(daemonLogRoot), + LogviewerLogSearchHandler logSearchHandler = new LogviewerLogSearchHandler(stormConf, Paths + .get(logRoot), Paths.get(daemonLogRoot), resourceAuthorizer, metricsRegistry); - IHttpCredentialsPlugin httpCredsHandler = ServerAuthUtils.getUiHttpCredentialsPlugin(stormConf); + IHttpCredentialsPlugin httpCredsHandler = ServerAuthUtils + .getUiHttpCredentialsPlugin(stormConf); - singletons.add(new LogviewerResource(logviewer, profileHandler, logDownloadHandler, logSearchHandler, + singletons.add(new LogviewerResource(logviewer, profileHandler, logDownloadHandler, + logSearchHandler, httpCredsHandler, metricsRegistry)); singletons.add(new AuthorizationExceptionMapper()); } @@ -97,11 +102,14 @@ public static void setup(Map stormConf, StormMetricsRegistry met * Note that if anything goes wrong, this will throw an Error and exit. */ private String logRootDir(String appenderName) { - Appender appender = ((LoggerContext) LogManager.getContext()).getConfiguration().getAppender(appenderName); - if (appenderName != null && appender != null && RollingFileAppender.class.isInstance(appender)) { + Appender appender = ((LoggerContext) LogManager.getContext()).getConfiguration() + .getAppender(appenderName); + if (appenderName != null && appender != null && RollingFileAppender.class + .isInstance(appender)) { return new File(((RollingFileAppender) appender).getFileName()).getParent(); } else { - throw new RuntimeException("Log viewer could not find configured appender, or the appender is not a FileAppender. " + throw new RuntimeException("Log viewer could not find configured appender, or the " + + "appender is not a FileAppender. " + "Please check that the appender name configured in storm and log4j agree."); } } diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/webapp/LogviewerResource.java b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/webapp/LogviewerResource.java index 74564436c9d..7fee970039b 100644 --- a/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/webapp/LogviewerResource.java +++ b/storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/webapp/LogviewerResource.java @@ -20,17 +20,14 @@ import com.codahale.metrics.Meter; import com.codahale.metrics.Timer; - import jakarta.servlet.http.HttpServletRequest; import jakarta.ws.rs.GET; import jakarta.ws.rs.Path; import jakarta.ws.rs.PathParam; import jakarta.ws.rs.core.Context; import jakarta.ws.rs.core.Response; - import java.io.IOException; import java.util.Map; - import org.apache.commons.lang3.BooleanUtils; import org.apache.commons.lang3.StringUtils; import org.apache.storm.daemon.common.JsonResponseBuilder; @@ -90,30 +87,46 @@ public class LogviewerResource { * @param httpCredsHandler {@link IHttpCredentialsPlugin} * @param metricsRegistry The metrics registry */ - public LogviewerResource(LogviewerLogPageHandler logviewerParam, LogviewerProfileHandler profileHandler, + public LogviewerResource(LogviewerLogPageHandler logviewerParam, + LogviewerProfileHandler profileHandler, LogviewerLogDownloadHandler logDownloadHandler, LogviewerLogSearchHandler logSearchHandler, IHttpCredentialsPlugin httpCredsHandler, StormMetricsRegistry metricsRegistry) { - this.meterLogPageHttpRequests = metricsRegistry.registerMeter("logviewer:num-log-page-http-requests"); + this.meterLogPageHttpRequests = metricsRegistry + .registerMeter("logviewer:num-log-page-http-requests"); this.meterDaemonLogPageHttpRequests = metricsRegistry.registerMeter( "logviewer:num-daemonlog-page-http-requests"); this.meterDownloadLogFileHttpRequests = metricsRegistry.registerMeter( "logviewer:num-download-log-file-http-requests"); this.meterDownloadLogDaemonFileHttpRequests = metricsRegistry.registerMeter( "logviewer:num-download-log-daemon-file-http-requests"); - this.meterListLogsHttpRequests = metricsRegistry.registerMeter("logviewer:num-list-logs-http-requests"); - this.numSearchLogRequests = metricsRegistry.registerMeter("logviewer:num-search-logs-requests"); - this.numDeepSearchArchived = metricsRegistry.registerMeter("logviewer:num-deep-search-requests-with-archived"); - this.numDeepSearchNonArchived = metricsRegistry.registerMeter("logviewer:num-deep-search-requests-without-archived"); - this.numReadLogExceptions = metricsRegistry.registerMeter(ExceptionMeterNames.NUM_READ_LOG_EXCEPTIONS); - this.numReadDaemonLogExceptions = metricsRegistry.registerMeter(ExceptionMeterNames.NUM_READ_DAEMON_LOG_EXCEPTIONS); - this.numListLogExceptions = metricsRegistry.registerMeter(ExceptionMeterNames.NUM_LIST_LOG_EXCEPTIONS); - this.numListDumpExceptions = metricsRegistry.registerMeter(ExceptionMeterNames.NUM_LIST_DUMP_EXCEPTIONS); - this.numDownloadDumpExceptions = metricsRegistry.registerMeter(ExceptionMeterNames.NUM_DOWNLOAD_DUMP_EXCEPTIONS); - this.numDownloadLogExceptions = metricsRegistry.registerMeter(ExceptionMeterNames.NUM_DOWNLOAD_LOG_EXCEPTIONS); - this.numDownloadDaemonLogExceptions = metricsRegistry.registerMeter(ExceptionMeterNames.NUM_DOWNLOAD_DAEMON_LOG_EXCEPTIONS); - this.numSearchExceptions = metricsRegistry.registerMeter(ExceptionMeterNames.NUM_SEARCH_EXCEPTIONS); - this.searchLogRequestDuration = metricsRegistry.registerTimer("logviewer:search-requests-duration-ms"); - this.deepSearchRequestDuration = metricsRegistry.registerTimer("logviewer:deep-search-request-duration-ms"); + this.meterListLogsHttpRequests = metricsRegistry + .registerMeter("logviewer:num-list-logs-http-requests"); + this.numSearchLogRequests = metricsRegistry + .registerMeter("logviewer:num-search-logs-requests"); + this.numDeepSearchArchived = metricsRegistry + .registerMeter("logviewer:num-deep-search-requests-with-archived"); + this.numDeepSearchNonArchived = metricsRegistry + .registerMeter("logviewer:num-deep-search-requests-without-archived"); + this.numReadLogExceptions = metricsRegistry + .registerMeter(ExceptionMeterNames.NUM_READ_LOG_EXCEPTIONS); + this.numReadDaemonLogExceptions = metricsRegistry + .registerMeter(ExceptionMeterNames.NUM_READ_DAEMON_LOG_EXCEPTIONS); + this.numListLogExceptions = metricsRegistry + .registerMeter(ExceptionMeterNames.NUM_LIST_LOG_EXCEPTIONS); + this.numListDumpExceptions = metricsRegistry + .registerMeter(ExceptionMeterNames.NUM_LIST_DUMP_EXCEPTIONS); + this.numDownloadDumpExceptions = metricsRegistry + .registerMeter(ExceptionMeterNames.NUM_DOWNLOAD_DUMP_EXCEPTIONS); + this.numDownloadLogExceptions = metricsRegistry + .registerMeter(ExceptionMeterNames.NUM_DOWNLOAD_LOG_EXCEPTIONS); + this.numDownloadDaemonLogExceptions = metricsRegistry + .registerMeter(ExceptionMeterNames.NUM_DOWNLOAD_DAEMON_LOG_EXCEPTIONS); + this.numSearchExceptions = metricsRegistry + .registerMeter(ExceptionMeterNames.NUM_SEARCH_EXCEPTIONS); + this.searchLogRequestDuration = metricsRegistry + .registerTimer("logviewer:search-requests-duration-ms"); + this.deepSearchRequestDuration = metricsRegistry + .registerTimer("logviewer:deep-search-request-duration-ms"); this.logviewer = logviewerParam; this.profileHandler = profileHandler; this.logDownloadHandler = logDownloadHandler; @@ -131,8 +144,10 @@ public Response log(@Context HttpServletRequest request) throws IOException { try { String user = httpCredsHandler.getUserName(request); - Integer start = request.getParameter("start") != null ? parseIntegerFromMap(request.getParameterMap(), "start") : null; - Integer length = request.getParameter("length") != null ? parseIntegerFromMap(request.getParameterMap(), "length") : null; + Integer start = request.getParameter("start") != null ? parseIntegerFromMap(request + .getParameterMap(), "start") : null; + Integer length = request.getParameter("length") != null ? parseIntegerFromMap(request + .getParameterMap(), "length") : null; String decodedFileName = Utils.urlDecodeUtf8(request.getParameter("file")); String grep = request.getParameter("grep"); return logviewer.logPage(decodedFileName, start, length, grep, user); @@ -155,8 +170,10 @@ public Response daemonLog(@Context HttpServletRequest request) throws IOExceptio try { String user = httpCredsHandler.getUserName(request); - Integer start = request.getParameter("start") != null ? parseIntegerFromMap(request.getParameterMap(), "start") : null; - Integer length = request.getParameter("length") != null ? parseIntegerFromMap(request.getParameterMap(), "length") : null; + Integer start = request.getParameter("start") != null ? parseIntegerFromMap(request + .getParameterMap(), "start") : null; + Integer length = request.getParameter("length") != null ? parseIntegerFromMap(request + .getParameterMap(), "length") : null; String decodedFileName = Utils.urlDecodeUtf8(request.getParameter("file")); String grep = request.getParameter("grep"); return logviewer.daemonLogPage(decodedFileName, start, length, grep, user); @@ -181,7 +198,8 @@ public Response searchLogs(@Context HttpServletRequest request) throws IOExcepti String callback = request.getParameter("callbackParameterName"); String origin = request.getHeader("Origin"); - return logviewer.listLogFiles(user, portStr != null ? Integer.parseInt(portStr) : null, topologyId, callback, origin); + return logviewer.listLogFiles(user, portStr != null ? Integer.parseInt(portStr) : null, + topologyId, callback, origin); } /** @@ -199,7 +217,8 @@ public Response listLogs(@Context HttpServletRequest request) throws IOException String origin = request.getHeader("Origin"); try { - return logviewer.listLogFiles(user, portStr != null ? Integer.parseInt(portStr) : null, topologyId, callback, origin); + return logviewer.listLogFiles(user, portStr != null ? Integer.parseInt(portStr) : null, + topologyId, callback, origin); } catch (IOException e) { numListLogExceptions.mark(); throw e; @@ -211,7 +230,8 @@ public Response listLogs(@Context HttpServletRequest request) throws IOException */ @GET @Path("/dumps/{topo-id}/{host-port}") - public Response listDumpFiles(@PathParam("topo-id") String topologyId, @PathParam("host-port") String hostPort, + public Response listDumpFiles(@PathParam("topo-id") String topologyId, + @PathParam("host-port") String hostPort, @Context HttpServletRequest request) throws IOException { String user = httpCredsHandler.getUserName(request); try { @@ -227,7 +247,8 @@ public Response listDumpFiles(@PathParam("topo-id") String topologyId, @PathPara */ @GET @Path("/dumps/{topo-id}/{host-port}/{filename}") - public Response downloadDumpFile(@PathParam("topo-id") String topologyId, @PathParam("host-port") String hostPort, + public Response downloadDumpFile(@PathParam("topo-id") String topologyId, + @PathParam("host-port") String hostPort, @PathParam("filename") String fileName, @Context HttpServletRequest request) throws IOException { String user = httpCredsHandler.getUserName(request); @@ -301,7 +322,8 @@ public Response search(@Context HttpServletRequest request) throws IOException { } catch (InvalidRequestException e) { LOG.error(e.getMessage(), e); int statusCode = 400; - return new JsonResponseBuilder().setData(UIHelpers.exceptionToJson(e, statusCode)).setCallback(callback) + return new JsonResponseBuilder().setData(UIHelpers.exceptionToJson(e, statusCode)) + .setCallback(callback) .setStatus(statusCode).build(); } catch (IOException e) { numSearchExceptions.mark(); @@ -333,16 +355,19 @@ public Response deepSearch(@PathParam("topoId") String topologyId, numDeepSearchNonArchived.mark(); } try (Timer.Context t = deepSearchRequestDuration.time()) { - return logSearchHandler.deepSearchLogsForTopology(topologyId, user, searchString, numMatchesStr, portStr, startFileOffset, + return logSearchHandler.deepSearchLogsForTopology(topologyId, user, searchString, + numMatchesStr, portStr, startFileOffset, startByteOffset, alsoSearchArchived, callback, origin); } } - private int parseIntegerFromMap(Map map, String parameterKey) throws InvalidRequestException { + private int parseIntegerFromMap(Map map, + String parameterKey) throws InvalidRequestException { try { return Integer.parseInt(map.get(parameterKey)[0]); } catch (NumberFormatException ex) { - throw new InvalidRequestException("Could not make an integer out of the query parameter '" + throw new InvalidRequestException("Could not make an integer out of the query " + + "parameter '" + parameterKey + "'", ex); } } diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/ui/FilterConfiguration.java b/storm-webapp/src/main/java/org/apache/storm/daemon/ui/FilterConfiguration.java index be9ecd5f16b..5309672dc97 100644 --- a/storm-webapp/src/main/java/org/apache/storm/daemon/ui/FilterConfiguration.java +++ b/storm-webapp/src/main/java/org/apache/storm/daemon/ui/FilterConfiguration.java @@ -1,12 +1,17 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at *

      * http://www.apache.org/licenses/LICENSE-2.0 *

      - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ @@ -21,6 +26,7 @@ public class FilterConfiguration { /** * FilterConfiguration. + * * @param filterClass filterClass * @param filterParams filterParams */ @@ -32,11 +38,13 @@ public FilterConfiguration(String filterClass, Map filterParams) /** * FilterConfiguration. + * * @param filterClass filterClass * @param filterName filterName * @param filterParams filterParams */ - public FilterConfiguration(String filterClass, String filterName, Map filterParams) { + public FilterConfiguration(String filterClass, String filterName, Map filterParams) { this.filterClass = filterClass; this.filterName = filterName; this.filterParams = filterParams; diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/ui/IConfigurator.java b/storm-webapp/src/main/java/org/apache/storm/daemon/ui/IConfigurator.java index 93a557b8aa1..5fda3c5bb79 100644 --- a/storm-webapp/src/main/java/org/apache/storm/daemon/ui/IConfigurator.java +++ b/storm-webapp/src/main/java/org/apache/storm/daemon/ui/IConfigurator.java @@ -1,12 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. The ASF licenses this file to + * you under the Apache License, Version + * 2.0 (the "License"); you may not use this file except in compliance with the License. You may + * obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

      http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + *

      Unless required by applicable law or agreed to in writing, software distributed under the + * License + * is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions * and limitations under the License. */ diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/ui/TestingFilter.java b/storm-webapp/src/main/java/org/apache/storm/daemon/ui/TestingFilter.java index eb18f92873c..f4b3a4dc9f9 100644 --- a/storm-webapp/src/main/java/org/apache/storm/daemon/ui/TestingFilter.java +++ b/storm-webapp/src/main/java/org/apache/storm/daemon/ui/TestingFilter.java @@ -33,7 +33,8 @@ /** * A simple UI filter that should only be used for testing purposes. To set a given user name, - * set an init parameter of USER_NAME to be the name of the user you want the UI to always impersonate. + * set an init parameter of USER_NAME to be the name of the user you want the UI to always + * impersonate. */ public class TestingFilter implements Filter { private static final Logger LOG = LoggerFactory.getLogger(TestingFilter.class); @@ -50,25 +51,27 @@ public void init(FilterConfig filterConfig) throws ServletException { } @Override - public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain) + public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, + FilterChain filterChain) throws IOException, ServletException { - ServletRequest filteredRequest = new HttpServletRequestWrapper((HttpServletRequest) servletRequest) { - @Override + ServletRequest filteredRequest = + new HttpServletRequestWrapper((HttpServletRequest) servletRequest) { + @Override public String getRemoteUser() { - return userName; - } + return userName; + } - @Override + @Override public Principal getUserPrincipal() { - return () -> userName; - } - }; + return () -> userName; + } + }; LOG.debug("Changing user name to {}", userName); filterChain.doFilter(filteredRequest, servletResponse); } @Override public void destroy() { - //NOOP + // NOOP } } diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/ui/UIHelpers.java b/storm-webapp/src/main/java/org/apache/storm/daemon/ui/UIHelpers.java index aaa323fd850..f0ba0fbdfcd 100644 --- a/storm-webapp/src/main/java/org/apache/storm/daemon/ui/UIHelpers.java +++ b/storm-webapp/src/main/java/org/apache/storm/daemon/ui/UIHelpers.java @@ -111,7 +111,6 @@ import org.eclipse.jetty.server.ServerConnector; import org.eclipse.jetty.server.SslConnectionFactory; import org.eclipse.jetty.util.ssl.SslContextFactory; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -135,6 +134,7 @@ public class UIHelpers { /** * Prettify uptime string. + * * @param val val. * @param dividers dividers. * @return prettified uptime string. @@ -159,6 +159,7 @@ public static String prettyUptimeStr(String val, Object[][] dividers) { /** * Prettify uptime string. + * * @param sec uptime in seconds. * @return prettified uptime string. */ @@ -168,6 +169,7 @@ public static String prettyUptimeSec(String sec) { /** * prettyUptimeSec. + * * @param secs secs * @return prettyUptimeSec */ @@ -177,6 +179,7 @@ public static String prettyUptimeSec(int secs) { /** * prettyUptimeMs. + * * @param ms ms * @return prettyUptimeMs */ @@ -186,6 +189,7 @@ public static String prettyUptimeMs(String ms) { /** * prettyUptimeMs. + * * @param ms ms * @return prettyUptimeMs */ @@ -194,7 +198,8 @@ public static String prettyUptimeMs(int ms) { } /** - * url formatter for log links. + * Url formatter for log links. + * * @param fmt string format * @param args hostname and other arguments. * @return string formatter @@ -209,6 +214,7 @@ public static String urlFormat(String fmt, Object... args) { /** * Prettified executor info. + * * @param e from Nimbus call * @return prettified executor info string */ @@ -218,6 +224,7 @@ public static String prettyExecutorInfo(ExecutorInfo e) { /** * Unauthorized user json. + * * @param user User id. * @return Unauthorized user json. */ @@ -280,6 +287,7 @@ public static void configSsl(Server server, Integer port, String ksPath, /** * configSsl. + * * @param server server * @param port port * @param ksPath ksPath @@ -314,6 +322,7 @@ public static void configSsl(Server server, Integer port, String ksPath, /** * corsFilterHandle. + * * @return corsFilterHandle */ public static FilterHolder corsFilterHandle() { @@ -330,6 +339,7 @@ public static FilterHolder corsFilterHandle() { /** * mkAccessLoggingFilterHandle. + * * @return mkAccessLoggingFilterHandle */ public static FilterHolder mkAccessLoggingFilterHandle() { @@ -338,12 +348,14 @@ public static FilterHolder mkAccessLoggingFilterHandle() { } - public static void configFilter(Server server, Servlet servlet, List filtersConfs) { + public static void configFilter(Server server, Servlet servlet, + List filtersConfs) { configFilter(server, servlet, filtersConfs, null); } /** * Config filter. + * * @param server Server * @param servlet Servlet * @param filtersConfs FiltersConfs @@ -367,6 +379,7 @@ public static void configFilter(Server server, Servlet servlet, /** * Config filters. + * * @param context Servlet context * @param filtersConfs filter confs */ @@ -399,7 +412,8 @@ public static void configFilters(ServletContextHandler context, /** * Construct a Jetty Server instance. */ - public static Server jettyCreateServer(Integer port, String host, Integer httpsPort, Boolean disableHttpBinding) { + public static Server jettyCreateServer(Integer port, String host, Integer httpsPort, + Boolean disableHttpBinding) { return jettyCreateServer(port, host, httpsPort, null, disableHttpBinding); } @@ -410,7 +424,8 @@ public static Server jettyCreateServer(Integer port, String host, Integer httpsPort, Integer headerBufferSize, Boolean disableHttpBinding) { Server server = new Server(); - if (httpsPort == null || httpsPort <= 0 || disableHttpBinding == null || disableHttpBinding == false) { + if (httpsPort == null || httpsPort <= 0 || disableHttpBinding == null + || disableHttpBinding == false) { HttpConfiguration httpConfig = new HttpConfiguration(); httpConfig.setSendDateHeader(true); if (null != headerBufferSize) { @@ -455,7 +470,8 @@ public static void stormRunJetty(Integer port, Integer headerBufferSize, * It is read once, like the rest of the daemon configuration, so a change needs a restart. */ private static boolean jsonpEnabled = - ObjectReader.getBoolean(ConfigUtils.readStormConfig().get(DaemonConfig.UI_ENABLE_JSONP), false); + ObjectReader.getBoolean(ConfigUtils.readStormConfig().get(DaemonConfig.UI_ENABLE_JSONP), + false); @VisibleForTesting static void setJsonpEnabled(boolean enabled) { @@ -480,6 +496,7 @@ private static String sanitizeJsonpCallback(String callback) { /** * wrapJsonInCallback. + * * @param callback callbackParameterName (must already be validated) * @param response response * @return wrapJsonInCallback @@ -490,6 +507,7 @@ public static String wrapJsonInCallback(String callback, String response) { /** * getJsonResponseHeaders. + * * @param callback callbackParameterName * @param headers headers * @return getJsonResponseHeaders @@ -519,11 +537,13 @@ public static Map getJsonResponseHeaders(String callback, Map headers) { public static String getJsonResponseBody(Object data, String callback, boolean needSerialize) { String safeCallback = sanitizeJsonpCallback(callback); String serializedData = needSerialize ? JSONValue.toJSONString(data) : (String) data; - return safeCallback != null ? wrapJsonInCallback(safeCallback, serializedData) : serializedData; + return safeCallback != null ? wrapJsonInCallback(safeCallback, + serializedData) : serializedData; } /** * Converts exception into json map. + * * @param ex Exception to be converted. * @param statusCode Status code to be returned. * @return Map to be converted into json. @@ -545,12 +565,14 @@ public static Response makeStandardResponse(Object data, String callback) { return makeStandardResponse(data, callback, true, Response.Status.OK); } - public static Response makeStandardResponse(Object data, String callback, Response.Status status) { + public static Response makeStandardResponse(Object data, String callback, + Response.Status status) { return makeStandardResponse(data, callback, true, status); } /** * makeStandardResponse. + * * @param data data * @param callback callbackParameterName * @param needsSerialization needsSerialization @@ -567,8 +589,10 @@ public static Response makeStandardResponse( return responseBuilder.build(); } - private static final AtomicReference>> MEMORIZED_VERSIONS = new AtomicReference<>(); - private static final AtomicReference> MEMORIZED_FULL_VERSION = new AtomicReference<>(); + private static final AtomicReference>> MEMORIZED_VERSIONS = + new AtomicReference<>(); + private static final AtomicReference> MEMORIZED_FULL_VERSION = + new AtomicReference<>(); private static Map toJsonStruct(IVersionInfo info) { Map ret = new HashMap<>(); @@ -582,6 +606,7 @@ private static Map toJsonStruct(IVersionInfo info) { /** * Converts thrift call result into map fit for UI/api. + * * @param clusterSummary Obtained from Nimbus. * @param user User Making request * @param conf Storm Conf @@ -592,7 +617,7 @@ public static Map getClusterSummary(ClusterSummary clusterSummar Map result = new HashMap(); if (MEMORIZED_VERSIONS.get() == null) { - //Races are okay this is just to avoid extra work for each page load. + // Races are okay this is just to avoid extra work for each page load. NavigableMap versionsMap = Utils.getAlternativeVersionsMap(conf); List> versionList = new ArrayList<>(); for (Map.Entry entry : versionsMap.entrySet()) { @@ -664,12 +689,15 @@ public static Map getClusterSummary(ClusterSummary clusterSummar double supervisorUsedCpu = supervisorSummaries.stream().mapToDouble(SupervisorSummary::get_used_cpu).sum(); result.put("availCpu", supervisorTotalCpu - supervisorUsedCpu); - result.put("fragmentedMem", supervisorSummaries.stream().mapToDouble(SupervisorSummary::get_fragmented_mem).sum()); - result.put("fragmentedCpu", supervisorSummaries.stream().mapToDouble(SupervisorSummary::get_fragmented_cpu).sum()); + result.put("fragmentedMem", supervisorSummaries.stream() + .mapToDouble(SupervisorSummary::get_fragmented_mem).sum()); + result.put("fragmentedCpu", supervisorSummaries.stream() + .mapToDouble(SupervisorSummary::get_fragmented_cpu).sum()); result.put("schedulerDisplayResource", conf.get(DaemonConfig.SCHEDULER_DISPLAY_RESOURCE)); result.put("memAssignedPercentUtil", supervisorTotalMemory > 0 - ? StatsUtil.floatStr((supervisorUsedMemory * 100.0) / supervisorTotalMemory) : "0.0"); + ? StatsUtil + .floatStr((supervisorUsedMemory * 100.0) / supervisorTotalMemory) : "0.0"); result.put("cpuAssignedPercentUtil", supervisorTotalCpu > 0 ? StatsUtil.floatStr((supervisorUsedCpu * 100.0) / supervisorTotalCpu) : "0.0"); result.put("bugtracker-url", conf.get(DaemonConfig.UI_PROJECT_BUGTRACKER_URL)); @@ -678,8 +706,10 @@ public static Map getClusterSummary(ClusterSummary clusterSummar Map usedGenericResources = new HashMap<>(); Map totalGenericResources = new HashMap<>(); for (SupervisorSummary ss : supervisorSummaries) { - usedGenericResources = NormalizedResourceRequest.addResourceMap(usedGenericResources, ss.get_used_generic_resources()); - totalGenericResources = NormalizedResourceRequest.addResourceMap(totalGenericResources, ss.get_total_resources()); + usedGenericResources = NormalizedResourceRequest.addResourceMap(usedGenericResources, ss + .get_used_generic_resources()); + totalGenericResources = NormalizedResourceRequest.addResourceMap(totalGenericResources, + ss.get_total_resources()); } Map availGenericResources = NormalizedResourceRequest .subtractResourceMap(totalGenericResources, usedGenericResources); @@ -692,7 +722,8 @@ private static String prettifyGenericResources(Map resourceMap) if (resourceMap == null) { return null; } - TreeMap treeGenericResources = new TreeMap<>(); // use TreeMap for deterministic ordering + TreeMap treeGenericResources = + new TreeMap<>(); // use TreeMap for deterministic ordering treeGenericResources.putAll(resourceMap); NormalizedResourceRequest.removeNonGenericResources(treeGenericResources); return treeGenericResources.toString() @@ -702,6 +733,7 @@ private static String prettifyGenericResources(Map resourceMap) /** * Prettify OwnerResourceSummary. + * * @param ownerResourceSummary ownerResourceSummary * @return Map of prettified OwnerResourceSummary. */ @@ -766,6 +798,7 @@ public static Map unpackOwnerResourceSummary( /** * Get prettified ownerResourceSummaries. + * * @param ownerResourceSummaries ownerResourceSummaries from thrift call * @param conf Storm conf * @return map to be converted to json. @@ -787,6 +820,7 @@ public static Map getOwnerResourceSummaries( /** * getTopologyMap. + * * @param topologySummary topologySummary * @return getTopologyMap */ @@ -810,14 +844,16 @@ public static Map getTopologyMap(TopologySummary topologySummary topologySummary.get_requested_memoffheap() + topologySummary.get_assigned_memonheap()); result.put("requestedCpu", topologySummary.get_requested_cpu()); - result.put("requestedGenericResources", prettifyGenericResources(topologySummary.get_requested_generic_resources())); + result.put("requestedGenericResources", prettifyGenericResources(topologySummary + .get_requested_generic_resources())); result.put("assignedMemOnHeap", topologySummary.get_assigned_memonheap()); result.put("assignedMemOffHeap", topologySummary.get_assigned_memoffheap()); result.put("assignedTotalMem", topologySummary.get_assigned_memoffheap() + topologySummary.get_assigned_memonheap()); result.put("assignedCpu", topologySummary.get_assigned_cpu()); - result.put("assignedGenericResources", prettifyGenericResources(topologySummary.get_assigned_generic_resources())); + result.put("assignedGenericResources", prettifyGenericResources(topologySummary + .get_assigned_generic_resources())); result.put("topologyVersion", topologySummary.get_topology_version()); result.put("stormVersion", topologySummary.get_storm_version()); return result; @@ -825,6 +861,7 @@ public static Map getTopologyMap(TopologySummary topologySummary /** * Get a specific owner resource summary. + * * @param ownerResourceSummaries Result from thrift call. * @param client client * @param id Owner id. @@ -852,6 +889,7 @@ public static Map getOwnerResourceSummary( /** * getTopologiesMap. + * * @param id id * @param topologies topologies * @return getTopologiesMap @@ -869,6 +907,7 @@ private static List getTopologiesMap(String id, List topol /** * getLogviewerLink. + * * @param host host * @param fname fname * @param config config @@ -888,6 +927,7 @@ public static String getLogviewerLink(String host, String fname, /** * Get log link to nimbus log. + * * @param host nimbus host name * @param config storm config * @return log link. @@ -903,6 +943,7 @@ public static String getNimbusLogLink(String host, Map config) { /** * Get log link to supervisor log. + * * @param host supervisor host name * @param config storm config * @return log link. @@ -918,6 +959,7 @@ public static String getSupervisorLogLink(String host, Map confi /** * Get log link to supervisor log. + * * @param host supervisor host name * @param config storm config * @return log link. @@ -933,6 +975,7 @@ public static String getWorkerLogLink(String host, int port, /** * Get supervisor info in a map. + * * @param supervisorSummary from nimbus call. * @param config Storm config. * @return prettified supervisor info map. @@ -989,6 +1032,7 @@ public static Map getPrettifiedSupervisorMap( /** * Get topology history. + * * @param topologyHistory from Nimbus call. * @return map ready to be returned. */ @@ -1000,6 +1044,7 @@ public static Map getTopologyHistoryInfo(TopologyHistoryInfo top /** * Check if logviewer is secure. + * * @param config Storm config. * @return true if logiviwer is secure. */ @@ -1015,6 +1060,7 @@ public static boolean isSecureLogviewer(Map config) { /** * Get logviewer port depending on whether the logviewer is secure or not. + * * @param config Storm config. * @return appropriate port. */ @@ -1027,6 +1073,7 @@ public static int getLogviewerPort(Map config) { /** * getWorkerSummaries. + * * @param supervisorPageInfo supervisorPageInfo * @param config config * @return getWorkerSummaries @@ -1044,11 +1091,13 @@ public static List getWorkerSummaries(SupervisorPageInfo supervisorPageInfo /** * getWorkerSummaryMap. + * * @param workerSummary workerSummary * @param config config * @return getWorkerSummaryMap */ - private static Map getWorkerSummaryMap(WorkerSummary workerSummary, Map config) { + private static Map getWorkerSummaryMap(WorkerSummary workerSummary, Map config) { Map result = new HashMap(); result.put("supervisorId", workerSummary.get_supervisor_id()); result.put("host", workerSummary.get_host()); @@ -1070,6 +1119,7 @@ private static Map getWorkerSummaryMap(WorkerSummary workerSummary, Map getSupervisorSummary( /** * getSupervisorsMap. + * * @param supervisors supervisors * @param config config * @return getSupervisorsMap @@ -1106,6 +1157,7 @@ private static List getSupervisorsMap(List supervisors, /** * addLogviewerInfo. + * * @param config config * @param result result */ @@ -1120,6 +1172,7 @@ private static void addLogviewerInfo(Map config, Map getSupervisorPageInfo( Map result = new HashMap<>(); result.put("workers", getWorkerSummaries(supervisorPageInfo, config)); result.put("schedulerDisplayResource", config.get(DaemonConfig.SCHEDULER_DISPLAY_RESOURCE)); - List supervisorMaps = getSupervisorsMap(supervisorPageInfo.get_supervisor_summaries(), config); + List supervisorMaps = getSupervisorsMap(supervisorPageInfo.get_supervisor_summaries(), + config); result.put("supervisors", supervisorMaps); addLogviewerInfo(config, result); return result; @@ -1137,6 +1191,7 @@ public static Map getSupervisorPageInfo( /** * getAllTopologiesSummary. + * * @param topologies topologies * @param config config * @return getAllTopologiesSummary @@ -1151,6 +1206,7 @@ public static Map getAllTopologiesSummary( /** * getWindowHint. + * * @param window window * @return getWindowHint */ @@ -1163,6 +1219,7 @@ public static String getWindowHint(String window) { /** * getStatDisplayMap. + * * @param rawDisplayMap rawDisplayMap * @return getStatDisplayMap */ @@ -1177,6 +1234,7 @@ public static Map getStatDisplayMap(Map rawDispl /** * getTopologySummary. + * * @param topologyPageInfo topologyPageInfo * @param window window * @param config config @@ -1186,7 +1244,8 @@ public static Map getStatDisplayMap(Map rawDispl public static Map getTopologySummary(TopologyPageInfo topologyPageInfo, String window, Map config, String remoteUser) { Map result = new HashMap(); - Map topologyConf = (Map) JSONValue.parse(topologyPageInfo.get_topology_conf()); + Map topologyConf = (Map) JSONValue.parse(topologyPageInfo + .get_topology_conf()); int messageTimeout = (int) topologyConf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS); Map unpackedTopologyPageInfo = unpackTopologyInfo(topologyPageInfo, window, config); @@ -1205,6 +1264,7 @@ public static Map getTopologySummary(TopologyPageInfo topologyPa /** * getStatDisplayMapLong. + * * @param windowToTransferred windowToTransferred * @return getStatDisplayMapLong */ @@ -1218,6 +1278,7 @@ private static Map getStatDisplayMapLong(Map windowT /** * getCommonAggStatsMap. + * * @param commonAggregateStats commonAggregateStats * @return getCommonAggStatsMap */ @@ -1232,14 +1293,17 @@ private static Map getCommonAggStatsMap(CommonAggregateStats com if (commonAggregateStats.is_set_resources_map()) { result.put( "requestedMemOnHeap", - commonAggregateStats.get_resources_map().get(Constants.COMMON_ONHEAP_MEMORY_RESOURCE_NAME) + commonAggregateStats.get_resources_map() + .get(Constants.COMMON_ONHEAP_MEMORY_RESOURCE_NAME) ); result.put( "requestedMemOffHeap", - commonAggregateStats.get_resources_map().get(Constants.COMMON_OFFHEAP_MEMORY_RESOURCE_NAME)); + commonAggregateStats.get_resources_map() + .get(Constants.COMMON_OFFHEAP_MEMORY_RESOURCE_NAME)); result.put( "requestedCpu", - commonAggregateStats.get_resources_map().get(Constants.COMMON_CPU_RESOURCE_NAME)); + commonAggregateStats.get_resources_map() + .get(Constants.COMMON_CPU_RESOURCE_NAME)); result.put( "requestedGenericResourcesComp", prettifyGenericResources(commonAggregateStats.get_resources_map())); @@ -1249,6 +1313,7 @@ private static Map getCommonAggStatsMap(CommonAggregateStats com /** * getTruncatedErrorString. + * * @param errorString errorString * @return getTruncatedErrorString */ @@ -1258,6 +1323,7 @@ private static String getTruncatedErrorString(String errorString) { /** * getSpoutAggStatsMap. + * * @param componentAggregateStats componentAggregateStats * @param window window * @return getSpoutAggStatsMap @@ -1265,7 +1331,8 @@ private static String getTruncatedErrorString(String errorString) { private static Map getSpoutAggStatsMap( ComponentAggregateStats componentAggregateStats, String window) { Map result = new HashMap(); - SpoutAggregateStats spoutAggregateStats = componentAggregateStats.get_specific_stats().get_spout(); + SpoutAggregateStats spoutAggregateStats = componentAggregateStats.get_specific_stats() + .get_spout(); CommonAggregateStats commonStats = componentAggregateStats.get_common_stats(); result.put("window", window); result.put("windowPretty", getWindowHint(window)); @@ -1276,12 +1343,14 @@ private static Map getSpoutAggStatsMap( result.put("completeLatency", spoutAggregateStats.get_complete_latency_ms()); ErrorInfo lastError = componentAggregateStats.get_last_error(); - result.put("lastError", Objects.isNull(lastError) ? "" : getTruncatedErrorString(lastError.get_error())); + result.put("lastError", Objects.isNull(lastError) ? "" : getTruncatedErrorString(lastError + .get_error())); return result; } /** * getBoltAggStatsMap. + * * @param componentAggregateStats componentAggregateStats * @param window window * @return getBoltAggStatsMap @@ -1296,16 +1365,20 @@ private static Map getBoltAggStatsMap( result.put("transferred", commonStats.get_transferred()); result.put("acked", commonStats.get_acked()); result.put("failed", commonStats.get_failed()); - BoltAggregateStats boltAggregateStats = componentAggregateStats.get_specific_stats().get_bolt(); - result.put("executeLatency", StatsUtil.floatStr(boltAggregateStats.get_execute_latency_ms())); + BoltAggregateStats boltAggregateStats = componentAggregateStats.get_specific_stats() + .get_bolt(); + result.put("executeLatency", StatsUtil.floatStr(boltAggregateStats + .get_execute_latency_ms())); result.put("executed", boltAggregateStats.get_executed()); - result.put("processLatency", StatsUtil.floatStr(boltAggregateStats.get_process_latency_ms())); + result.put("processLatency", StatsUtil.floatStr(boltAggregateStats + .get_process_latency_ms())); result.put("capacity", StatsUtil.floatStr(boltAggregateStats.get_capacity())); return result; } /** * nullToZero. + * * @param value value * @return nullToZero */ @@ -1315,6 +1388,7 @@ private static Long nullToZero(Long value) { /** * nullToZero. + * * @param value value * @return nullToZero */ @@ -1324,6 +1398,7 @@ private static Double nullToZero(Double value) { /** * getBoltInputStats. + * * @param globalStreamId globalStreamId * @param componentAggregateStats componentAggregateStats * @return getBoltInputStats @@ -1331,15 +1406,18 @@ private static Double nullToZero(Double value) { private static Map getBoltInputStats(GlobalStreamId globalStreamId, ComponentAggregateStats componentAggregateStats) { Map result = new HashMap(); - SpecificAggregateStats specificAggregateStats = componentAggregateStats.get_specific_stats(); + SpecificAggregateStats specificAggregateStats = componentAggregateStats + .get_specific_stats(); BoltAggregateStats boltAggregateStats = specificAggregateStats.get_bolt(); CommonAggregateStats commonAggregateStats = componentAggregateStats.get_common_stats(); String componentId = globalStreamId.get_componentId(); result.put("component", componentId); result.put("encodedComponentId", Utils.urlEncodeUtf8(componentId)); result.put("stream", globalStreamId.get_streamId()); - result.put("executeLatency", StatsUtil.floatStr(boltAggregateStats.get_execute_latency_ms())); - result.put("processLatency", StatsUtil.floatStr(boltAggregateStats.get_process_latency_ms())); + result.put("executeLatency", StatsUtil.floatStr(boltAggregateStats + .get_execute_latency_ms())); + result.put("processLatency", StatsUtil.floatStr(boltAggregateStats + .get_process_latency_ms())); result.put("executed", nullToZero(boltAggregateStats.get_executed())); result.put("acked", nullToZero(commonAggregateStats.get_acked())); result.put("failed", nullToZero(commonAggregateStats.get_failed())); @@ -1348,6 +1426,7 @@ private static Map getBoltInputStats(GlobalStreamId globalStream /** * getBoltOutputStats. + * * @param streamId streamId * @param componentAggregateStats componentAggregateStats * @return getBoltOutputStats @@ -1364,20 +1443,23 @@ private static Map getBoltOutputStats(String streamId, /** * getSpoutOutputStats. + * * @param streamId streamId * @param componentAggregateStats componentAggregateStats * @return getSpoutOutputStats */ private static Map getSpoutOutputStats(String streamId, ComponentAggregateStats componentAggregateStats) { - SpecificAggregateStats specificAggregateStats = componentAggregateStats.get_specific_stats(); + SpecificAggregateStats specificAggregateStats = componentAggregateStats + .get_specific_stats(); SpoutAggregateStats spoutAggregateStats = specificAggregateStats.get_spout(); Map result = new HashMap(); result.put("stream", streamId); CommonAggregateStats commonStats = componentAggregateStats.get_common_stats(); result.put("emitted", nullToZero(commonStats.get_emitted())); result.put("transferred", nullToZero(commonStats.get_transferred())); - result.put("completeLatency", StatsUtil.floatStr(spoutAggregateStats.get_complete_latency_ms())); + result.put("completeLatency", StatsUtil.floatStr(spoutAggregateStats + .get_complete_latency_ms())); result.put("acked", nullToZero(commonStats.get_acked())); result.put("failed", nullToZero(commonStats.get_failed())); return result; @@ -1385,12 +1467,14 @@ private static Map getSpoutOutputStats(String streamId, /** * getBoltExecutorStats. + * * @param topologyId topologyId * @param config config * @param executorAggregateStats executorAggregateStats * @return getBoltExecutorStats */ - private static Map getBoltExecutorStats(String topologyId, Map config, + private static Map getBoltExecutorStats(String topologyId, Map config, ExecutorAggregateStats executorAggregateStats) { Map result = new HashMap(); ExecutorSummary executorSummary = executorAggregateStats.get_exec_summary(); @@ -1410,12 +1494,15 @@ private static Map getBoltExecutorStats(String topologyId, Map getBoltExecutorStats(String topologyId, Map getSpoutExecutorStats(String topologyId, Map config, + private static Map getSpoutExecutorStats(String topologyId, Map config, ExecutorAggregateStats executorAggregateStats) { Map result = new HashMap(); ExecutorSummary executorSummary = executorAggregateStats.get_exec_summary(); ExecutorInfo executorInfo = executorSummary.get_executor_info(); ComponentAggregateStats componentAggregateStats = executorAggregateStats.get_stats(); - SpecificAggregateStats specificAggregateStats = componentAggregateStats.get_specific_stats(); + SpecificAggregateStats specificAggregateStats = componentAggregateStats + .get_specific_stats(); SpoutAggregateStats spoutAggregateStats = specificAggregateStats.get_spout(); CommonAggregateStats commonAggregateStats = componentAggregateStats.get_common_stats(); String executorId = prettyExecutorInfo(executorInfo); @@ -1449,7 +1539,8 @@ private static Map getSpoutExecutorStats(String topologyId, Map< result.put("port", port); result.put("emitted", nullToZero(commonAggregateStats.get_emitted())); result.put("transferred", nullToZero(commonAggregateStats.get_transferred())); - result.put("completeLatency", StatsUtil.floatStr(spoutAggregateStats.get_complete_latency_ms())); + result.put("completeLatency", StatsUtil.floatStr(spoutAggregateStats + .get_complete_latency_ms())); result.put("acked", nullToZero(commonAggregateStats.get_acked())); result.put("failed", nullToZero(commonAggregateStats.get_failed())); result.put("workerLogLink", getWorkerLogLink(host, port, config, topologyId)); @@ -1458,7 +1549,8 @@ private static Map getSpoutExecutorStats(String topologyId, Map< /** * getComponentLastErrorInfo. - * Internal helper method that populates a hashmap with the component's most recently reported error. + * Internal helper method that populates a hashmap with the component's most recently reported + * error. * If the component has no such error reported, an empty "template" suitable for return over the * REST api is returned. * @@ -1467,11 +1559,13 @@ private static Map getSpoutExecutorStats(String topologyId, Map< * @param topologyId topologyId. * @return Map of values representing details about the most recently reported error. */ - private static Map getComponentLastErrorInfo(ErrorInfo lastError, Map config, String topologyId) { + private static Map getComponentLastErrorInfo(ErrorInfo lastError, Map config, + String topologyId) { Map result = new HashMap<>(); // Maintain backwards compatibility by defaulting these fields to empty string or null. - // If the lastError parameter is non-null, these keys will be populated with the appropriate values below. + // If the lastError parameter is non-null, these keys will be populated with the appropriate + // values below. result.put("lastError", ""); result.put("errorHost", ""); result.put("errorPort", (Integer) null); @@ -1487,11 +1581,14 @@ private static Map getComponentLastErrorInfo(ErrorInfo lastError /** * getComponentErrorInfo. + * * @param errorInfo errorInfo * @param config config * @param topologyId topologyId - * @param asLastError Pass a value of true if the result is to be used as part of a components 'lastError' response. - * Pass a value of false if the result is to be used as part of a components 'errors' response. + * @param asLastError Pass a value of true if the result is to be used as part of a components + * 'lastError' response. + * Pass a value of false if the result is to be used as part of a components 'errors' + * response. * @return getComponentErrorInfo */ private static Map getComponentErrorInfo(ErrorInfo errorInfo, Map config, @@ -1517,6 +1614,7 @@ private static Map getComponentErrorInfo(ErrorInfo errorInfo, Ma /** * getComponentErrors. + * * @param errorInfoList errorInfoList * @param topologyId topologyId * @param config config @@ -1536,6 +1634,7 @@ private static Map getComponentErrors(List errorInfoL /** * getTopologyErrors. + * * @param errorInfoList errorInfoList * @param topologyId topologyId * @param config config @@ -1555,6 +1654,7 @@ private static Map getTopologyErrors(List errorInfoLi /** * getTopologySpoutAggStatsMap. + * * @param componentAggregateStats componentAggregateStats * @param spoutId spoutId * @return getTopologySpoutAggStatsMap @@ -1566,14 +1666,18 @@ private static Map getTopologySpoutAggStatsMap(ComponentAggregat result.putAll(getCommonAggStatsMap(commonStats)); result.put("spoutId", spoutId); result.put("encodedSpoutId", Utils.urlEncodeUtf8(spoutId)); - SpoutAggregateStats spoutAggregateStats = componentAggregateStats.get_specific_stats().get_spout(); - result.put("completeLatency", StatsUtil.floatStr(spoutAggregateStats.get_complete_latency_ms())); - result.putAll(getComponentLastErrorInfo(componentAggregateStats.get_last_error(), config, topologyId)); + SpoutAggregateStats spoutAggregateStats = componentAggregateStats.get_specific_stats() + .get_spout(); + result.put("completeLatency", StatsUtil.floatStr(spoutAggregateStats + .get_complete_latency_ms())); + result.putAll(getComponentLastErrorInfo(componentAggregateStats.get_last_error(), config, + topologyId)); return result; } /** * getTopologyBoltAggStatsMap. + * * @param componentAggregateStats componentAggregateStats * @param boltId boltId * @return getTopologyBoltAggStatsMap @@ -1585,26 +1689,34 @@ private static Map getTopologyBoltAggStatsMap(ComponentAggregate result.putAll(getCommonAggStatsMap(commonStats)); result.put("boltId", boltId); result.put("encodedBoltId", Utils.urlEncodeUtf8(boltId)); - BoltAggregateStats boltAggregateStats = componentAggregateStats.get_specific_stats().get_bolt(); + BoltAggregateStats boltAggregateStats = componentAggregateStats.get_specific_stats() + .get_bolt(); result.put("capacity", StatsUtil.floatStr(boltAggregateStats.get_capacity())); - result.put("executeLatency", StatsUtil.floatStr(boltAggregateStats.get_execute_latency_ms())); + result.put("executeLatency", StatsUtil.floatStr(boltAggregateStats + .get_execute_latency_ms())); result.put("executed", boltAggregateStats.get_executed()); - result.put("processLatency", StatsUtil.floatStr(boltAggregateStats.get_process_latency_ms())); - result.putAll(getComponentLastErrorInfo(componentAggregateStats.get_last_error(), config, topologyId)); + result.put("processLatency", StatsUtil.floatStr(boltAggregateStats + .get_process_latency_ms())); + result.putAll(getComponentLastErrorInfo(componentAggregateStats.get_last_error(), config, + topologyId)); return result; } /** * getTopologyStatsMap. + * * @param topologyStats topologyStats * @return getTopologyStatsMap */ private static List getTopologyStatsMap(TopologyStats topologyStats) { List result = new ArrayList(); - Map emittedStatDisplayMap = getStatDisplayMapLong(topologyStats.get_window_to_emitted()); - Map transferred = getStatDisplayMapLong(topologyStats.get_window_to_transferred()); - Map completeLatency = getStatDisplayMap(topologyStats.get_window_to_complete_latencies_ms()); + Map emittedStatDisplayMap = getStatDisplayMapLong(topologyStats + .get_window_to_emitted()); + Map transferred = getStatDisplayMapLong(topologyStats + .get_window_to_transferred()); + Map completeLatency = getStatDisplayMap(topologyStats + .get_window_to_complete_latencies_ms()); Map acked = getStatDisplayMapLong(topologyStats.get_window_to_acked()); Map failed = getStatDisplayMapLong(topologyStats.get_window_to_failed()); for (String window : emittedStatDisplayMap.keySet()) { @@ -1613,7 +1725,8 @@ private static List getTopologyStatsMap(TopologyStats topologyStats) { temp.put("window", window); temp.put("emitted", emittedStatDisplayMap.get(window)); temp.put("transferred", transferred.get(window)); - temp.put("completeLatency", StatsUtil.floatStr(completeLatency.get(getWindowHint(window)))); + temp.put("completeLatency", StatsUtil.floatStr(completeLatency + .get(getWindowHint(window)))); temp.put("acked", acked.getOrDefault(window, 0L)); temp.put("failed", failed.getOrDefault(window, 0L)); @@ -1624,12 +1737,14 @@ private static List getTopologyStatsMap(TopologyStats topologyStats) { /** * unpackTopologyInfo. + * * @param topologyPageInfo topologyPageInfo * @param window window * @param config config * @return unpackTopologyInfo */ - private static Map unpackTopologyInfo(TopologyPageInfo topologyPageInfo, String window, Map config) { + private static Map unpackTopologyInfo(TopologyPageInfo topologyPageInfo, + String window, Map config) { Map result = new HashMap(); result.put("id", topologyPageInfo.get_id()); result.put("encodedId", Utils.urlEncodeUtf8(topologyPageInfo.get_id())); @@ -1646,23 +1761,34 @@ private static Map unpackTopologyInfo(TopologyPageInfo topologyP result.put("requestedMemOffHeap", topologyPageInfo.get_requested_memoffheap()); result.put("requestedCpu", topologyPageInfo.get_requested_cpu()); result.put("requestedTotalMem", - topologyPageInfo.get_requested_memonheap() + topologyPageInfo.get_requested_memoffheap() + topologyPageInfo.get_requested_memonheap() + topologyPageInfo + .get_requested_memoffheap() ); result.put("assignedMemOnHeap", topologyPageInfo.get_assigned_memonheap()); result.put("assignedMemOffHeap", topologyPageInfo.get_assigned_memoffheap()); result.put("assignedTotalMem", topologyPageInfo.get_assigned_memonheap() + topologyPageInfo.get_assigned_memoffheap()); result.put("assignedCpu", topologyPageInfo.get_assigned_cpu()); - result.put("requestedRegularOnHeapMem", topologyPageInfo.get_requested_regular_on_heap_memory()); - result.put("requestedSharedOnHeapMem", topologyPageInfo.get_requested_shared_on_heap_memory()); - result.put("requestedRegularOffHeapMem", topologyPageInfo.get_requested_regular_off_heap_memory()); - result.put("requestedSharedOffHeapMem", topologyPageInfo.get_requested_shared_off_heap_memory()); - result.put("requestedGenericResources", prettifyGenericResources(topologyPageInfo.get_requested_generic_resources())); - result.put("assignedRegularOnHeapMem", topologyPageInfo.get_assigned_regular_on_heap_memory()); - result.put("assignedSharedOnHeapMem", topologyPageInfo.get_assigned_shared_on_heap_memory()); - result.put("assignedRegularOffHeapMem", topologyPageInfo.get_assigned_regular_off_heap_memory()); - result.put("assignedSharedOffHeapMem", topologyPageInfo.get_assigned_shared_off_heap_memory()); - result.put("assignedGenericResources", prettifyGenericResources(topologyPageInfo.get_assigned_generic_resources())); + result.put("requestedRegularOnHeapMem", topologyPageInfo + .get_requested_regular_on_heap_memory()); + result.put("requestedSharedOnHeapMem", topologyPageInfo + .get_requested_shared_on_heap_memory()); + result.put("requestedRegularOffHeapMem", topologyPageInfo + .get_requested_regular_off_heap_memory()); + result.put("requestedSharedOffHeapMem", topologyPageInfo + .get_requested_shared_off_heap_memory()); + result.put("requestedGenericResources", prettifyGenericResources(topologyPageInfo + .get_requested_generic_resources())); + result.put("assignedRegularOnHeapMem", topologyPageInfo + .get_assigned_regular_on_heap_memory()); + result.put("assignedSharedOnHeapMem", topologyPageInfo + .get_assigned_shared_on_heap_memory()); + result.put("assignedRegularOffHeapMem", topologyPageInfo + .get_assigned_regular_off_heap_memory()); + result.put("assignedSharedOffHeapMem", topologyPageInfo + .get_assigned_shared_off_heap_memory()); + result.put("assignedGenericResources", prettifyGenericResources(topologyPageInfo + .get_assigned_generic_resources())); result.put("topologyStats", getTopologyStatsMap(topologyPageInfo.get_topology_stats())); List workerSummaries = new ArrayList(); if (topologyPageInfo.is_set_workers()) { @@ -1676,7 +1802,8 @@ private static Map unpackTopologyInfo(TopologyPageInfo topologyP List spoutStats = new ArrayList(); for (Map.Entry spoutEntry : spouts.entrySet()) { - spoutStats.add(getTopologySpoutAggStatsMap(spoutEntry.getValue(), spoutEntry.getKey(), config, topologyPageInfo.get_id())); + spoutStats.add(getTopologySpoutAggStatsMap(spoutEntry.getValue(), spoutEntry.getKey(), + config, topologyPageInfo.get_id())); } result.put("spouts", spoutStats); @@ -1684,7 +1811,8 @@ private static Map unpackTopologyInfo(TopologyPageInfo topologyP List boltStats = new ArrayList(); for (Map.Entry boltEntry : bolts.entrySet()) { - boltStats.add(getTopologyBoltAggStatsMap(boltEntry.getValue(), boltEntry.getKey(), config, topologyPageInfo.get_id())); + boltStats.add(getTopologyBoltAggStatsMap(boltEntry.getValue(), boltEntry.getKey(), + config, topologyPageInfo.get_id())); } result.put("bolts", boltStats); @@ -1708,6 +1836,7 @@ private static Map unpackTopologyInfo(TopologyPageInfo topologyP /** * getTopologyWorkers. + * * @param topologyInfo topologyInfo * @param config config * @return getTopologyWorkers. @@ -1730,20 +1859,24 @@ public static Map getTopologyWorkers(TopologyInfo topologyInfo, return result; } - /** * getTopologyLag. + * * @param userTopology userTopology * @param config config * @return getTopologyLag. */ - public static Map> getTopologyLag(StormTopology userTopology, Map config) { - Boolean disableLagMonitoring = (Boolean) (config.get(DaemonConfig.UI_DISABLE_SPOUT_LAG_MONITORING)); - return disableLagMonitoring ? Collections.EMPTY_MAP : TopologySpoutLag.lag(userTopology, config); + public static Map> getTopologyLag(StormTopology userTopology, + Map config) { + Boolean disableLagMonitoring = (Boolean) (config + .get(DaemonConfig.UI_DISABLE_SPOUT_LAG_MONITORING)); + return disableLagMonitoring ? Collections.EMPTY_MAP : TopologySpoutLag.lag(userTopology, + config); } /** * getBoltExecutors. + * * @param executorSummaries executorSummaries * @param stormTopology stormTopology * @param sys sys @@ -1753,9 +1886,11 @@ public static Map> getBoltExecutors(List> result = new HashMap(); for (ExecutorSummary executorSummary : executorSummaries) { - if (StatsUtil.componentType(stormTopology, executorSummary.get_component_id()).equals("bolt") + if (StatsUtil.componentType(stormTopology, executorSummary.get_component_id()) + .equals("bolt") && (sys || !Utils.isSystemId(executorSummary.get_component_id()))) { - List executorSummaryList = result.getOrDefault(executorSummary.get_component_id(), new ArrayList()); + List executorSummaryList = result.getOrDefault(executorSummary + .get_component_id(), new ArrayList()); executorSummaryList.add(executorSummary); result.put(executorSummary.get_component_id(), executorSummaryList); } @@ -1765,6 +1900,7 @@ public static Map> getBoltExecutors(List> getSpoutExecutors(List> result = new HashMap(); for (ExecutorSummary executorSummary : executorSummaries) { - if (StatsUtil.componentType(stormTopology, executorSummary.get_component_id()).equals("spout")) { - List executorSummaryList = result.getOrDefault(executorSummary.get_component_id(), new ArrayList()); + if (StatsUtil.componentType(stormTopology, executorSummary.get_component_id()) + .equals("spout")) { + List executorSummaryList = result.getOrDefault(executorSummary + .get_component_id(), new ArrayList()); executorSummaryList.add(executorSummary); result.put(executorSummary.get_component_id(), executorSummaryList); } @@ -1783,13 +1921,16 @@ public static Map> getSpoutExecutors(List 0 && Character.isLetter(streamName.charAt(0))) { return problemCharacterMatcher.replaceAll("_"); } else { @@ -1799,10 +1940,12 @@ public static String sanitizeStreamName(String streamName) { /** * sanitizeTransferredStats. + * * @param stats stats * @return sanitizeTransferredStats */ - public static Map> sanitizeTransferredStats(Map> stats) { + public static Map> sanitizeTransferredStats(Map> stats) { Map> result = new HashMap(); for (Map.Entry> entry : stats.entrySet()) { Map temp = new HashMap(); @@ -1816,6 +1959,7 @@ public static Map> sanitizeTransferredStats(Map getStatMapFromExecutorSummary(ExecutorSummary result.put(":uptime_secs", executorSummary.get_uptime_secs()); result.put(":transferred", null); if (executorSummary.is_set_stats()) { - result.put(":transferred", sanitizeTransferredStats(executorSummary.get_stats().get_transferred())); + result.put(":transferred", sanitizeTransferredStats(executorSummary.get_stats() + .get_transferred())); } return result; } - - /** * getInputMap. + * * @param entryInput entryInput * @return getInputMap */ @@ -1849,6 +1993,7 @@ public static Map getInputMap(Map.Entry getVisualizationData( getInfoOptions.set_num_err_choice(NumErrorsChoice.ONE); TopologyInfo topologyInfo = client.getTopologyInfoWithOpts(topoId, getInfoOptions); StormTopology stormTopology = client.getTopology(topoId); - Map> boltSummaries = getBoltExecutors(topologyInfo.get_executors(), stormTopology, sys); - Map> spoutSummaries = getSpoutExecutors(topologyInfo.get_executors(), stormTopology); + Map> boltSummaries = getBoltExecutors(topologyInfo + .get_executors(), stormTopology, sys); + Map> spoutSummaries = getSpoutExecutors(topologyInfo + .get_executors(), stormTopology); Map spoutSpecs = stormTopology.get_spouts(); Map boltSpecs = stormTopology.get_bolts(); @@ -1882,10 +2029,12 @@ public static Map getVisualizationData( spoutData.put(":transferred", spoutStreamsStats.get("transferred").get(window)); spoutData.put(":stats", spoutSummaries.get( spoutComponentId).stream().map( - UIHelpers::getStatMapFromExecutorSummary).collect(Collectors.toList())); + UIHelpers::getStatMapFromExecutorSummary).collect(Collectors + .toList())); spoutData.put( ":link", - UIHelpers.urlFormat("/component.html?id=%s&topology_id=%s", spoutComponentId, topoId) + UIHelpers.urlFormat("/component.html?id=%s&topology_id=%s", + spoutComponentId, topoId) ); spoutData.put(":inputs", @@ -1898,10 +2047,12 @@ public static Map getVisualizationData( for (Map.Entry boltEntry : boltSpecs.entrySet()) { String boltComponentId = boltEntry.getKey(); - if (boltSummaries.containsKey(boltComponentId) && (sys || !Utils.isSystemId(boltComponentId))) { + if (boltSummaries.containsKey(boltComponentId) && (sys || !Utils + .isSystemId(boltComponentId))) { Map boltMap = new HashMap(); boltMap.put(":type", "bolt"); - boltMap.put(":capacity", StatsUtil.computeBoltCapacity(boltSummaries.get(boltComponentId))); + boltMap.put(":capacity", StatsUtil.computeBoltCapacity(boltSummaries + .get(boltComponentId))); Map boltStreamsStats = StatsUtil.boltStreamsStats(boltSummaries.get(boltComponentId), sys); boltMap.put(":latency", boltStreamsStats.get("process-latencies").get(window)); @@ -1911,7 +2062,8 @@ public static Map getVisualizationData( UIHelpers::getStatMapFromExecutorSummary).collect(Collectors.toList())); boltMap.put( ":link", - UIHelpers.urlFormat("/component.html?id=%s&topology_id=%s", boltComponentId, topoId) + UIHelpers.urlFormat("/component.html?id=%s&topology_id=%s", boltComponentId, + topoId) ); boltMap.put(":inputs", @@ -1926,6 +2078,7 @@ public static Map getVisualizationData( /** * getStreamBox. + * * @param visualization visualization * @return getStreamBox */ @@ -1941,6 +2094,7 @@ public static Map getStreamBox(Object visualization) { /** * getBuildVisualization. + * * @param client client * @param config config * @param window window @@ -1954,26 +2108,30 @@ public static Map getBuildVisualization( throws TException { Map result = new HashMap(); Map visualizationData = getVisualizationData(client, window, id, sys); - List streamBoxes = visualizationData.entrySet().stream().map(UIHelpers::getStreamBox).collect(Collectors.toList()); + List streamBoxes = visualizationData.entrySet().stream().map(UIHelpers::getStreamBox) + .collect(Collectors.toList()); result.put("visualizationTable", Lists.partition(streamBoxes, 4)); return result; } /** * getActiveAction. + * * @param profileRequest profileRequest * @param config config * @param topologyId topologyId * @return getActiveAction */ - public static Map getActiveAction(ProfileRequest profileRequest, Map config, String topologyId) { + public static Map getActiveAction(ProfileRequest profileRequest, Map config, + String topologyId) { Map result = new HashMap(); result.put("host", profileRequest.get_nodeInfo().get_node()); result.put("port", String.valueOf(profileRequest.get_nodeInfo().get_port().toArray()[0])); result.put("dumplink", getWorkerDumpLink( profileRequest.get_nodeInfo().get_node(), - (Long) profileRequest.get_nodeInfo().get_port().toArray()[0], topologyId, config + (Long) profileRequest.get_nodeInfo().get_port() + .toArray()[0], topologyId, config )); result.put("timestamp", System.currentTimeMillis() - profileRequest.get_time_stamp()); return result; @@ -1981,6 +2139,7 @@ public static Map getActiveAction(ProfileRequest profileRequest, /** * getActiveProfileActions. + * * @param client client * @param id id * @param component component @@ -1988,10 +2147,13 @@ public static Map getActiveAction(ProfileRequest profileRequest, * @return getActiveProfileActions * @throws TException TException */ - public static List getActiveProfileActions(Nimbus.Iface client, String id, String component, Map config) throws TException { + public static List getActiveProfileActions(Nimbus.Iface client, String id, String component, + Map config) throws TException { List profileRequests = - client.getComponentPendingProfileActions(id, component, ProfileAction.JPROFILE_STOP); - return profileRequests.stream().map(x -> UIHelpers.getActiveAction(x, config, id)).collect(Collectors.toList()); + client.getComponentPendingProfileActions(id, component, + ProfileAction.JPROFILE_STOP); + return profileRequests.stream().map(x -> UIHelpers.getActiveAction(x, config, id)) + .collect(Collectors.toList()); } /** @@ -2003,7 +2165,8 @@ public static List getActiveProfileActions(Nimbus.Iface client, String id, Strin * @param config config * @return getWorkerDumpLink */ - public static String getWorkerDumpLink(String host, long port, String topologyId, Map config) { + public static String getWorkerDumpLink(String host, long port, String topologyId, Map config) { if (isSecureLogviewer(config)) { return UIHelpers.urlFormat( "https://%s:%s/api/v1/dumps/%s/%s", @@ -2024,6 +2187,7 @@ public static String getWorkerDumpLink(String host, long port, String topologyId /** * unpackBoltPageInfo. + * * @param componentPageInfo componentPageInfo * @param topologyId topologyId * @param window window @@ -2066,6 +2230,7 @@ public static Map unpackBoltPageInfo(ComponentPageInfo component /** * unpackSpoutPageInfo. + * * @param componentPageInfo componentPageInfo * @param topologyId topologyId * @param window window @@ -2101,6 +2266,7 @@ public static Map unpackSpoutPageInfo(ComponentPageInfo componen /** * getComponentPage. + * * @param client client * @param id id * @param component component @@ -2132,9 +2298,11 @@ public static Map getComponentPage( result.put("executors", componentPageInfo.get_num_executors()); result.put("tasks", componentPageInfo.get_num_tasks()); result.put("requestedMemOnHeap", - componentPageInfo.get_resources_map().get(Constants.COMMON_ONHEAP_MEMORY_RESOURCE_NAME)); + componentPageInfo.get_resources_map() + .get(Constants.COMMON_ONHEAP_MEMORY_RESOURCE_NAME)); result.put("requestedMemOffHeap", - componentPageInfo.get_resources_map().get(Constants.COMMON_OFFHEAP_MEMORY_RESOURCE_NAME)); + componentPageInfo.get_resources_map() + .get(Constants.COMMON_OFFHEAP_MEMORY_RESOURCE_NAME)); result.put("requestedCpu", componentPageInfo.get_resources_map().get(Constants.COMMON_CPU_RESOURCE_NAME)); result.put("requestedGenericResources", @@ -2145,9 +2313,11 @@ public static Map getComponentPage( result.put("topologyStatus", componentPageInfo.get_topology_status()); result.put("encodedTopologyId", Utils.urlEncodeUtf8(id)); result.put("window", window); - result.put("componentType", componentPageInfo.get_component_type().toString().toLowerCase()); + result.put("componentType", componentPageInfo.get_component_type().toString() + .toLowerCase()); result.put("windowHint", getWindowHint(window)); - result.put("debug", componentPageInfo.is_set_debug_options() && componentPageInfo.get_debug_options().is_enable()); + result.put("debug", componentPageInfo.is_set_debug_options() && componentPageInfo + .get_debug_options().is_enable()); double samplingPct = 10; if (componentPageInfo.is_set_debug_options()) { samplingPct = componentPageInfo.get_debug_options().get_samplingpct(); @@ -2156,7 +2326,8 @@ public static Map getComponentPage( String eventlogHost = componentPageInfo.get_eventlog_host(); if (null != eventlogHost && !eventlogHost.isEmpty()) { result.put("eventLogLink", getLogviewerLink(eventlogHost, - WebAppUtils.eventLogsFilename(id, String.valueOf(componentPageInfo.get_eventlog_port())), + WebAppUtils.eventLogsFilename(id, String.valueOf(componentPageInfo + .get_eventlog_port())), config, componentPageInfo.get_eventlog_port())); } result.put("profilingAndDebuggingCapable", !Utils.isOnWindows()); @@ -2169,13 +2340,15 @@ public static Map getComponentPage( /** * getTopolgoyLogConfig. + * * @param logConfig logConfig * @return getTopolgoyLogConfig */ public static Map getTopolgoyLogConfig(LogConfig logConfig) { Map result = new HashMap(); if (logConfig.is_set_named_logger_level()) { - for (Map.Entry entry : logConfig.get_named_logger_level().entrySet()) { + for (Map.Entry entry : logConfig.get_named_logger_level() + .entrySet()) { Map temp = new HashMap(); temp.put("target_level", entry.getValue().get_target_log_level()); temp.put("reset_level", entry.getValue().get_reset_log_level()); @@ -2191,6 +2364,7 @@ public static Map getTopolgoyLogConfig(LogConfig logConfig) { /** * getTopologyOpResponse. + * * @param id id * @param op op * @return getTopologyOpResponse @@ -2205,12 +2379,14 @@ public static Map getTopologyOpResponse(String id, String op) { /** * putTopologyActivate. + * * @param client client * @param id id * @return putTopologyActivate * @throws TException TException */ - public static Map putTopologyActivate(Nimbus.Iface client, String id) throws TException { + public static Map putTopologyActivate(Nimbus.Iface client, + String id) throws TException { GetInfoOptions getInfoOptions = new GetInfoOptions(); getInfoOptions.set_num_err_choice(NumErrorsChoice.NONE); TopologyInfo topologyInfo = client.getTopologyInfoWithOpts(id, getInfoOptions); @@ -2220,12 +2396,14 @@ public static Map putTopologyActivate(Nimbus.Iface client, Strin /** * putTopologyDeactivate. + * * @param client client * @param id id * @return putTopologyDeactivate * @throws TException TException */ - public static Map putTopologyDeactivate(Nimbus.Iface client, String id) throws TException { + public static Map putTopologyDeactivate(Nimbus.Iface client, + String id) throws TException { GetInfoOptions getInfoOptions = new GetInfoOptions(); getInfoOptions.set_num_err_choice(NumErrorsChoice.NONE); TopologyInfo topologyInfo = client.getTopologyInfoWithOpts(id, getInfoOptions); @@ -2235,6 +2413,7 @@ public static Map putTopologyDeactivate(Nimbus.Iface client, Str /** * putTopologyDebugActionSpct. + * * @param client client * @param id id * @param action action @@ -2248,12 +2427,14 @@ public static Map putTopologyDebugActionSpct( GetInfoOptions getInfoOptions = new GetInfoOptions(); getInfoOptions.set_num_err_choice(NumErrorsChoice.NONE); TopologyInfo topologyInfo = client.getTopologyInfoWithOpts(id, getInfoOptions); - client.debug(topologyInfo.get_name(), component, action.equals("enable"), Integer.parseInt(spct)); + client.debug(topologyInfo.get_name(), component, action.equals("enable"), Integer + .parseInt(spct)); return getTopologyOpResponse(id, "debug/" + action); } /** * putTopologyRebalance. + * * @param client client * @param id id * @param waitTime waitTime @@ -2273,13 +2454,15 @@ public static Map putTopologyRebalance( /** * putTopologyKill. + * * @param client client * @param id id * @param waitTime waitTime * @return putTopologyKill * @throws TException TException */ - public static Map putTopologyKill(Nimbus.Iface client, String id, String waitTime) throws TException { + public static Map putTopologyKill(Nimbus.Iface client, String id, + String waitTime) throws TException { GetInfoOptions getInfoOptions = new GetInfoOptions(); getInfoOptions.set_num_err_choice(NumErrorsChoice.NONE); TopologyInfo topologyInfo = client.getTopologyInfoWithOpts(id, getInfoOptions); @@ -2291,6 +2474,7 @@ public static Map putTopologyKill(Nimbus.Iface client, String id /** * setTopologyProfilingAction. + * * @param client client * @param id id * @param hostPort hostPort @@ -2315,6 +2499,7 @@ public static void setTopologyProfilingAction( /** * getTopologyProfilingStart. + * * @param client client * @param id id * @param hostPort hostPort @@ -2341,6 +2526,7 @@ public static Map getTopologyProfilingStart(Nimbus.Iface client, /** * getTopologyProfilingStop. + * * @param client client * @param id id * @param hostPort hostPort @@ -2360,6 +2546,7 @@ public static Map getTopologyProfilingStop(Nimbus.Iface client, /** * getProfilingDisabled. + * * @return getProfilingDisabled */ public static Map getProfilingDisabled() { @@ -2371,6 +2558,7 @@ public static Map getProfilingDisabled() { /** * getTopologyProfilingDump. + * * @param client client * @param id id * @param hostPort hostPort @@ -2378,7 +2566,8 @@ public static Map getProfilingDisabled() { * @return getTopologyProfilingDump * @throws TException TException */ - public static Map getTopologyProfilingDump(Nimbus.Iface client, String id, String hostPort, + public static Map getTopologyProfilingDump(Nimbus.Iface client, String id, + String hostPort, Map config) throws TException { setTopologyProfilingAction( client, id, hostPort, System.currentTimeMillis(), @@ -2404,6 +2593,7 @@ public static Map getTopologyProfilingDumpJstack(Nimbus.Iface cl /** * getTopologyProfilingRestartWorker. + * * @param client client * @param id id * @param hostPort hostPort @@ -2425,6 +2615,7 @@ public static Map getTopologyProfilingRestartWorker(Nimbus.Iface /** * getTopologyProfilingDumpHeap. + * * @param client client * @param id id * @param hostPort hostport @@ -2432,9 +2623,11 @@ public static Map getTopologyProfilingRestartWorker(Nimbus.Iface * @return getTopologyProfilingDumpHeap * @throws TException TException */ - public static Map getTopologyProfilingDumpHeap(Nimbus.Iface client, String id, String hostPort, + public static Map getTopologyProfilingDumpHeap(Nimbus.Iface client, String id, + String hostPort, Map config) throws TException { - setTopologyProfilingAction(client, id, hostPort, System.currentTimeMillis(), config, ProfileAction.JMAP_DUMP); + setTopologyProfilingAction(client, id, hostPort, System.currentTimeMillis(), config, + ProfileAction.JMAP_DUMP); Map result = new HashMap(); result.put("status", "ok"); result.put("id", hostPort); @@ -2443,6 +2636,7 @@ public static Map getTopologyProfilingDumpHeap(Nimbus.Iface clie /** * putTopologyLogLevel. + * * @param client client * @param namedLogLevel namedLogLevel * @param id id @@ -2462,7 +2656,8 @@ public static Map putTopologyLogLevel(Nimbus.Iface client, logLevel.unset_target_log_level(); } else { logLevel.set_action(LogLevelAction.UPDATE); - logLevel.set_target_log_level(org.apache.logging.log4j.Level.toLevel(targetLevel).name()); + logLevel.set_target_log_level(org.apache.logging.log4j.Level.toLevel(targetLevel) + .name()); logLevel.set_reset_log_level_timeout_secs(Math.toIntExact(timeout)); } LogConfig logConfig = new LogConfig(); @@ -2474,11 +2669,13 @@ public static Map putTopologyLogLevel(Nimbus.Iface client, /** * getNimbusSummary. + * * @param clusterInfo clusterInfo * @param config config * @return getNimbusSummary */ - public static Map getNimbusSummary(ClusterSummary clusterInfo, Map config) { + public static Map getNimbusSummary(ClusterSummary clusterInfo, Map config) { List nimbusSummaries = clusterInfo.get_nimbuses(); List nimbusSeeds = new ArrayList(); for (String nimbusHost : (List) config.get(Config.NIMBUS_SEEDS)) { @@ -2501,9 +2698,11 @@ public static Map getNimbusSummary(ClusterSummary clusterInfo, M nimbusSummaryMap.put("version", nimbusSummary.get_version()); nimbusSummaryMap.put("nimbusUpTimeSeconds", nimbusSummary.get_uptime_secs()); nimbusSummaryMap.put("nimbusUpTime", prettyUptimeSec(nimbusSummary.get_uptime_secs())); - nimbusSummaryMap.put("nimbusLogLink", getNimbusLogLink(nimbusSummary.get_host(), config)); + nimbusSummaryMap.put("nimbusLogLink", getNimbusLogLink(nimbusSummary.get_host(), + config)); resultSummaryList.add(nimbusSummaryMap); - nimbusSeeds.remove(nimbusSummary.get_host() + ":" + String.valueOf(nimbusSummary.get_port())); + nimbusSeeds.remove(nimbusSummary.get_host() + ":" + String.valueOf(nimbusSummary + .get_port())); } for (String nimbusSeed : nimbusSeeds) { @@ -2514,7 +2713,8 @@ public static Map getNimbusSummary(ClusterSummary clusterInfo, M nimbusSummaryMap.put("version", "Not applicable"); nimbusSummaryMap.put("nimbusUpTimeSeconds", "Not applicable"); nimbusSummaryMap.put("nimbusUpTime", "Not applicable"); - nimbusSummaryMap.put("nimbusLogLink", getNimbusLogLink(nimbusSeed.split(":")[0], config)); + nimbusSummaryMap.put("nimbusLogLink", getNimbusLogLink(nimbusSeed.split(":")[0], + config)); resultSummaryList.add(nimbusSummaryMap); } Map result = new HashMap(); diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/ui/UIServer.java b/storm-webapp/src/main/java/org/apache/storm/daemon/ui/UIServer.java index ee7c8e4ac61..fbc68b8cd7a 100644 --- a/storm-webapp/src/main/java/org/apache/storm/daemon/ui/UIServer.java +++ b/storm-webapp/src/main/java/org/apache/storm/daemon/ui/UIServer.java @@ -22,14 +22,12 @@ import static org.apache.storm.utils.ConfigUtils.STORM_HOME; import jakarta.servlet.DispatcherType; - import java.nio.file.Files; import java.nio.file.Paths; import java.util.Arrays; import java.util.EnumSet; import java.util.List; import java.util.Map; - import org.apache.storm.DaemonConfig; import org.apache.storm.daemon.drpc.webapp.ReqContextFilter; import org.apache.storm.daemon.ui.exceptionmappers.AuthorizationExceptionMapper; @@ -68,19 +66,22 @@ public class UIServer { /** * addRequestContextFilter. + * * @param context context * @param configName configName * @param conf conf */ public static void addRequestContextFilter(ServletContextHandler context, String configName, Map conf) { - IHttpCredentialsPlugin auth = ServerAuthUtils.getHttpCredentialsPlugin(conf, (String) conf.get(configName)); + IHttpCredentialsPlugin auth = ServerAuthUtils.getHttpCredentialsPlugin(conf, (String) conf + .get(configName)); ReqContextFilter filter = new ReqContextFilter(auth); context.addFilter(new FilterHolder(filter), "/*", EnumSet.allOf(DispatcherType.class)); } /** - * main. + * Main. + * * @param args args */ public static void main(String[] args) { @@ -93,18 +94,25 @@ public static void main(String[] args) { final String httpsKsType = (String) (conf.get(DaemonConfig.UI_HTTPS_KEYSTORE_TYPE)); final String httpsKeyPassword = (String) (conf.get(DaemonConfig.UI_HTTPS_KEY_PASSWORD)); final String httpsTsPath = (String) (conf.get(DaemonConfig.UI_HTTPS_TRUSTSTORE_PATH)); - final String httpsTsPassword = (String) (conf.get(DaemonConfig.UI_HTTPS_TRUSTSTORE_PASSWORD)); + final String httpsTsPassword = (String) (conf + .get(DaemonConfig.UI_HTTPS_TRUSTSTORE_PASSWORD)); final String httpsTsType = (String) (conf.get(DaemonConfig.UI_HTTPS_TRUSTSTORE_TYPE)); - final Boolean httpsWantClientAuth = (Boolean) (conf.get(DaemonConfig.UI_HTTPS_WANT_CLIENT_AUTH)); - final Boolean httpsNeedClientAuth = (Boolean) (conf.get(DaemonConfig.UI_HTTPS_NEED_CLIENT_AUTH)); - final Boolean disableHttpBinding = (Boolean) (conf.get(DaemonConfig.UI_DISABLE_HTTP_BINDING)); - final boolean enableSslReload = ObjectReader.getBoolean(conf.get(DaemonConfig.UI_HTTPS_ENABLE_SSL_RELOAD), false); + final Boolean httpsWantClientAuth = (Boolean) (conf + .get(DaemonConfig.UI_HTTPS_WANT_CLIENT_AUTH)); + final Boolean httpsNeedClientAuth = (Boolean) (conf + .get(DaemonConfig.UI_HTTPS_NEED_CLIENT_AUTH)); + final Boolean disableHttpBinding = (Boolean) (conf + .get(DaemonConfig.UI_DISABLE_HTTP_BINDING)); + final boolean enableSslReload = ObjectReader.getBoolean(conf + .get(DaemonConfig.UI_HTTPS_ENABLE_SSL_RELOAD), false); Server jettyServer = UIHelpers.jettyCreateServer( - (int) conf.get(DaemonConfig.UI_PORT), null, httpsPort, headerBufferSize, disableHttpBinding); + (int) conf + .get(DaemonConfig.UI_PORT), null, httpsPort, headerBufferSize, disableHttpBinding); - UIHelpers.configSsl(jettyServer, httpsPort, httpsKsPath, httpsKsPassword, httpsKsType, httpsKeyPassword, + UIHelpers.configSsl(jettyServer, httpsPort, httpsKsPath, httpsKsPassword, httpsKsType, + httpsKeyPassword, httpsTsPath, httpsTsPassword, httpsTsType, httpsNeedClientAuth, httpsWantClientAuth, enableSslReload); ServletContextHandler context = new ServletContextHandler(ServletContextHandler.SESSIONS); @@ -116,7 +124,8 @@ public static void main(String[] args) { (String) conf.get(DaemonConfig.UI_FILTER), (Map) conf.get(DaemonConfig.UI_FILTER_PARAMS) ); - final List filterConfigurationList = Arrays.asList(filterConfiguration); + final List filterConfigurationList = Arrays + .asList(filterConfiguration); UIHelpers.configFilters(context, filterConfigurationList); @@ -145,7 +154,8 @@ protected void configure() { // add special pathspec of static content mapped to the homePath ServletHolder holderHome = new ServletHolder("static-home", DefaultServlet.class); - String packagedStaticFileLocation = System.getProperty(STORM_HOME) + FILE_SEPARATOR + "public/"; + String packagedStaticFileLocation = System.getProperty(STORM_HOME) + FILE_SEPARATOR + + "public/"; if (Files.exists(Paths.get(packagedStaticFileLocation))) { holderHome.setInitParameter("resourceBase", packagedStaticFileLocation); @@ -166,7 +176,8 @@ protected void configure() { holderHome.setInitParameter("dirAllowed", "false"); holderHome.setInitParameter("pathInfoOnly", "true"); - context.addFilter(new FilterHolder(new HeaderResponseServletFilter(metricsRegistry)), "/*", EnumSet.allOf(DispatcherType.class)); + context.addFilter(new FilterHolder(new HeaderResponseServletFilter(metricsRegistry)), "/*", + EnumSet.allOf(DispatcherType.class)); context.addServlet(holderHome, "/*"); diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/ui/exceptionmappers/DefaultExceptionMapper.java b/storm-webapp/src/main/java/org/apache/storm/daemon/ui/exceptionmappers/DefaultExceptionMapper.java index 29403d2fc8e..f4ece0c3f64 100644 --- a/storm-webapp/src/main/java/org/apache/storm/daemon/ui/exceptionmappers/DefaultExceptionMapper.java +++ b/storm-webapp/src/main/java/org/apache/storm/daemon/ui/exceptionmappers/DefaultExceptionMapper.java @@ -34,6 +34,7 @@ public class DefaultExceptionMapper implements ExceptionMapper { /** * toResponse. + * * @param throwable throwable * @return response */ diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/ui/exceptionmappers/ExceptionMapperUtils.java b/storm-webapp/src/main/java/org/apache/storm/daemon/ui/exceptionmappers/ExceptionMapperUtils.java index 4db1d7da73d..ee61a20c16c 100644 --- a/storm-webapp/src/main/java/org/apache/storm/daemon/ui/exceptionmappers/ExceptionMapperUtils.java +++ b/storm-webapp/src/main/java/org/apache/storm/daemon/ui/exceptionmappers/ExceptionMapperUtils.java @@ -21,17 +21,16 @@ import jakarta.inject.Provider; import jakarta.servlet.http.HttpServletRequest; import jakarta.ws.rs.core.Response; - import org.apache.storm.daemon.common.JsonResponseBuilder; import org.apache.storm.daemon.ui.UIHelpers; import org.apache.storm.daemon.ui.resources.StormApiResource; import org.apache.storm.generated.AuthorizationException; - public class ExceptionMapperUtils { /** * getResponse. + * * @param ex ex * @param responseStatus responseStatus * @return getResponse @@ -40,7 +39,8 @@ public static Response getResponse(Exception ex, Response.Status responseStatus, Provider request) { String callback = null; if (request.get().getParameterMap().containsKey(StormApiResource.callbackParameterName)) { - callback = String.valueOf(request.get().getParameterMap().get(StormApiResource.callbackParameterName)); + callback = String.valueOf(request.get().getParameterMap() + .get(StormApiResource.callbackParameterName)); } return new JsonResponseBuilder().setData( UIHelpers.exceptionToJson(ex, responseStatus.getStatusCode())).setCallback(callback) @@ -49,16 +49,19 @@ public static Response getResponse(Exception ex, Response.Status responseStatus, /** * getResponse. + * * @param ex ex * @param request request * @return getResponse */ - public static Response getResponse(AuthorizationException ex, Provider request) { + public static Response getResponse(AuthorizationException ex, + Provider request) { return getResponse(ex, Response.Status.UNAUTHORIZED, request); } /** * getResponse. + * * @param ex ex * @return getResponse */ diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/ui/exceptionmappers/NotAliveExceptionMapper.java b/storm-webapp/src/main/java/org/apache/storm/daemon/ui/exceptionmappers/NotAliveExceptionMapper.java index e0d66dbc508..30e8a215536 100644 --- a/storm-webapp/src/main/java/org/apache/storm/daemon/ui/exceptionmappers/NotAliveExceptionMapper.java +++ b/storm-webapp/src/main/java/org/apache/storm/daemon/ui/exceptionmappers/NotAliveExceptionMapper.java @@ -27,7 +27,6 @@ import jakarta.ws.rs.ext.Provider; import org.apache.storm.generated.NotAliveException; - @Provider public class NotAliveExceptionMapper implements ExceptionMapper { diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/ui/filters/AuthorizedUserFilter.java b/storm-webapp/src/main/java/org/apache/storm/daemon/ui/filters/AuthorizedUserFilter.java index f26175892af..82c50347ec5 100644 --- a/storm-webapp/src/main/java/org/apache/storm/daemon/ui/filters/AuthorizedUserFilter.java +++ b/storm-webapp/src/main/java/org/apache/storm/daemon/ui/filters/AuthorizedUserFilter.java @@ -74,22 +74,26 @@ public class AuthorizedUserFilter implements ContainerRequestFilter { /** * makeResponse. + * * @param ex ex * @param request request * @param statusCode statusCode * @return error response */ - public static Response makeResponse(Exception ex, ContainerRequestContext request, int statusCode) { + public static Response makeResponse(Exception ex, ContainerRequestContext request, + int statusCode) { String callback = null; - if (request.getMediaType() != null && request.getMediaType().equals(MediaType.APPLICATION_JSON_TYPE)) { + if (request.getMediaType() != null && request.getMediaType() + .equals(MediaType.APPLICATION_JSON_TYPE)) { try { String json = IOUtils.toString(request.getEntityStream(), Charsets.UTF_8); InputStream in = IOUtils.toInputStream(json); request.setEntityStream(in); Map requestBody = (Map) JSONValue.parse(json); if (requestBody.containsKey(StormApiResource.callbackParameterName)) { - callback = String.valueOf(requestBody.get(StormApiResource.callbackParameterName)); + callback = String.valueOf(requestBody + .get(StormApiResource.callbackParameterName)); } } catch (IOException e) { LOG.error("Exception while trying to get callback ", e); @@ -127,17 +131,20 @@ public void filter(ContainerRequestContext containerRequestContext) { Map topoConf = null; if (annotation.needsTopoId()) { - final String topoId = containerRequestContext.getUriInfo().getPathParameters().get("id").get(0); + final String topoId = containerRequestContext.getUriInfo().getPathParameters().get("id") + .get(0); try (NimbusClient nimbusClient = NimbusClient.Builder.withConf(conf).build()) { topoConf = (Map) JSONValue.parse(nimbusClient.getClient().getTopologyConf(topoId)); } catch (AuthorizationException ae) { - LOG.error("Nimbus isn't allowing {} to access the topology conf of {}. {}", ReqContext.context(), topoId, ae.get_msg()); + LOG.error("Nimbus isn't allowing {} to access the topology conf of {}. {}", + ReqContext.context(), topoId, ae.get_msg()); containerRequestContext.abortWith(makeResponse(ae, containerRequestContext, 403)); return; } catch (TException e) { LOG.error("Unable to fetch topo conf for {} due to ", topoId, e); containerRequestContext.abortWith( - makeResponse(new IOException("Unable to fetch topo conf for topo id " + topoId, e), + makeResponse(new IOException("Unable to fetch topo conf for topo id " + topoId, + e), containerRequestContext, 500) ); return; @@ -163,16 +170,20 @@ public void filter(ContainerRequestContext containerRequestContext) { containerRequestContext.abortWith( makeResponse(new AuthorizationException( - "user '" + realUser + "' is not authorized to impersonate user '" - + user + "' from host '" + remoteAddress.toString() + "'. Please" - + "see SECURITY.MD to learn how to configure impersonation ACL." + "user '" + realUser + + "' is not authorized to impersonate user '" + + user + "' from host '" + remoteAddress.toString() + + "'. Please" + + "see SECURITY.MD to learn how to configure " + + "impersonation ACL." ), containerRequestContext, 401) ); return; } } else { LOG.warn("Principal {} is trying to impersonate {} but {} is not configured. " - + "This is a potential security hole. Please see SECURITY.MD to learn how to " + + "This is a potential security hole. Please see SECURITY.MD to " + + "learn how to " + "configure an impersonation authorizer.", reqContext.realPrincipal().toString(), reqContext.principal().toString(), DaemonConfig.NIMBUS_IMPERSONATION_AUTHORIZER); diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/ui/filters/HeaderResponseFilter.java b/storm-webapp/src/main/java/org/apache/storm/daemon/ui/filters/HeaderResponseFilter.java index 394849dc417..672c77e8730 100644 --- a/storm-webapp/src/main/java/org/apache/storm/daemon/ui/filters/HeaderResponseFilter.java +++ b/storm-webapp/src/main/java/org/apache/storm/daemon/ui/filters/HeaderResponseFilter.java @@ -29,7 +29,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; - @Provider public class HeaderResponseFilter implements ContainerResponseFilter { public static final Logger LOG = LoggerFactory.getLogger(HeaderResponseFilter.class); diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/ui/resources/AuthNimbusOp.java b/storm-webapp/src/main/java/org/apache/storm/daemon/ui/resources/AuthNimbusOp.java index 530f4af2763..311a62fdd49 100644 --- a/storm-webapp/src/main/java/org/apache/storm/daemon/ui/resources/AuthNimbusOp.java +++ b/storm-webapp/src/main/java/org/apache/storm/daemon/ui/resources/AuthNimbusOp.java @@ -27,12 +27,14 @@ public @interface AuthNimbusOp { /** * nimbusOP. + * * @return nimbusOp */ String value(); /** * needsTopoId. + * * @return needsTopoId */ boolean needsTopoId() default false; diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/ui/resources/StormApiResource.java b/storm-webapp/src/main/java/org/apache/storm/daemon/ui/resources/StormApiResource.java index c84f86f8b98..7519b2d1066 100644 --- a/storm-webapp/src/main/java/org/apache/storm/daemon/ui/resources/StormApiResource.java +++ b/storm-webapp/src/main/java/org/apache/storm/daemon/ui/resources/StormApiResource.java @@ -80,25 +80,44 @@ public class StormApiResource { @Inject public StormApiResource(StormMetricsRegistry metricsRegistry) { - this.clusterConfigurationRequestMeter = metricsRegistry.registerMeter("ui:num-cluster-configuration-http-requests"); - this.clusterSummaryRequestMeter = metricsRegistry.registerMeter("ui:num-cluster-summary-http-requests"); - this.nimbusSummaryRequestMeter = metricsRegistry.registerMeter("ui:num-nimbus-summary-http-requests"); - this.supervisorRequestMeter = metricsRegistry.registerMeter("ui:num-supervisor-http-requests"); - this.supervisorSummaryRequestMeter = metricsRegistry.registerMeter("ui:num-supervisor-summary-http-requests"); - this.allTopologiesSummaryRequestMeter = metricsRegistry.registerMeter("ui:num-all-topologies-summary-http-requests"); - this.topologyPageRequestMeter = metricsRegistry.registerMeter("ui:num-topology-page-http-requests"); - this.topologyMetricRequestMeter = metricsRegistry.registerMeter("ui:num-topology-metric-http-requests"); - this.buildVisualizationRequestMeter = metricsRegistry.registerMeter("ui:num-build-visualization-http-requests"); - this.mkVisualizationDataRequestMeter = metricsRegistry.registerMeter("ui:num-mk-visualization-data-http-requests"); - this.componentPageRequestMeter = metricsRegistry.registerMeter("ui:num-component-page-http-requests"); - this.logConfigRequestMeter = metricsRegistry.registerMeter("ui:num-log-config-http-requests"); - this.activateTopologyRequestMeter = metricsRegistry.registerMeter("ui:num-activate-topology-http-requests"); - this.deactivateTopologyRequestMeter = metricsRegistry.registerMeter("ui:num-deactivate-topology-http-requests"); - this.debugTopologyRequestMeter = metricsRegistry.registerMeter("ui:num-debug-topology-http-requests"); - this.componentOpResponseRequestMeter = metricsRegistry.registerMeter("ui:num-component-op-response-http-requests"); - this.topologyOpResponseMeter = metricsRegistry.registerMeter("ui:num-topology-op-response-http-requests"); - this.topologyLagRequestMeter = metricsRegistry.registerMeter("ui:num-topology-lag-http-requests"); - this.getOwnerResourceSummariesMeter = metricsRegistry.registerMeter("ui:num-get-owner-resource-summaries-http-request"); + this.clusterConfigurationRequestMeter = metricsRegistry + .registerMeter("ui:num-cluster-configuration-http-requests"); + this.clusterSummaryRequestMeter = metricsRegistry + .registerMeter("ui:num-cluster-summary-http-requests"); + this.nimbusSummaryRequestMeter = metricsRegistry + .registerMeter("ui:num-nimbus-summary-http-requests"); + this.supervisorRequestMeter = metricsRegistry + .registerMeter("ui:num-supervisor-http-requests"); + this.supervisorSummaryRequestMeter = metricsRegistry + .registerMeter("ui:num-supervisor-summary-http-requests"); + this.allTopologiesSummaryRequestMeter = metricsRegistry + .registerMeter("ui:num-all-topologies-summary-http-requests"); + this.topologyPageRequestMeter = metricsRegistry + .registerMeter("ui:num-topology-page-http-requests"); + this.topologyMetricRequestMeter = metricsRegistry + .registerMeter("ui:num-topology-metric-http-requests"); + this.buildVisualizationRequestMeter = metricsRegistry + .registerMeter("ui:num-build-visualization-http-requests"); + this.mkVisualizationDataRequestMeter = metricsRegistry + .registerMeter("ui:num-mk-visualization-data-http-requests"); + this.componentPageRequestMeter = metricsRegistry + .registerMeter("ui:num-component-page-http-requests"); + this.logConfigRequestMeter = metricsRegistry + .registerMeter("ui:num-log-config-http-requests"); + this.activateTopologyRequestMeter = metricsRegistry + .registerMeter("ui:num-activate-topology-http-requests"); + this.deactivateTopologyRequestMeter = metricsRegistry + .registerMeter("ui:num-deactivate-topology-http-requests"); + this.debugTopologyRequestMeter = metricsRegistry + .registerMeter("ui:num-debug-topology-http-requests"); + this.componentOpResponseRequestMeter = metricsRegistry + .registerMeter("ui:num-component-op-response-http-requests"); + this.topologyOpResponseMeter = metricsRegistry + .registerMeter("ui:num-topology-op-response-http-requests"); + this.topologyLagRequestMeter = metricsRegistry + .registerMeter("ui:num-topology-lag-http-requests"); + this.getOwnerResourceSummariesMeter = metricsRegistry + .registerMeter("ui:num-get-owner-resource-summaries-http-request"); } /** @@ -187,7 +206,8 @@ public Response getOwnerResource(@PathParam("id") String id, try (NimbusClient nimbusClient = NimbusClient.Builder.withConf(config).build()) { return UIHelpers.makeStandardResponse( UIHelpers.getOwnerResourceSummary( - nimbusClient.getClient().getOwnerResourceSummaries(id), nimbusClient.getClient(), + nimbusClient.getClient().getOwnerResourceSummaries(id), nimbusClient + .getClient(), id, config), callback ); @@ -337,7 +357,8 @@ public Response getTopologyMetrics(@PathParam("id") String id, try (NimbusClient nimbusClient = NimbusClient.Builder.withConf(config).build()) { return UIHelpers.makeStandardResponse( UIHelpers.getTopologySummary( - nimbusClient.getClient().getTopologyPageInfo(id, window, sys), window, config, user + nimbusClient.getClient().getTopologyPageInfo(id, window, + sys), window, config, user ), callback ); @@ -377,7 +398,8 @@ public Response getTopologyVisializationInit(@PathParam("id") String id, mkVisualizationDataRequestMeter.mark(); try (NimbusClient nimbusClient = NimbusClient.Builder.withConf(config).build()) { return UIHelpers.makeStandardResponse( - UIHelpers.getBuildVisualization(nimbusClient.getClient(), config, window, id, sys), + UIHelpers.getBuildVisualization(nimbusClient.getClient(), config, window, id, + sys), callback ); } @@ -465,7 +487,8 @@ public Response putTopologyLogconfig(@PathParam("id") String id, String body, try (NimbusClient nimbusClient = NimbusClient.Builder.withConf(config).build()) { return UIHelpers.makeStandardResponse( UIHelpers.putTopologyLogLevel(nimbusClient.getClient(), - ((Map) JSONValue.parse(body)).get("namedLoggerLevels"), id), + ((Map) JSONValue.parse(body)) + .get("namedLoggerLevels"), id), callback ); } @@ -608,7 +631,8 @@ public Response getTopologyProfilingStart(@PathParam("id") String id, @QueryParam(callbackParameterName) String callback) throws TException { try (NimbusClient nimbusClient = NimbusClient.Builder.withConf(config).build()) { return UIHelpers.makeStandardResponse( - UIHelpers.getTopologyProfilingStart(nimbusClient.getClient(), id, hostPort, timeout, config), + UIHelpers.getTopologyProfilingStart(nimbusClient.getClient(), id, hostPort, + timeout, config), callback ); } @@ -626,7 +650,8 @@ public Response getTopologyProfilingStop(@PathParam("id") String id, @QueryParam(callbackParameterName) String callback) throws TException { try (NimbusClient nimbusClient = NimbusClient.Builder.withConf(config).build()) { return UIHelpers.makeStandardResponse( - UIHelpers.getTopologyProfilingStop(nimbusClient.getClient(), id, hostPort, config), + UIHelpers.getTopologyProfilingStop(nimbusClient.getClient(), id, hostPort, + config), callback ); } @@ -644,7 +669,8 @@ public Response getTopologyProfilingDumpProfile(@PathParam("id") String id, @QueryParam(callbackParameterName) String callback) throws TException { try (NimbusClient nimbusClient = NimbusClient.Builder.withConf(config).build()) { return UIHelpers.makeStandardResponse( - UIHelpers.getTopologyProfilingDump(nimbusClient.getClient(), id, hostPort, config), + UIHelpers.getTopologyProfilingDump(nimbusClient.getClient(), id, hostPort, + config), callback ); } diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/utils/ListFunctionalSupport.java b/storm-webapp/src/main/java/org/apache/storm/daemon/utils/ListFunctionalSupport.java index ed6622bb920..50e4fc11641 100644 --- a/storm-webapp/src/main/java/org/apache/storm/daemon/utils/ListFunctionalSupport.java +++ b/storm-webapp/src/main/java/org/apache/storm/daemon/utils/ListFunctionalSupport.java @@ -41,7 +41,7 @@ public static T first(List list) { } /** - * get the last element in list. + * Get the last element in list. * * @param list list to get * @return the last element. null if list is null or empty. @@ -60,7 +60,8 @@ public static T last(List list) { * @param list list to get * @param count element count to get * @return the first element. null if list is null. - * elements in a new list may be less than count if there're not enough elements in the list. + * elements in a new list may be less than count if there're not enough elements in the + * list. */ public static List takeLast(List list, int count) { if (list == null) { @@ -82,7 +83,8 @@ public static List takeLast(List list, int count) { * * @param list the list * @param count element count to drop - * @return newly created sublist that drops the first N elements from origin list. null if list is null. + * @return newly created sublist that drops the first N elements from origin list. null if list + * is null. */ public static List drop(List list, int count) { if (list == null) { @@ -98,7 +100,8 @@ public static List drop(List list, int count) { * Drop the only first element and create a new list. equivalent to drop(list, 1). * * @param list the list - * @return newly created sublist that drops the first element from origin list. null if list is null. + * @return newly created sublist that drops the first element from origin list. null if list is + * null. */ public static List rest(List list) { return drop(list, 1); diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/utils/PathUtil.java b/storm-webapp/src/main/java/org/apache/storm/daemon/utils/PathUtil.java index 0aed50f67ff..d0e5760d7a4 100644 --- a/storm-webapp/src/main/java/org/apache/storm/daemon/utils/PathUtil.java +++ b/storm-webapp/src/main/java/org/apache/storm/daemon/utils/PathUtil.java @@ -25,6 +25,7 @@ public class PathUtil { /** * Truncates path to the last numElements. + * * @param path The path to truncate. * @param numElements The number of elements to preserve at the end of the path. * @return The truncated path. diff --git a/storm-webapp/src/main/java/org/apache/storm/daemon/utils/StreamUtil.java b/storm-webapp/src/main/java/org/apache/storm/daemon/utils/StreamUtil.java index adfd075c604..4ca7d7d598c 100644 --- a/storm-webapp/src/main/java/org/apache/storm/daemon/utils/StreamUtil.java +++ b/storm-webapp/src/main/java/org/apache/storm/daemon/utils/StreamUtil.java @@ -31,7 +31,8 @@ private StreamUtil() { /** * Skips over and discards N bytes of data from the input stream. *

      - * FileInputStream#skip may not work the first time, so ensure it successfully skips the given number of bytes. + * FileInputStream#skip may not work the first time, so ensure it successfully skips the given + * number of bytes. * * @param stream the stream to skip * @param n bytes to skip diff --git a/storm-webapp/src/test/java/org/apache/storm/daemon/drpc/DRPCServerTest.java b/storm-webapp/src/test/java/org/apache/storm/daemon/drpc/DRPCServerTest.java index 7e446d91f8c..d76e918dae8 100644 --- a/storm-webapp/src/test/java/org/apache/storm/daemon/drpc/DRPCServerTest.java +++ b/storm-webapp/src/test/java/org/apache/storm/daemon/drpc/DRPCServerTest.java @@ -22,8 +22,6 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.fail; -import org.junit.jupiter.api.AfterAll; - import java.io.InputStream; import java.net.URL; import java.util.HashMap; @@ -33,7 +31,6 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; - import org.apache.storm.Config; import org.apache.storm.DaemonConfig; import org.apache.storm.drpc.DRPCInvocationsClient; @@ -42,11 +39,11 @@ import org.apache.storm.metric.StormMetricsRegistry; import org.apache.storm.security.auth.SimpleTransportPlugin; import org.apache.storm.utils.DRPCClient; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - public class DRPCServerTest { private static final Logger LOG = LoggerFactory.getLogger(DRPCServerTest.class); private static final ExecutorService exec = Executors.newCachedThreadPool(); @@ -56,12 +53,14 @@ public static void close() { exec.shutdownNow(); } - private static DRPCRequest getNextAvailableRequest(DRPCInvocationsClient invoke, String func) throws Exception { + private static DRPCRequest getNextAvailableRequest(DRPCInvocationsClient invoke, + String func) throws Exception { DRPCRequest request = null; long timedout = System.currentTimeMillis() + 5_000; while (System.currentTimeMillis() < timedout) { request = invoke.getClient().fetchRequest(func); - if (request != null && request.get_request_id() != null && !request.get_request_id().isEmpty()) { + if (request != null && request.get_request_id() != null && !request.get_request_id() + .isEmpty()) { return request; } Thread.sleep(1); @@ -93,8 +92,10 @@ public void testGoodThrift() throws Exception { try (DRPCServer server = new DRPCServer(conf, new StormMetricsRegistry())) { server.start(); try (DRPCClient client = new DRPCClient(conf, "localhost", server.getDrpcPort()); - DRPCInvocationsClient invoke = new DRPCInvocationsClient(conf, "localhost", server.getDrpcInvokePort())) { - final Future found = exec.submit(() -> client.getClient().execute("testing", "test")); + DRPCInvocationsClient invoke = new DRPCInvocationsClient(conf, "localhost", server + .getDrpcInvokePort())) { + final Future found = exec.submit(() -> client.getClient().execute("testing", + "test")); DRPCRequest request = getNextAvailableRequest(invoke, "testing"); assertNotNull(request); assertEquals("test", request.get_func_args()); @@ -112,8 +113,10 @@ public void testFailedThrift() throws Exception { try (DRPCServer server = new DRPCServer(conf, new StormMetricsRegistry())) { server.start(); try (DRPCClient client = new DRPCClient(conf, "localhost", server.getDrpcPort()); - DRPCInvocationsClient invoke = new DRPCInvocationsClient(conf, "localhost", server.getDrpcInvokePort())) { - Future found = exec.submit(() -> client.getClient().execute("testing", "test")); + DRPCInvocationsClient invoke = new DRPCInvocationsClient(conf, "localhost", + server.getDrpcInvokePort())) { + Future found = exec.submit(() -> client.getClient().execute("testing", + "test")); DRPCRequest request = getNextAvailableRequest(invoke, "testing"); assertNotNull(request); assertEquals("test", request.get_func_args()); @@ -125,8 +128,8 @@ public void testFailedThrift() throws Exception { } catch (ExecutionException e) { Throwable t = e.getCause(); assertEquals(t.getClass(), DRPCExecutionException.class); - //Don't know a better way to validate that it failed. - assertEquals("Request failed", ((DRPCExecutionException)t).get_msg()); + // Don't know a better way to validate that it failed. + assertEquals("Request failed", ((DRPCExecutionException) t).get_msg()); } } } @@ -150,10 +153,12 @@ public void testGoodHttpGet() throws Exception { Map conf = getConf(0, 0, 0); try (DRPCServer server = new DRPCServer(conf, new StormMetricsRegistry())) { server.start(); - //TODO need a better way to do this + // TODO: need a better way to do this Thread.sleep(2000); - try (DRPCInvocationsClient invoke = new DRPCInvocationsClient(conf, "localhost", server.getDrpcInvokePort())) { - final Future found = exec.submit(() -> doGet(server.getHttpServerPort(), "testing", "test")); + try (DRPCInvocationsClient invoke = new DRPCInvocationsClient(conf, "localhost", server + .getDrpcInvokePort())) { + final Future found = exec.submit(() -> doGet(server.getHttpServerPort(), + "testing", "test")); DRPCRequest request = getNextAvailableRequest(invoke, "testing"); assertNotNull(request); assertEquals("test", request.get_func_args()); @@ -171,10 +176,12 @@ public void testFailedHttpGet() throws Exception { Map conf = getConf(0, 0, 0); try (DRPCServer server = new DRPCServer(conf, new StormMetricsRegistry())) { server.start(); - //TODO need a better way to do this + // TODO: need a better way to do this Thread.sleep(2000); - try (DRPCInvocationsClient invoke = new DRPCInvocationsClient(conf, "localhost", server.getDrpcInvokePort())) { - Future found = exec.submit(() -> doGet(server.getHttpServerPort(), "testing", "test")); + try (DRPCInvocationsClient invoke = new DRPCInvocationsClient(conf, "localhost", server + .getDrpcInvokePort())) { + Future found = exec.submit(() -> doGet(server.getHttpServerPort(), + "testing", "test")); DRPCRequest request = getNextAvailableRequest(invoke, "testing"); assertNotNull(request); assertEquals("test", request.get_func_args()); @@ -185,8 +192,8 @@ public void testFailedHttpGet() throws Exception { fail("exec did not throw an exception"); } catch (ExecutionException e) { LOG.warn("Got Expected Exception", e); - //Getting the exact response code is a bit more complex. - //TODO should use a better client + // Getting the exact response code is a bit more complex. + // TODO: should use a better client } } } diff --git a/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogDownloadHandlerTest.java b/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogDownloadHandlerTest.java index 0d40fce8db5..3deb5dbaf95 100644 --- a/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogDownloadHandlerTest.java +++ b/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogDownloadHandlerTest.java @@ -31,14 +31,14 @@ import static org.mockito.Mockito.when; import com.google.common.net.HttpHeaders; +import jakarta.ws.rs.core.Response; +import jakarta.ws.rs.core.StreamingOutput; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.Map; -import jakarta.ws.rs.core.Response; -import jakarta.ws.rs.core.StreamingOutput; import org.apache.storm.daemon.logviewer.utils.ResourceAuthorizer; import org.apache.storm.daemon.logviewer.utils.WorkerLogs; import org.apache.storm.metric.StormMetricsRegistry; @@ -52,20 +52,25 @@ public class LogviewerLogDownloadHandlerTest { public void testDownloadLogFile() throws IOException { try (TmpPath rootPath = new TmpPath()) { - LogviewerLogDownloadHandler handler = createHandlerTraversalTests(rootPath.getFile().toPath()); + LogviewerLogDownloadHandler handler = createHandlerTraversalTests(rootPath.getFile() + .toPath()); - Response topoAResponse = handler.downloadLogFile("host", "topoA/1111/worker.log", "user"); - Response topoBResponse = handler.downloadLogFile("host", "topoB/1111/worker.log", "user"); + Response topoAResponse = handler.downloadLogFile("host", "topoA/1111/worker.log", + "user"); + Response topoBResponse = handler.downloadLogFile("host", "topoB/1111/worker.log", + "user"); Utils.forceDelete(rootPath.toString()); assertThat(topoAResponse.getStatus(), is(Response.Status.OK.getStatusCode())); assertThat(topoAResponse.getEntity(), not(nullValue())); - String topoAContentDisposition = topoAResponse.getHeaderString(HttpHeaders.CONTENT_DISPOSITION); + String topoAContentDisposition = topoAResponse + .getHeaderString(HttpHeaders.CONTENT_DISPOSITION); assertThat(topoAContentDisposition, containsString("host-topoA-1111-worker.log")); assertThat(topoBResponse.getStatus(), is(Response.Status.OK.getStatusCode())); assertThat(topoBResponse.getEntity(), not(nullValue())); - String topoBContentDisposition = topoBResponse.getHeaderString(HttpHeaders.CONTENT_DISPOSITION); + String topoBContentDisposition = topoBResponse + .getHeaderString(HttpHeaders.CONTENT_DISPOSITION); assertThat(topoBContentDisposition, containsString("host-topoB-1111-worker.log")); } } @@ -74,9 +79,10 @@ public void testDownloadLogFile() throws IOException { public void testDownloadLogFileTraversal() throws IOException { try (TmpPath rootPath = new TmpPath()) { - LogviewerLogDownloadHandler handler = createHandlerTraversalTests(rootPath.getFile().toPath()); + LogviewerLogDownloadHandler handler = createHandlerTraversalTests(rootPath.getFile() + .toPath()); - Response topoAResponse = handler.downloadLogFile("host","../nimbus.log", "user"); + Response topoAResponse = handler.downloadLogFile("host", "../nimbus.log", "user"); Utils.forceDelete(rootPath.toString()); @@ -88,9 +94,10 @@ public void testDownloadLogFileTraversal() throws IOException { public void testDownloadDaemonLogFile() throws IOException { try (TmpPath rootPath = new TmpPath()) { - LogviewerLogDownloadHandler handler = createHandlerTraversalTests(rootPath.getFile().toPath()); + LogviewerLogDownloadHandler handler = createHandlerTraversalTests(rootPath.getFile() + .toPath()); - Response response = handler.downloadDaemonLogFile("host","nimbus.log", "user"); + Response response = handler.downloadDaemonLogFile("host", "nimbus.log", "user"); Utils.forceDelete(rootPath.toString()); @@ -105,9 +112,11 @@ public void testDownloadDaemonLogFile() throws IOException { public void testDownloadDaemonLogFilePathIntoWorkerLogs() throws IOException { try (TmpPath rootPath = new TmpPath()) { - LogviewerLogDownloadHandler handler = createHandlerTraversalTests(rootPath.getFile().toPath()); + LogviewerLogDownloadHandler handler = createHandlerTraversalTests(rootPath.getFile() + .toPath()); - Response response = handler.downloadDaemonLogFile("host","workers-artifacts/topoA/1111/worker.log", "user"); + Response response = handler.downloadDaemonLogFile("host", + "workers-artifacts/topoA/1111/worker.log", "user"); Utils.forceDelete(rootPath.toString()); @@ -119,9 +128,10 @@ public void testDownloadDaemonLogFilePathIntoWorkerLogs() throws IOException { public void testDownloadDaemonLogFilePathOutsideLogRoot() throws IOException { try (TmpPath rootPath = new TmpPath()) { - LogviewerLogDownloadHandler handler = createHandlerTraversalTests(rootPath.getFile().toPath()); + LogviewerLogDownloadHandler handler = createHandlerTraversalTests(rootPath.getFile() + .toPath()); - Response response = handler.downloadDaemonLogFile("host","../evil.sh", "user"); + Response response = handler.downloadDaemonLogFile("host", "../evil.sh", "user"); Utils.forceDelete(rootPath.toString()); @@ -139,11 +149,14 @@ public void testDownloadLogFileUnauthorizedUserDoesNotChangeLogFilePermission() Files.createFile(file); ResourceAuthorizer resourceAuthorizer = mock(ResourceAuthorizer.class); - when(resourceAuthorizer.isUserAllowedToAccessFile(anyString(), anyString())).thenReturn(false); + when(resourceAuthorizer.isUserAllowedToAccessFile(anyString(), anyString())) + .thenReturn(false); WorkerLogs workerLogs = mock(WorkerLogs.class); - LogviewerLogDownloadHandler handler = new LogviewerLogDownloadHandler(workerLogRoot.toString(), - daemonLogRoot.toString(), workerLogs, resourceAuthorizer, new StormMetricsRegistry()); + LogviewerLogDownloadHandler handler = new LogviewerLogDownloadHandler(workerLogRoot + .toString(), + daemonLogRoot + .toString(), workerLogs, resourceAuthorizer, new StormMetricsRegistry()); Response response = handler.downloadLogFile("host", "topoA/1111/worker.log", "user"); @@ -164,11 +177,14 @@ public void testDownloadLogFileAuthorizedUserSetsLogFilePermission() throws IOEx Files.createFile(file); ResourceAuthorizer resourceAuthorizer = mock(ResourceAuthorizer.class); - when(resourceAuthorizer.isUserAllowedToAccessFile(anyString(), anyString())).thenReturn(true); + when(resourceAuthorizer.isUserAllowedToAccessFile(anyString(), anyString())) + .thenReturn(true); WorkerLogs workerLogs = mock(WorkerLogs.class); - LogviewerLogDownloadHandler handler = new LogviewerLogDownloadHandler(workerLogRoot.toString(), - daemonLogRoot.toString(), workerLogs, resourceAuthorizer, new StormMetricsRegistry()); + LogviewerLogDownloadHandler handler = new LogviewerLogDownloadHandler(workerLogRoot + .toString(), + daemonLogRoot + .toString(), workerLogs, resourceAuthorizer, new StormMetricsRegistry()); Response response = handler.downloadLogFile("host", "topoA/1111/worker.log", "user"); @@ -190,8 +206,10 @@ public void testDownloadDaemonLogFileDoesNotChangeLogFilePermission() throws IOE WorkerLogs workerLogs = mock(WorkerLogs.class); - LogviewerLogDownloadHandler handler = new LogviewerLogDownloadHandler(workerLogRoot.toString(), - daemonLogRoot.toString(), workerLogs, new ResourceAuthorizer(Utils.readStormConfig()), new StormMetricsRegistry()); + LogviewerLogDownloadHandler handler = new LogviewerLogDownloadHandler(workerLogRoot + .toString(), + daemonLogRoot.toString(), workerLogs, new ResourceAuthorizer(Utils + .readStormConfig()), new StormMetricsRegistry()); Response response = handler.downloadDaemonLogFile("host", "nimbus.log", "user"); @@ -208,7 +226,8 @@ public void testDownloadDaemonLogFileUnauthorizedUser() throws IOException { ResourceAuthorizer resourceAuthorizer = mock(ResourceAuthorizer.class); when(resourceAuthorizer.isUserAllowedToAccessDaemonFile(anyString())).thenReturn(false); - LogviewerLogDownloadHandler handler = createHandlerTraversalTests(rootPath.getFile().toPath(), resourceAuthorizer); + LogviewerLogDownloadHandler handler = createHandlerTraversalTests(rootPath.getFile() + .toPath(), resourceAuthorizer); Response response = handler.downloadDaemonLogFile("host", "nimbus.log", "user"); @@ -224,13 +243,17 @@ public void testDownloadDaemonLogFileAuthorizedUser() throws IOException { ResourceAuthorizer resourceAuthorizer = mock(ResourceAuthorizer.class); when(resourceAuthorizer.isUserAllowedToAccessDaemonFile(anyString())).thenReturn(true); - LogviewerLogDownloadHandler handler = createHandlerTraversalTests(rootPath.getFile().toPath(), resourceAuthorizer); - //Give the daemon log some content, so that the response is only empty if the file was not served. - Files.writeString(rootPath.getFile().toPath().resolve("logs").resolve("nimbus.log"), "nimbus log content"); + LogviewerLogDownloadHandler handler = createHandlerTraversalTests(rootPath.getFile() + .toPath(), resourceAuthorizer); + // Give the daemon log some content, so that the response is only empty if the file was + // not served. + Files.writeString(rootPath.getFile().toPath().resolve("logs").resolve("nimbus.log"), + "nimbus log content"); Response response = handler.downloadDaemonLogFile("host", "nimbus.log", "user"); int status = response.getStatus(); - String content = status == Response.Status.OK.getStatusCode() ? readEntity(response) : null; + String content = status == Response.Status.OK.getStatusCode() + ? readEntity(response) : null; Utils.forceDelete(rootPath.toString()); @@ -249,10 +272,12 @@ private String readEntity(Response response) throws IOException { } private LogviewerLogDownloadHandler createHandlerTraversalTests(Path rootPath) throws IOException { - return createHandlerTraversalTests(rootPath, new ResourceAuthorizer(Utils.readStormConfig())); + return createHandlerTraversalTests(rootPath, new ResourceAuthorizer(Utils + .readStormConfig())); } - private LogviewerLogDownloadHandler createHandlerTraversalTests(Path rootPath, ResourceAuthorizer resourceAuthorizer) + private LogviewerLogDownloadHandler createHandlerTraversalTests(Path rootPath, + ResourceAuthorizer resourceAuthorizer) throws IOException { Path daemonLogRoot = rootPath.resolve("logs"); Path fileOutsideDaemonRoot = rootPath.resolve("evil.sh"); @@ -275,7 +300,8 @@ private LogviewerLogDownloadHandler createHandlerTraversalTests(Path rootPath, R Map stormConf = Utils.readStormConfig(); StormMetricsRegistry metricsRegistry = new StormMetricsRegistry(); return new LogviewerLogDownloadHandler(workerLogRoot.toString(), daemonLogRoot.toString(), - new WorkerLogs(stormConf, workerLogRoot, metricsRegistry), resourceAuthorizer, metricsRegistry); + new WorkerLogs(stormConf, workerLogRoot, + metricsRegistry), resourceAuthorizer, metricsRegistry); } } diff --git a/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogPageHandlerTest.java b/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogPageHandlerTest.java index 0d277cf8a0e..2a650349e9e 100644 --- a/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogPageHandlerTest.java +++ b/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogPageHandlerTest.java @@ -31,17 +31,14 @@ import static org.mockito.Mockito.when; import com.fasterxml.jackson.databind.ObjectMapper; - +import jakarta.ws.rs.core.Response; import java.io.File; import java.io.IOException; import java.nio.file.Files; -import java.nio.file.Paths; import java.nio.file.Path; +import java.nio.file.Paths; import java.util.List; import java.util.Map; - -import jakarta.ws.rs.core.Response; - import org.apache.storm.daemon.logviewer.utils.LogviewerResponseBuilder; import org.apache.storm.daemon.logviewer.utils.ResourceAuthorizer; import org.apache.storm.daemon.logviewer.utils.WorkerLogs; @@ -57,7 +54,8 @@ public class LogviewerLogPageHandlerTest { */ @Test public void testListLogFiles() throws IOException { - String rootPath = Files.createTempDirectory("workers-artifacts").toFile().getCanonicalPath(); + String rootPath = Files.createTempDirectory("workers-artifacts").toFile() + .getCanonicalPath(); File file1 = new File(String.join(File.separator, rootPath, "topoA", "1111"), "worker.log"); File file2 = new File(String.join(File.separator, rootPath, "topoA", "2222"), "worker.log"); File file3 = new File(String.join(File.separator, rootPath, "topoB", "1111"), "worker.log"); @@ -73,7 +71,8 @@ public void testListLogFiles() throws IOException { Map stormConf = Utils.readStormConfig(); StormMetricsRegistry metricsRegistry = new StormMetricsRegistry(); LogviewerLogPageHandler handler = new LogviewerLogPageHandler(rootPath, rootPath, - new WorkerLogs(stormConf, Paths.get(rootPath), metricsRegistry), new ResourceAuthorizer(stormConf), metricsRegistry); + new WorkerLogs(stormConf, Paths.get(rootPath), + metricsRegistry), new ResourceAuthorizer(stormConf), metricsRegistry); final Response expectedAll = LogviewerResponseBuilder.buildSuccessJsonResponse( List.of(String.join(File.separator, "topoA", "1111", "worker.log"), @@ -98,7 +97,8 @@ public void testListLogFiles() throws IOException { final Response returnedAll = handler.listLogFiles("user", null, null, null, origin); final Response returnedFilterPort = handler.listLogFiles("user", 1111, null, null, origin); - final Response returnedFilterTopoId = handler.listLogFiles("user", null, "topoB", null, origin); + final Response returnedFilterTopoId = handler.listLogFiles("user", null, "topoB", null, + origin); Utils.forceDelete(rootPath); @@ -112,9 +112,11 @@ public void testListLogFiles() throws IOException { */ @Test public void testListLogFilesFiltersFilesTheUserMayNotAccess() throws IOException { - String rootPath = Files.createTempDirectory("workers-artifacts").toFile().getCanonicalPath(); + String rootPath = Files.createTempDirectory("workers-artifacts").toFile() + .getCanonicalPath(); File file1 = new File(String.join(File.separator, rootPath, "topoA", "1111"), "worker.log"); - File file2 = new File(String.join(File.separator, rootPath, "topoA", "1111"), "worker.log.1"); + File file2 = new File(String.join(File.separator, rootPath, "topoA", "1111"), + "worker.log.1"); File file3 = new File(String.join(File.separator, rootPath, "topoB", "1111"), "worker.log"); file1.getParentFile().mkdirs(); @@ -128,9 +130,11 @@ public void testListLogFilesFiltersFilesTheUserMayNotAccess() throws IOException Map stormConf = Utils.readStormConfig(); StormMetricsRegistry metricsRegistry = new StormMetricsRegistry(); ResourceAuthorizer resourceAuthorizer = mock(ResourceAuthorizer.class); - when(resourceAuthorizer.isUserAllowedToAccessFile(anyString(), startsWith(topoAPortDir))).thenReturn(true); + when(resourceAuthorizer.isUserAllowedToAccessFile(anyString(), startsWith(topoAPortDir))) + .thenReturn(true); LogviewerLogPageHandler handler = new LogviewerLogPageHandler(rootPath, rootPath, - new WorkerLogs(stormConf, Paths.get(rootPath), metricsRegistry), resourceAuthorizer, metricsRegistry); + new WorkerLogs(stormConf, Paths.get(rootPath), + metricsRegistry), resourceAuthorizer, metricsRegistry); final Response returned = handler.listLogFiles("user", null, null, null, origin); @@ -140,11 +144,13 @@ public void testListLogFilesFiltersFilesTheUserMayNotAccess() throws IOException assertEquals(List.of(String.join(File.separator, topoAPortDir, "worker.log"), String.join(File.separator, topoAPortDir, "worker.log.1")), files); - //The authorization only depends on the port directory, so it is checked once per port directory, not once per file. + // The authorization only depends on the port directory, so it is checked once per port + // directory, not once per file. verify(resourceAuthorizer, times(2)).isUserAllowedToAccessFile(anyString(), anyString()); } - private void assertEqualsJsonResponse(Response expected, Response actual, Class entityClass) throws IOException { + private void assertEqualsJsonResponse(Response expected, Response actual, + Class entityClass) throws IOException { ObjectMapper objectMapper = new ObjectMapper(); T entityFromExpected = objectMapper.readValue((String) expected.getEntity(), entityClass); T entityFromActual = objectMapper.readValue((String) actual.getEntity(), entityClass); @@ -158,9 +164,11 @@ private void assertEqualsJsonResponse(Response expected, Response actual, Cl public void testListLogFilesOutsideLogRoot() throws IOException { try (TmpPath rootPath = new TmpPath()) { String origin = "www.origin.server.net"; - LogviewerLogPageHandler handler = createHandlerForTraversalTests(rootPath.getFile().toPath()); + LogviewerLogPageHandler handler = createHandlerForTraversalTests(rootPath.getFile() + .toPath()); - //The response should be empty, since you should not be able to list files outside the worker log root. + // The response should be empty, since you should not be able to list files outside the + // worker log root. final Response expected = LogviewerResponseBuilder.buildSuccessJsonResponse( List.of(), null, @@ -176,13 +184,14 @@ public void testListLogFilesOutsideLogRoot() throws IOException { @Test public void testLogPageOutsideLogRoot() throws Exception { try (TmpPath rootPath = new TmpPath()) { - LogviewerLogPageHandler handler = createHandlerForTraversalTests(rootPath.getFile().toPath()); + LogviewerLogPageHandler handler = createHandlerForTraversalTests(rootPath.getFile() + .toPath()); final Response returned = handler.logPage("../nimbus.log", 0, 100, null, "user"); Utils.forceDelete(rootPath.toString()); - //Should not show files outside worker log root. + // Should not show files outside worker log root. assertThat(returned.getStatus(), is(Response.Status.NOT_FOUND.getStatusCode())); } } @@ -190,13 +199,14 @@ public void testLogPageOutsideLogRoot() throws Exception { @Test public void testDaemonLogPageOutsideLogRoot() throws Exception { try (TmpPath rootPath = new TmpPath()) { - LogviewerLogPageHandler handler = createHandlerForTraversalTests(rootPath.getFile().toPath()); + LogviewerLogPageHandler handler = createHandlerForTraversalTests(rootPath.getFile() + .toPath()); final Response returned = handler.daemonLogPage("../evil.sh", 0, 100, null, "user"); Utils.forceDelete(rootPath.toString()); - //Should not show files outside daemon log root. + // Should not show files outside daemon log root. assertThat(returned.getStatus(), is(Response.Status.NOT_FOUND.getStatusCode())); } } @@ -204,13 +214,15 @@ public void testDaemonLogPageOutsideLogRoot() throws Exception { @Test public void testDaemonLogPagePathIntoWorkerLogs() throws Exception { try (TmpPath rootPath = new TmpPath()) { - LogviewerLogPageHandler handler = createHandlerForTraversalTests(rootPath.getFile().toPath()); + LogviewerLogPageHandler handler = createHandlerForTraversalTests(rootPath.getFile() + .toPath()); - final Response returned = handler.daemonLogPage("workers-artifacts/topoA/worker.log", 0, 100, null, "user"); + final Response returned = handler.daemonLogPage("workers-artifacts/topoA/worker.log", 0, + 100, null, "user"); Utils.forceDelete(rootPath.toString()); - //Should not show files outside log root. + // Should not show files outside log root. assertThat(returned.getStatus(), is(Response.Status.NOT_FOUND.getStatusCode())); } } @@ -220,9 +232,12 @@ public void testDaemonLogPageUnauthorizedUser() throws Exception { try (TmpPath rootPath = new TmpPath()) { ResourceAuthorizer resourceAuthorizer = mock(ResourceAuthorizer.class); when(resourceAuthorizer.isUserAllowedToAccessDaemonFile(anyString())).thenReturn(false); - LogviewerLogPageHandler handler = createHandlerForTraversalTests(rootPath.getFile().toPath(), resourceAuthorizer); - //Give the daemon log some content, so that an unauthorized request is the only reason not to render the page. - Files.writeString(rootPath.getFile().toPath().resolve("logs").resolve("nimbus.log"), "nimbus log content"); + LogviewerLogPageHandler handler = createHandlerForTraversalTests(rootPath.getFile() + .toPath(), resourceAuthorizer); + // Give the daemon log some content, so that an unauthorized request is the only reason + // not to render the page. + Files.writeString(rootPath.getFile().toPath().resolve("logs").resolve("nimbus.log"), + "nimbus log content"); final Response returned = handler.daemonLogPage("nimbus.log", 0, 100, null, "user"); @@ -237,8 +252,10 @@ public void testDaemonLogPageAuthorizedUser() throws Exception { try (TmpPath rootPath = new TmpPath()) { ResourceAuthorizer resourceAuthorizer = mock(ResourceAuthorizer.class); when(resourceAuthorizer.isUserAllowedToAccessDaemonFile(anyString())).thenReturn(true); - LogviewerLogPageHandler handler = createHandlerForTraversalTests(rootPath.getFile().toPath(), resourceAuthorizer); - Files.writeString(rootPath.getFile().toPath().resolve("logs").resolve("nimbus.log"), "nimbus log content"); + LogviewerLogPageHandler handler = createHandlerForTraversalTests(rootPath.getFile() + .toPath(), resourceAuthorizer); + Files.writeString(rootPath.getFile().toPath().resolve("logs").resolve("nimbus.log"), + "nimbus log content"); final Response returned = handler.daemonLogPage("nimbus.log", 0, 100, null, "user"); @@ -251,10 +268,12 @@ public void testDaemonLogPageAuthorizedUser() throws Exception { } private LogviewerLogPageHandler createHandlerForTraversalTests(Path rootPath) throws IOException { - return createHandlerForTraversalTests(rootPath, new ResourceAuthorizer(Utils.readStormConfig())); + return createHandlerForTraversalTests(rootPath, new ResourceAuthorizer(Utils + .readStormConfig())); } - private LogviewerLogPageHandler createHandlerForTraversalTests(Path rootPath, ResourceAuthorizer resourceAuthorizer) + private LogviewerLogPageHandler createHandlerForTraversalTests(Path rootPath, + ResourceAuthorizer resourceAuthorizer) throws IOException { Path daemonLogRoot = rootPath.resolve("logs"); Path fileOutsideDaemonRoot = rootPath.resolve("evil.sh"); @@ -277,6 +296,7 @@ private LogviewerLogPageHandler createHandlerForTraversalTests(Path rootPath, Re Map stormConf = Utils.readStormConfig(); StormMetricsRegistry metricsRegistry = new StormMetricsRegistry(); return new LogviewerLogPageHandler(workerLogRoot.toString(), daemonLogRoot.toString(), - new WorkerLogs(stormConf, workerLogRoot, metricsRegistry), resourceAuthorizer, metricsRegistry); + new WorkerLogs(stormConf, workerLogRoot, + metricsRegistry), resourceAuthorizer, metricsRegistry); } } diff --git a/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogSearchHandlerTest.java b/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogSearchHandlerTest.java index 90c15c861bb..a0d1db8b427 100644 --- a/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogSearchHandlerTest.java +++ b/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/handler/LogviewerLogSearchHandlerTest.java @@ -35,7 +35,7 @@ import static org.mockito.internal.verification.VerificationModeFactory.times; import com.fasterxml.jackson.databind.ObjectMapper; - +import jakarta.ws.rs.core.Response; import java.io.File; import java.io.IOException; import java.net.UnknownHostException; @@ -48,9 +48,6 @@ import java.util.List; import java.util.Map; import java.util.function.Function; - -import jakarta.ws.rs.core.Response; - import org.apache.storm.DaemonConfig; import org.apache.storm.daemon.logviewer.LogviewerConstant; import org.apache.storm.daemon.logviewer.utils.ResourceAuthorizer; @@ -76,10 +73,12 @@ public class SearchViaRestApi { private final String logviewerUrlPrefix = "http://" + expectedHost + ":" + expectedPort; /* - * When we click a link to the logviewer, we expect the match line to be somewhere near the middle of the page. So we subtract half + * When we click a link to the logviewer, we expect the match line to be somewhere near the + * middle of the page. So we subtract half * of the default page length from the offset at which we found the match. */ - private final Function expOffsetFn = arg -> (LogviewerConstant.DEFAULT_BYTES_PER_PAGE / 2 - arg); + private final Function expOffsetFn = + arg -> (LogviewerConstant.DEFAULT_BYTES_PER_PAGE / 2 - arg); @Test public void testSearchViaRestApiThrowsIfBogusFileIsGiven() throws InvalidRequestException { @@ -99,7 +98,8 @@ public void testLogviewerLinkCentersTheMatchInThePage() throws UnknownHostExcept when(mockedUtil.hostname()).thenReturn(expectedHost); - String actualUrl = handler.urlToMatchCenteredInLogPage(new byte[42], new File(expectedFname).toPath(), 27526, 8888); + String actualUrl = handler.urlToMatchCenteredInLogPage(new byte[42], + new File(expectedFname).toPath(), 27526, 8888); assertEquals("http://" + expectedHost + ":" + expectedPort + "/api/v1/log?file=" + expectedFname + "&start=1947&length=" + LogviewerConstant.DEFAULT_BYTES_PER_PAGE, actualUrl); @@ -120,7 +120,8 @@ public void testLogviewerLinkCentersTheMatchInThePageDaemon() throws UnknownHost when(mockedUtil.hostname()).thenReturn(expectedHost); - String actualUrl = handler.urlToMatchCenteredInLogPageDaemonFile(new byte[42], new File(expectedFname).toPath(), 27526, 8888); + String actualUrl = handler.urlToMatchCenteredInLogPageDaemonFile(new byte[42], + new File(expectedFname).toPath(), 27526, 8888); assertEquals("http://" + expectedHost + ":" + expectedPort + "/api/v1/daemonlog?file=" + expectedFname + "&start=1947&length=" + LogviewerConstant.DEFAULT_BYTES_PER_PAGE, actualUrl); @@ -212,7 +213,8 @@ public void testAreallySmallLogFile() throws Exception { matches.add(buildMatchData(7, "000000 ", " 000000\n", pattern, - "/api/v1/log?file=test" + encodedFileSeparator() + "resources" + encodedFileSeparator() + file.getName() + "/api/v1/log?file=test" + encodedFileSeparator() + "resources" + + encodedFileSeparator() + file.getName() + "&start=0&length=51200" )); @@ -255,7 +257,8 @@ public void testAreallySmallLogDaemonFile() throws InvalidRequestException, Unkn expected.put("matches", matches); LogviewerLogSearchHandler handler = getSearchHandlerWithPort(expectedPort); - Map searchResult = handler.substringSearchDaemonLog(file.toPath(), pattern); + Map searchResult = handler.substringSearchDaemonLog(file.toPath(), + pattern); assertEquals(expected, searchResult); } finally { @@ -286,7 +289,8 @@ public void testNoOffsetReturnedWhenFileEndsOnBufferOffset() throws Exception { ".".repeat(128), "", pattern, - "/api/v1/log?file=test" + encodedFileSeparator() + "resources" + encodedFileSeparator() + file.getName() + "/api/v1/log?file=test" + encodedFileSeparator() + "resources" + + encodedFileSeparator() + file.getName() + "&start=0&length=51200" )); @@ -294,7 +298,8 @@ public void testNoOffsetReturnedWhenFileEndsOnBufferOffset() throws Exception { LogviewerLogSearchHandler handler = getSearchHandlerWithPort(expectedPort); Map searchResult = handler.substringSearch(file.toPath(), pattern); - Map searchResult2 = handler.substringSearch(file.toPath(), pattern, 1); + Map searchResult2 = handler.substringSearch(file.toPath(), pattern, + 1); assertEquals(expected, searchResult); assertEquals(expected, searchResult2); @@ -582,7 +587,8 @@ public void testReturnsZeroMatchesForUnseenPattern() throws UnknownHostException } } - private Map buildMatchData(int byteOffset, String beforeString, String afterString, + private Map buildMatchData(int byteOffset, String beforeString, + String afterString, String matchString, String logviewerUrlPath) { Map match = new HashMap<>(); match.put("byteOffset", byteOffset); @@ -613,15 +619,20 @@ public void testFindNMatches() { final LogviewerLogSearchHandler handler = getSearchHandler(); - final List> matches1 = handler.findNMatches(files, 20, 0, 0, "needle").getMatches(); - final List> matches2 = handler.findNMatches(files, 20, 0, 126, "needle").getMatches(); - final List> matches3 = handler.findNMatches(files, 20, 1, 0, "needle").getMatches(); + final List> matches1 = handler.findNMatches(files, 20, 0, 0, + "needle").getMatches(); + final List> matches2 = handler.findNMatches(files, 20, 0, 126, + "needle").getMatches(); + final List> matches3 = handler.findNMatches(files, 20, 1, 0, + "needle").getMatches(); assertEquals(2, matches1.size()); assertEquals(4, ((List) matches1.get(0).get("matches")).size()); assertEquals(4, ((List) matches1.get(1).get("matches")).size()); - assertEquals(String.join(File.separator, "test", "resources", "logviewer-search-context-tests.log.test"), matches1.get(0).get("fileName")); - assertEquals(String.join(File.separator, "test", "resources", "logviewer-search-context-tests.log.gz"), matches1.get(1).get("fileName")); + assertEquals(String.join(File.separator, "test", "resources", + "logviewer-search-context-tests.log.test"), matches1.get(0).get("fileName")); + assertEquals(String.join(File.separator, "test", "resources", + "logviewer-search-context-tests.log.gz"), matches1.get(1).get("fileName")); assertEquals(2, ((List) matches2.get(0).get("matches")).size()); assertEquals(4, ((List) matches2.get(1).get("matches")).size()); @@ -671,7 +682,8 @@ public void tearDown() { public void testAllPortsAndSearchArchivedIsTrue() throws IOException { LogviewerLogSearchHandler handler = getStubbedSearchHandler(); - handler.deepSearchLogsForTopology("", null, "search", "20", "*", "20", "199", true, null, null); + handler.deepSearchLogsForTopology("", null, "search", "20", "*", "20", "199", true, + null, null); ArgumentCaptor files = ArgumentCaptor.forClass(List.class); ArgumentCaptor numMatches = ArgumentCaptor.forClass(Integer.class); @@ -679,11 +691,13 @@ public void testAllPortsAndSearchArchivedIsTrue() throws IOException { ArgumentCaptor offset = ArgumentCaptor.forClass(Integer.class); ArgumentCaptor search = ArgumentCaptor.forClass(String.class); - verify(handler, times(4)).findNMatches(files.capture(), numMatches.capture(), fileOffset.capture(), + verify(handler, times(4)).findNMatches(files.capture(), numMatches.capture(), fileOffset + .capture(), offset.capture(), search.capture()); verify(handler, times(4)).logsForPort(isNull(), any()); - // File offset and byte offset should always be zero when searching multiple workers (multiple ports). + // File offset and byte offset should always be zero when searching multiple workers + // (multiple ports). assertEquals(logFiles, files.getAllValues().get(0)); assertEquals(Integer.valueOf(20), numMatches.getAllValues().get(0)); assertEquals(Integer.valueOf(0), fileOffset.getAllValues().get(0)); @@ -713,7 +727,8 @@ public void testAllPortsAndSearchArchivedIsTrue() throws IOException { public void testAllPortsAndSearchArchivedIsFalse() throws IOException { LogviewerLogSearchHandler handler = getStubbedSearchHandler(); - handler.deepSearchLogsForTopology("", null, "search", "20", null, "20", "199", false, null, null); + handler.deepSearchLogsForTopology("", null, "search", "20", null, "20", "199", false, + null, null); ArgumentCaptor files = ArgumentCaptor.forClass(List.class); ArgumentCaptor numMatches = ArgumentCaptor.forClass(Integer.class); @@ -721,11 +736,13 @@ public void testAllPortsAndSearchArchivedIsFalse() throws IOException { ArgumentCaptor offset = ArgumentCaptor.forClass(Integer.class); ArgumentCaptor search = ArgumentCaptor.forClass(String.class); - verify(handler, times(4)).findNMatches(files.capture(), numMatches.capture(), fileOffset.capture(), + verify(handler, times(4)).findNMatches(files.capture(), numMatches.capture(), fileOffset + .capture(), offset.capture(), search.capture()); verify(handler, times(4)).logsForPort(isNull(), any()); - // File offset and byte offset should always be zero when searching multiple workers (multiple ports). + // File offset and byte offset should always be zero when searching multiple workers + // (multiple ports). assertEquals(Collections.singletonList(logFiles.get(0)), files.getAllValues().get(0)); assertEquals(Integer.valueOf(20), numMatches.getAllValues().get(0)); assertEquals(Integer.valueOf(0), fileOffset.getAllValues().get(0)); @@ -755,7 +772,8 @@ public void testAllPortsAndSearchArchivedIsFalse() throws IOException { public void testOnePortAndSearchArchivedIsTrueAndNotFileOffset() throws IOException { LogviewerLogSearchHandler handler = getStubbedSearchHandler(); - handler.deepSearchLogsForTopology("", null, "search", "20", "6700", "0", "0", true, null, null); + handler.deepSearchLogsForTopology("", null, "search", "20", "6700", "0", "0", true, + null, null); ArgumentCaptor files = ArgumentCaptor.forClass(List.class); ArgumentCaptor numMatches = ArgumentCaptor.forClass(Integer.class); @@ -763,7 +781,8 @@ public void testOnePortAndSearchArchivedIsTrueAndNotFileOffset() throws IOExcept ArgumentCaptor offset = ArgumentCaptor.forClass(Integer.class); ArgumentCaptor search = ArgumentCaptor.forClass(String.class); - verify(handler, times(1)).findNMatches(files.capture(), numMatches.capture(), fileOffset.capture(), + verify(handler, times(1)).findNMatches(files.capture(), numMatches.capture(), fileOffset + .capture(), offset.capture(), search.capture()); verify(handler).logsForPort(isNull(), any()); @@ -778,7 +797,8 @@ public void testOnePortAndSearchArchivedIsTrueAndNotFileOffset() throws IOExcept public void testOnePortAndSearchArchivedIsTrueAndFileOffsetIs1() throws IOException { LogviewerLogSearchHandler handler = getStubbedSearchHandler(); - handler.deepSearchLogsForTopology("", null, "search", "20", "6700", "1", "0", true, null, null); + handler.deepSearchLogsForTopology("", null, "search", "20", "6700", "1", "0", true, + null, null); ArgumentCaptor files = ArgumentCaptor.forClass(List.class); ArgumentCaptor numMatches = ArgumentCaptor.forClass(Integer.class); @@ -786,7 +806,8 @@ public void testOnePortAndSearchArchivedIsTrueAndFileOffsetIs1() throws IOExcept ArgumentCaptor offset = ArgumentCaptor.forClass(Integer.class); ArgumentCaptor search = ArgumentCaptor.forClass(String.class); - verify(handler, times(1)).findNMatches(files.capture(), numMatches.capture(), fileOffset.capture(), + verify(handler, times(1)).findNMatches(files.capture(), numMatches.capture(), fileOffset + .capture(), offset.capture(), search.capture()); verify(handler).logsForPort(isNull(), any()); @@ -801,7 +822,8 @@ public void testOnePortAndSearchArchivedIsTrueAndFileOffsetIs1() throws IOExcept public void testOnePortAndSearchArchivedIsFalseAndFileOffsetIs1() throws IOException { LogviewerLogSearchHandler handler = getStubbedSearchHandler(); - handler.deepSearchLogsForTopology("", null, "search", "20", "6700", "1", "0", false, null, null); + handler.deepSearchLogsForTopology("", null, "search", "20", "6700", "1", "0", false, + null, null); ArgumentCaptor files = ArgumentCaptor.forClass(List.class); ArgumentCaptor numMatches = ArgumentCaptor.forClass(Integer.class); @@ -809,7 +831,8 @@ public void testOnePortAndSearchArchivedIsFalseAndFileOffsetIs1() throws IOExcep ArgumentCaptor offset = ArgumentCaptor.forClass(Integer.class); ArgumentCaptor search = ArgumentCaptor.forClass(String.class); - verify(handler, times(1)).findNMatches(files.capture(), numMatches.capture(), fileOffset.capture(), + verify(handler, times(1)).findNMatches(files.capture(), numMatches.capture(), fileOffset + .capture(), offset.capture(), search.capture()); verify(handler).logsForPort(isNull(), any()); @@ -825,9 +848,11 @@ public void testOnePortAndSearchArchivedIsFalseAndFileOffsetIs1() throws IOExcep public void testOnePortAndSearchArchivedIsTrueAndFileOffsetIs1AndByteOffsetIs100() throws IOException { LogviewerLogSearchHandler handler = getStubbedSearchHandler(); - handler.deepSearchLogsForTopology("", null, "search", "20", "6700", "1", "100", true, null, null); + handler.deepSearchLogsForTopology("", null, "search", "20", "6700", "1", "100", true, + null, null); - verify(handler, times(1)).findNMatches(anyList(), anyInt(), anyInt(), anyInt(), anyString()); + verify(handler, times(1)).findNMatches(anyList(), anyInt(), anyInt(), anyInt(), + anyString()); verify(handler, times(1)).logsForPort(isNull(), any()); } @@ -835,7 +860,8 @@ public void testOnePortAndSearchArchivedIsTrueAndFileOffsetIs1AndByteOffsetIs100 public void testBadPortAndSearchArchivedIsFalseAndFileOffsetIs1() throws IOException { LogviewerLogSearchHandler handler = getStubbedSearchHandler(); - handler.deepSearchLogsForTopology("", null, "search", "20", "2700", "1", "0", false, null, null); + handler.deepSearchLogsForTopology("", null, "search", "20", "2700", "1", "0", false, + null, null); ArgumentCaptor files = ArgumentCaptor.forClass(List.class); ArgumentCaptor numMatches = ArgumentCaptor.forClass(Integer.class); @@ -844,14 +870,16 @@ public void testBadPortAndSearchArchivedIsFalseAndFileOffsetIs1() throws IOExcep ArgumentCaptor search = ArgumentCaptor.forClass(String.class); // Called with a bad port (not in the config) No searching should be done. - verify(handler, never()).findNMatches(files.capture(), numMatches.capture(), fileOffset.capture(), + verify(handler, never()).findNMatches(files.capture(), numMatches.capture(), fileOffset + .capture(), offset.capture(), search.capture()); verify(handler, never()).logsForPort(anyString(), any()); } private LogviewerLogSearchHandler getStubbedSearchHandler() { Map stormConf = Utils.readStormConfig(); - LogviewerLogSearchHandler handler = new LogviewerLogSearchHandler(stormConf, topoPath, Paths.get(""), + LogviewerLogSearchHandler handler = new LogviewerLogSearchHandler(stormConf, topoPath, + Paths.get(""), new ResourceAuthorizer(stormConf), new StormMetricsRegistry()); handler = spy(handler); @@ -861,7 +889,8 @@ private LogviewerLogSearchHandler getStubbedSearchHandler() { int fileOffset = (Integer) arguments[2]; String search = (String) arguments[4]; - return new LogviewerLogSearchHandler.Matched(fileOffset, search, Collections.emptyList(), METRIC_SCANNED_FILES); + return new LogviewerLogSearchHandler.Matched(fileOffset, search, Collections + .emptyList(), METRIC_SCANNED_FILES); }).when(handler).findNMatches(any(), anyInt(), anyInt(), anyInt(), any()); return handler; @@ -878,10 +907,12 @@ public void testSearchDaemonLogFileUnauthorizedUser() throws Exception { Map stormConf = Utils.readStormConfig(); ResourceAuthorizer resourceAuthorizer = mock(ResourceAuthorizer.class); when(resourceAuthorizer.isUserAllowedToAccessDaemonFile(anyString())).thenReturn(false); - LogviewerLogSearchHandler handler = new LogviewerLogSearchHandler(stormConf, Paths.get(""), daemonLogRoot, + LogviewerLogSearchHandler handler = new LogviewerLogSearchHandler(stormConf, Paths + .get(""), daemonLogRoot, resourceAuthorizer, new StormMetricsRegistry()); - Response response = handler.searchLogFile("nimbus.log", "user", true, "needle", null, null, null, null); + Response response = handler.searchLogFile("nimbus.log", "user", true, "needle", null, + null, null, null); assertEquals(403, response.getStatus()); } @@ -897,16 +928,20 @@ public void testSearchDaemonLogFileAuthorizedUser() throws Exception { Map stormConf = Utils.readStormConfig(); ResourceAuthorizer resourceAuthorizer = mock(ResourceAuthorizer.class); when(resourceAuthorizer.isUserAllowedToAccessDaemonFile(anyString())).thenReturn(true); - LogviewerLogSearchHandler handler = new LogviewerLogSearchHandler(stormConf, Paths.get(""), daemonLogRoot, + LogviewerLogSearchHandler handler = new LogviewerLogSearchHandler(stormConf, Paths + .get(""), daemonLogRoot, resourceAuthorizer, new StormMetricsRegistry()); - Response response = handler.searchLogFile("nimbus.log", "user", true, "needle", null, null, null, null); + Response response = handler.searchLogFile("nimbus.log", "user", true, "needle", null, + null, null, null); assertEquals(200, response.getStatus()); - Map entity = new ObjectMapper().readValue((String) response.getEntity(), Map.class); + Map entity = new ObjectMapper().readValue((String) response.getEntity(), + Map.class); assertEquals("needle", entity.get("searchString")); assertEquals("yes", entity.get("isDaemon")); - //A match must actually be reported, an empty match list would mean the file was never read. + // A match must actually be reported, an empty match list would mean the file was never + // read. List matches = (List) entity.get("matches"); assertEquals(1, matches.size()); assertEquals("needle", ((Map) matches.get(0)).get("matchString")); diff --git a/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/handler/LogviewerProfileHandlerTest.java b/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/handler/LogviewerProfileHandlerTest.java index 136390175e7..a22b8af24fc 100644 --- a/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/handler/LogviewerProfileHandlerTest.java +++ b/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/handler/LogviewerProfileHandlerTest.java @@ -26,12 +26,12 @@ import static org.hamcrest.MatcherAssert.assertThat; import com.google.common.net.HttpHeaders; +import jakarta.ws.rs.core.Response; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.Map; -import jakarta.ws.rs.core.Response; import org.apache.storm.daemon.logviewer.utils.ResourceAuthorizer; import org.apache.storm.metric.StormMetricsRegistry; import org.apache.storm.testing.TmpPath; @@ -44,7 +44,8 @@ public class LogviewerProfileHandlerTest { public void testListDumpFiles() throws Exception { try (TmpPath rootPath = new TmpPath()) { - LogviewerProfileHandler handler = createHandlerTraversalTests(rootPath.getFile().toPath()); + LogviewerProfileHandler handler = createHandlerTraversalTests(rootPath.getFile() + .toPath()); Response topoAResponse = handler.listDumpFiles("topoA", "localhost:1111", "user"); Response topoBResponse = handler.listDumpFiles("topoB", "localhost:1111", "user"); @@ -67,7 +68,8 @@ public void testListDumpFiles() throws Exception { public void testListDumpFilesTraversalInTopoId() throws Exception { try (TmpPath rootPath = new TmpPath()) { - LogviewerProfileHandler handler = createHandlerTraversalTests(rootPath.getFile().toPath()); + LogviewerProfileHandler handler = createHandlerTraversalTests(rootPath.getFile() + .toPath()); Response response = handler.listDumpFiles("../../", "localhost:logs", "user"); @@ -81,7 +83,8 @@ public void testListDumpFilesTraversalInTopoId() throws Exception { public void testListDumpFilesTraversalInPort() throws Exception { try (TmpPath rootPath = new TmpPath()) { - LogviewerProfileHandler handler = createHandlerTraversalTests(rootPath.getFile().toPath()); + LogviewerProfileHandler handler = createHandlerTraversalTests(rootPath.getFile() + .toPath()); Response response = handler.listDumpFiles("../", "localhost:../logs", "user"); @@ -95,20 +98,25 @@ public void testListDumpFilesTraversalInPort() throws Exception { public void testDownloadDumpFile() throws IOException { try (TmpPath rootPath = new TmpPath()) { - LogviewerProfileHandler handler = createHandlerTraversalTests(rootPath.getFile().toPath()); + LogviewerProfileHandler handler = createHandlerTraversalTests(rootPath.getFile() + .toPath()); - Response topoAResponse = handler.downloadDumpFile("topoA", "localhost:1111", "worker.jfr", "user"); - Response topoBResponse = handler.downloadDumpFile("topoB", "localhost:1111", "worker.txt", "user"); + Response topoAResponse = handler.downloadDumpFile("topoA", "localhost:1111", + "worker.jfr", "user"); + Response topoBResponse = handler.downloadDumpFile("topoB", "localhost:1111", + "worker.txt", "user"); Utils.forceDelete(rootPath.toString()); assertThat(topoAResponse.getStatus(), is(Response.Status.OK.getStatusCode())); assertThat(topoAResponse.getEntity(), not(nullValue())); - String topoAContentDisposition = topoAResponse.getHeaderString(HttpHeaders.CONTENT_DISPOSITION); + String topoAContentDisposition = topoAResponse + .getHeaderString(HttpHeaders.CONTENT_DISPOSITION); assertThat(topoAContentDisposition, containsString("localhost-topoA-1111-worker.jfr")); assertThat(topoBResponse.getStatus(), is(Response.Status.OK.getStatusCode())); assertThat(topoBResponse.getEntity(), not(nullValue())); - String topoBContentDisposition = topoBResponse.getHeaderString(HttpHeaders.CONTENT_DISPOSITION); + String topoBContentDisposition = topoBResponse + .getHeaderString(HttpHeaders.CONTENT_DISPOSITION); assertThat(topoBContentDisposition, containsString("localhost-topoB-1111-worker.txt")); } } @@ -117,9 +125,11 @@ public void testDownloadDumpFile() throws IOException { public void testDownloadDumpFileTraversalInTopoId() throws IOException { try (TmpPath rootPath = new TmpPath()) { - LogviewerProfileHandler handler = createHandlerTraversalTests(rootPath.getFile().toPath()); + LogviewerProfileHandler handler = createHandlerTraversalTests(rootPath.getFile() + .toPath()); - Response topoAResponse = handler.downloadDumpFile("../../", "localhost:logs", "daemon-dump.bin", "user"); + Response topoAResponse = handler.downloadDumpFile("../../", "localhost:logs", + "daemon-dump.bin", "user"); Utils.forceDelete(rootPath.toString()); @@ -131,9 +141,11 @@ public void testDownloadDumpFileTraversalInTopoId() throws IOException { public void testDownloadDumpFileTraversalInPort() throws IOException { try (TmpPath rootPath = new TmpPath()) { - LogviewerProfileHandler handler = createHandlerTraversalTests(rootPath.getFile().toPath()); + LogviewerProfileHandler handler = createHandlerTraversalTests(rootPath.getFile() + .toPath()); - Response topoAResponse = handler.downloadDumpFile("../", "localhost:../logs", "daemon-dump.bin", "user"); + Response topoAResponse = handler.downloadDumpFile("../", "localhost:../logs", + "daemon-dump.bin", "user"); Utils.forceDelete(rootPath.toString()); @@ -162,7 +174,8 @@ private LogviewerProfileHandler createHandlerTraversalTests(Path rootPath) throw Map stormConf = Utils.readStormConfig(); StormMetricsRegistry metricsRegistry = new StormMetricsRegistry(); - return new LogviewerProfileHandler(workerLogRoot.toString(), new ResourceAuthorizer(stormConf), metricsRegistry); + return new LogviewerProfileHandler(workerLogRoot.toString(), + new ResourceAuthorizer(stormConf), metricsRegistry); } } diff --git a/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/testsupport/ArgumentsVerifier.java b/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/testsupport/ArgumentsVerifier.java index 0f50238c6e1..7e9075cf1bb 100644 --- a/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/testsupport/ArgumentsVerifier.java +++ b/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/testsupport/ArgumentsVerifier.java @@ -30,7 +30,8 @@ public class ArgumentsVerifier { /** * Asserting that method is called with expected first argument. * - * @param verifyConsumer Consumer implementation that takes ArgumentCaptor and call 'Mockito.verify' + * @param verifyConsumer Consumer implementation that takes ArgumentCaptor and call + * 'Mockito.verify' * @param argClazz Class type for argument * @param expectedArg expected argument */ diff --git a/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/testsupport/MockRemovableFileBuilder.java b/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/testsupport/MockRemovableFileBuilder.java index 31204ce5b93..68427f58aa3 100644 --- a/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/testsupport/MockRemovableFileBuilder.java +++ b/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/testsupport/MockRemovableFileBuilder.java @@ -1,22 +1,25 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. - * See the NOTICE file distributed with this work for additional information regarding copyright ownership. + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. + * See the NOTICE file distributed with this work for additional information regarding copyright + * ownership. * The ASF licenses this file to you under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

      - * http://www.apache.org/licenses/LICENSE-2.0 - *

      - * Unless required by applicable law or agreed to in writing, + * you may not use this file except in compliance with the License. You may obtain a copy of the + * License at + * + *

      http://www.apache.org/licenses/LICENSE-2.0 + * + *

      Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and limitations under the License. + * See the License for the specific language governing permissions and limitations under the + * License. */ package org.apache.storm.daemon.logviewer.testsupport; -import org.mockito.Mockito; - import java.io.File; +import org.mockito.Mockito; public class MockRemovableFileBuilder extends MockFileBuilder { @Override diff --git a/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/utils/LogCleanerTest.java b/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/utils/LogCleanerTest.java index 004b99a164e..9e47695716f 100644 --- a/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/utils/LogCleanerTest.java +++ b/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/utils/LogCleanerTest.java @@ -44,10 +44,8 @@ import java.util.Set; import java.util.SortedSet; import java.util.TreeSet; - import java.util.function.Predicate; import java.util.stream.IntStream; - import org.apache.storm.daemon.supervisor.SupervisorUtils; import org.apache.storm.generated.LSWorkerHeartbeat; import org.apache.storm.metric.StormMetricsRegistry; @@ -73,7 +71,8 @@ public void testMkFileFilterForLogCleanup() throws IOException { StormMetricsRegistry metricRegistry = new StormMetricsRegistry(); WorkerLogs workerLogs = new WorkerLogs(conf, Paths.get(""), metricRegistry); - LogCleaner logCleaner = new LogCleaner(conf, workerLogs, new DirectoryCleaner(metricRegistry), null, metricRegistry); + LogCleaner logCleaner = new LogCleaner(conf, workerLogs, + new DirectoryCleaner(metricRegistry), null, metricRegistry); final long nowMillis = Time.currentTimeMillis(); final long cutoffMillis = logCleaner.cleanupCutoffAgeMillis(nowMillis); @@ -99,8 +98,10 @@ public void testMkFileFilterForLogCleanup() throws IOException { Predicate fileFilter = logCleaner.mkFileFilterForLogCleanup(nowMillis); - matchingFiles.forEach(p -> assertTrue(fileFilter.test(p), "Missing " + p.getFileName())); - excludedFiles.forEach(p -> assertFalse(fileFilter.test(p), "Not excluded " + p.getFileName())); + matchingFiles.forEach(p -> assertTrue(fileFilter.test(p), "Missing " + p + .getFileName())); + excludedFiles.forEach(p -> assertFalse(fileFilter.test(p), "Not excluded " + p + .getFileName())); } } @@ -138,7 +139,7 @@ private Path createDir(Path parentDir, String name, long lastModifiedMs) { } /** - * cleaner deletes oldest files in each worker dir if files are larger than per-dir quota. + * Cleaner deletes oldest files in each worker dir if files are larger than per-dir quota. */ @Test public void testPerWorkerDirectoryCleanup() throws IOException { @@ -163,7 +164,8 @@ public void testPerWorkerDirectoryCleanup() throws IOException { Map conf = Utils.readStormConfig(); StormMetricsRegistry metricRegistry = new StormMetricsRegistry(); WorkerLogs workerLogs = new WorkerLogs(conf, rootDir, metricRegistry); - LogCleaner logCleaner = new LogCleaner(conf, workerLogs, new DirectoryCleaner(metricRegistry), rootDir, metricRegistry); + LogCleaner logCleaner = new LogCleaner(conf, workerLogs, + new DirectoryCleaner(metricRegistry), rootDir, metricRegistry); List deletedFiles = logCleaner.perWorkerDirCleanup(1200) .stream() @@ -190,7 +192,8 @@ public void testGlobalLogCleanup() throws Exception { Path port3Dir = createDir(topo2Dir, "port3"); IntStream.range(0, 10) - .forEach(idx -> createFile(port1Dir, "A" + idx + ".log", nowMillis + 100L * idx, 200)); + .forEach(idx -> createFile(port1Dir, "A" + idx + ".log", nowMillis + 100L * idx, + 200)); IntStream.range(0, 10) .forEach(idx -> createFile(port2Dir, "B" + idx, nowMillis + 100L * idx, 200)); IntStream.range(0, 10) @@ -205,14 +208,15 @@ public SortedSet getAliveWorkerDirs() { } }; - LogCleaner logCleaner = new LogCleaner(conf, stubbedWorkerLogs, new DirectoryCleaner(metricRegistry), rootDir, metricRegistry); + LogCleaner logCleaner = new LogCleaner(conf, stubbedWorkerLogs, + new DirectoryCleaner(metricRegistry), rootDir, metricRegistry); int deletedFiles = logCleaner.globalLogCleanup(2400).deletedFiles; assertEquals(18, deletedFiles); } } /** - * return directories for workers that are not alive. + * Return directories for workers that are not alive. */ @Test public void testGetDeadWorkerDirs() throws Exception { @@ -252,17 +256,19 @@ public SortedSet getLogDirs(Set logDirs, Predicate predicate } }; - LogCleaner logCleaner = new LogCleaner(conf, stubbedWorkerLogs, new DirectoryCleaner(metricRegistry), null, metricRegistry); + LogCleaner logCleaner = new LogCleaner(conf, stubbedWorkerLogs, + new DirectoryCleaner(metricRegistry), null, metricRegistry); when(mockedSupervisorUtils.readWorkerHeartbeatsImpl(anyMap())).thenReturn(idToHb); - assertEquals(Sets.newSet(expectedDir2, expectedDir3), logCleaner.getDeadWorkerDirs(nowSecs, logDirs)); + assertEquals(Sets.newSet(expectedDir2, expectedDir3), logCleaner + .getDeadWorkerDirs(nowSecs, logDirs)); } finally { SupervisorUtils.resetInstance(); } } /** - * cleanup function forceDeletes files of dead workers. + * Cleanup function forceDeletes files of dead workers. */ @Test public void testCleanupFn() throws IOException { @@ -274,7 +280,8 @@ public void testCleanupFn() throws IOException { StormMetricsRegistry metricRegistry = new StormMetricsRegistry(); WorkerLogs stubbedWorkerLogs = new WorkerLogs(conf, Paths.get(""), metricRegistry); - LogCleaner logCleaner = new LogCleaner(conf, stubbedWorkerLogs, new DirectoryCleaner(metricRegistry), null, metricRegistry) { + LogCleaner logCleaner = new LogCleaner(conf, stubbedWorkerLogs, + new DirectoryCleaner(metricRegistry), null, metricRegistry) { @Override Set selectDirsForCleanup(long nowMillis) { return Collections.emptySet(); diff --git a/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/utils/LogviewerResponseBuilderTest.java b/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/utils/LogviewerResponseBuilderTest.java index 4a06c3fc261..10735cccd2c 100644 --- a/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/utils/LogviewerResponseBuilderTest.java +++ b/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/utils/LogviewerResponseBuilderTest.java @@ -23,9 +23,7 @@ import static org.hamcrest.Matchers.nullValue; import jakarta.ws.rs.core.Response; - import java.util.Collections; - import org.junit.jupiter.api.Test; public class LogviewerResponseBuilderTest { diff --git a/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/utils/ResourceAuthorizerTest.java b/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/utils/ResourceAuthorizerTest.java index 96407bca1ac..b6ed240dc99 100644 --- a/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/utils/ResourceAuthorizerTest.java +++ b/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/utils/ResourceAuthorizerTest.java @@ -35,7 +35,6 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; - import org.apache.storm.DaemonConfig; import org.apache.storm.daemon.logviewer.testsupport.ArgumentsVerifier; import org.apache.storm.utils.Utils; @@ -46,7 +45,7 @@ public class ResourceAuthorizerTest { /** - * allow cluster admin. + * Allow cluster admin. */ @Test public void testAuthorizedLogUserAllowClusterAdmin() { @@ -57,7 +56,8 @@ public void testAuthorizedLogUserAllowClusterAdmin() { ResourceAuthorizer authorizer = spy(new ResourceAuthorizer(conf)); - doReturn(new ResourceAuthorizer.LogUserGroupWhitelist(Collections.emptySet(), Collections.emptySet())) + doReturn(new ResourceAuthorizer.LogUserGroupWhitelist(Collections.emptySet(), Collections + .emptySet())) .when(authorizer).getLogUserGroupWhitelist(anyString()); doReturn(Collections.emptySet()).when(authorizer).getUserGroups(anyString()); @@ -68,7 +68,7 @@ public void testAuthorizedLogUserAllowClusterAdmin() { } /** - * ignore any cluster-set topology.users topology.groups. + * Ignore any cluster-set topology.users topology.groups. */ @Test public void testAuthorizedLogUserIgnoreAnyClusterSetTopologyUsersAndTopologyGroups() { @@ -80,7 +80,8 @@ public void testAuthorizedLogUserIgnoreAnyClusterSetTopologyUsersAndTopologyGrou ResourceAuthorizer authorizer = spy(new ResourceAuthorizer(conf)); - doReturn(new ResourceAuthorizer.LogUserGroupWhitelist(Collections.emptySet(), Collections.emptySet())) + doReturn(new ResourceAuthorizer.LogUserGroupWhitelist(Collections.emptySet(), Collections + .emptySet())) .when(authorizer).getLogUserGroupWhitelist(anyString()); doReturn(Collections.singleton("alice-group")).when(authorizer).getUserGroups(anyString()); @@ -91,7 +92,7 @@ public void testAuthorizedLogUserIgnoreAnyClusterSetTopologyUsersAndTopologyGrou } /** - * allow cluster logs user. + * Allow cluster logs user. */ @Test public void testAuthorizedLogUserAllowClusterLogsUser() { @@ -102,7 +103,8 @@ public void testAuthorizedLogUserAllowClusterLogsUser() { ResourceAuthorizer authorizer = spy(new ResourceAuthorizer(conf)); - doReturn(new ResourceAuthorizer.LogUserGroupWhitelist(Collections.emptySet(), Collections.emptySet())) + doReturn(new ResourceAuthorizer.LogUserGroupWhitelist(Collections.emptySet(), Collections + .emptySet())) .when(authorizer).getLogUserGroupWhitelist(anyString()); doReturn(Collections.emptySet()).when(authorizer).getUserGroups(anyString()); @@ -113,7 +115,7 @@ public void testAuthorizedLogUserAllowClusterLogsUser() { } /** - * allow whitelisted topology user. + * Allow whitelisted topology user. */ @Test public void testAuthorizedLogUserAllowWhitelistedTopologyUser() { @@ -123,7 +125,8 @@ public void testAuthorizedLogUserAllowWhitelistedTopologyUser() { ResourceAuthorizer authorizer = spy(new ResourceAuthorizer(conf)); - doReturn(new ResourceAuthorizer.LogUserGroupWhitelist(Collections.singleton("alice"), Collections.emptySet())) + doReturn(new ResourceAuthorizer.LogUserGroupWhitelist(Collections.singleton("alice"), + Collections.emptySet())) .when(authorizer).getLogUserGroupWhitelist(anyString()); doReturn(Collections.emptySet()).when(authorizer).getUserGroups(anyString()); @@ -134,7 +137,7 @@ public void testAuthorizedLogUserAllowWhitelistedTopologyUser() { } /** - * allow whitelisted topology group. + * Allow whitelisted topology group. */ @Test public void testAuthorizedLogUserAllowWhitelistedTopologyGroup() { @@ -144,7 +147,8 @@ public void testAuthorizedLogUserAllowWhitelistedTopologyGroup() { ResourceAuthorizer authorizer = spy(new ResourceAuthorizer(conf)); - doReturn(new ResourceAuthorizer.LogUserGroupWhitelist(Collections.emptySet(), Collections.singleton("alice-group"))) + doReturn(new ResourceAuthorizer.LogUserGroupWhitelist(Collections.emptySet(), Collections + .singleton("alice-group"))) .when(authorizer).getLogUserGroupWhitelist(anyString()); doReturn(Collections.singleton("alice-group")).when(authorizer).getUserGroups(anyString()); @@ -155,7 +159,7 @@ public void testAuthorizedLogUserAllowWhitelistedTopologyGroup() { } /** - * disallow user not in nimbus admin, topo user, logs user, or whitelist. + * Disallow user not in nimbus admin, topo user, logs user, or whitelist. */ @Test public void testAuthorizedLogUserDisallowUserNotInNimbusAdminNorTopoUserNorLogsUserNotWhitelist() { @@ -165,7 +169,8 @@ public void testAuthorizedLogUserDisallowUserNotInNimbusAdminNorTopoUserNorLogsU ResourceAuthorizer authorizer = spy(new ResourceAuthorizer(conf)); - doReturn(new ResourceAuthorizer.LogUserGroupWhitelist(Collections.emptySet(), Collections.emptySet())) + doReturn(new ResourceAuthorizer.LogUserGroupWhitelist(Collections.emptySet(), Collections + .emptySet())) .when(authorizer).getLogUserGroupWhitelist(anyString()); doReturn(Collections.emptySet()).when(authorizer).getUserGroups(anyString()); @@ -176,7 +181,7 @@ public void testAuthorizedLogUserDisallowUserNotInNimbusAdminNorTopoUserNorLogsU } /** - * disallow upward path traversal in filenames. + * Disallow upward path traversal in filenames. */ @Test public void testFailOnUpwardPathTraversal() { @@ -215,7 +220,7 @@ public void authorizationFailsWhenFilterConfigured() { } /** - * daemon logs are allowed for cluster logs users and cluster admins only. + * Daemon logs are allowed for cluster logs users and cluster admins only. */ @Test public void testAuthorizedDaemonLogUserAllowsClusterLogsUserAndClusterAdmin() { @@ -235,7 +240,7 @@ public void testAuthorizedDaemonLogUserAllowsClusterLogsUserAndClusterAdmin() { } /** - * daemon logs are allowed for a user whose groups are listed in logs.groups. + * Daemon logs are allowed for a user whose groups are listed in logs.groups. */ @Test public void testAuthorizedDaemonLogUserAllowsClusterLogsGroup() { @@ -248,12 +253,13 @@ public void testAuthorizedDaemonLogUserAllowsClusterLogsGroup() { doReturn(Collections.singleton("alice-group")).when(authorizer).getUserGroups(anyString()); - //alice is not named in logs.users nor in nimbus.admins, she is authorized purely by her group membership + // alice is not named in logs.users nor in nimbus.admins, she is authorized purely by her + // group membership assertTrue(authorizer.isAuthorizedDaemonLogUser("alice")); } /** - * daemon logs are allowed for a user whose groups are listed in nimbus.admins.groups. + * Daemon logs are allowed for a user whose groups are listed in nimbus.admins.groups. */ @Test public void testAuthorizedDaemonLogUserAllowsClusterAdminGroup() { @@ -266,12 +272,14 @@ public void testAuthorizedDaemonLogUserAllowsClusterAdminGroup() { doReturn(Collections.singleton("admin-group")).when(authorizer).getUserGroups(anyString()); - //alice is not named in logs.users nor in nimbus.admins, she is authorized purely by her group membership + // alice is not named in logs.users nor in nimbus.admins, she is authorized purely by her + // group membership assertTrue(authorizer.isAuthorizedDaemonLogUser("alice")); } /** - * daemon logs are denied for a user whose groups match neither logs.groups nor nimbus.admins.groups. + * Daemon logs are denied for a user whose groups match neither logs.groups nor + * nimbus.admins.groups. */ @Test public void testAuthorizedDaemonLogUserDisallowsUnrelatedGroup() { @@ -283,13 +291,14 @@ public void testAuthorizedDaemonLogUserDisallowsUnrelatedGroup() { ResourceAuthorizer authorizer = spy(new ResourceAuthorizer(conf)); - doReturn(Collections.singleton("mallory-group")).when(authorizer).getUserGroups(anyString()); + doReturn(Collections.singleton("mallory-group")).when(authorizer) + .getUserGroups(anyString()); assertFalse(authorizer.isAuthorizedDaemonLogUser("mallory")); } /** - * daemon log access via the UI filter is granted by group membership alone. + * Daemon log access via the UI filter is granted by group membership alone. */ @Test public void testUserAllowedToAccessDaemonFileByGroupWhenFilterConfigured() { @@ -307,7 +316,7 @@ public void testUserAllowedToAccessDaemonFileByGroupWhenFilterConfigured() { } /** - * daemon log access consults the cluster level lists once a filter is configured. + * Daemon log access consults the cluster level lists once a filter is configured. */ @Test public void daemonLogAuthorizationFailsWhenFilterConfigured() { @@ -319,7 +328,8 @@ public void daemonLogAuthorizationFailsWhenFilterConfigured() { doReturn(Collections.emptySet()).when(authorizer).getUserGroups(anyString()); - assertTrue(authorizer.isUserAllowedToAccessDaemonFile("bob")); // no filter configured, allow anyone + assertTrue(authorizer + .isUserAllowedToAccessDaemonFile("bob")); // no filter configured, allow anyone conf.put(DaemonConfig.LOGVIEWER_FILTER, "someFilter"); assertTrue(authorizer.isUserAllowedToAccessDaemonFile("alice")); diff --git a/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/utils/WorkerLogsTest.java b/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/utils/WorkerLogsTest.java index 6a5ae7dcc72..3886ae07643 100644 --- a/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/utils/WorkerLogsTest.java +++ b/storm-webapp/src/test/java/org/apache/storm/daemon/logviewer/utils/WorkerLogsTest.java @@ -29,7 +29,6 @@ import java.util.Collections; import java.util.Map; import java.util.Optional; - import java.util.SortedSet; import java.util.TreeSet; import org.apache.storm.daemon.supervisor.SupervisorUtils; @@ -46,7 +45,8 @@ public class WorkerLogsTest { @Test public void testIdentifyWorkerLogDirs() throws Exception { try (TmpPath testDir = new TmpPath()) { - Path port1Dir = Files.createDirectories(testDir.getFile().toPath().resolve("workers-artifacts/topo1/port1")); + Path port1Dir = Files.createDirectories(testDir.getFile().toPath() + .resolve("workers-artifacts/topo1/port1")); Path metaFile = Files.createFile(testDir.getFile().toPath().resolve("worker.yaml")); String expId = "id12345"; @@ -56,7 +56,8 @@ public void testIdentifyWorkerLogDirs() throws Exception { SupervisorUtils.setInstance(mockedSupervisorUtils); Map stormConf = Utils.readStormConfig(); - WorkerLogs workerLogs = new WorkerLogs(stormConf, port1Dir, new StormMetricsRegistry()) { + WorkerLogs workerLogs = new WorkerLogs(stormConf, port1Dir, + new StormMetricsRegistry()) { @Override public Optional getMetadataFileForWorkerLogDir(Path logDir) { return Optional.of(metaFile); @@ -69,7 +70,8 @@ public String getWorkerIdFromMetadataFile(Path metaFile) { }; when(mockedSupervisorUtils.readWorkerHeartbeatsImpl(anyMap())).thenReturn(null); - assertEquals(expected, workerLogs.getLogDirs(Collections.singleton(port1Dir), (wid) -> true)); + assertEquals(expected, workerLogs.getLogDirs(Collections.singleton(port1Dir), + (wid) -> true)); } finally { SupervisorUtils.resetInstance(); } diff --git a/storm-webapp/src/test/java/org/apache/storm/daemon/ui/UIHelpersTest.java b/storm-webapp/src/test/java/org/apache/storm/daemon/ui/UIHelpersTest.java index 09a31f24403..1bd095d087a 100644 --- a/storm-webapp/src/test/java/org/apache/storm/daemon/ui/UIHelpersTest.java +++ b/storm-webapp/src/test/java/org/apache/storm/daemon/ui/UIHelpersTest.java @@ -18,6 +18,21 @@ package org.apache.storm.daemon.ui; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import net.minidev.json.JSONValue; import org.apache.storm.Config; import org.apache.storm.Constants; import org.apache.storm.generated.BoltAggregateStats; @@ -29,7 +44,6 @@ import org.apache.storm.generated.TopologyPageInfo; import org.apache.storm.generated.TopologyStats; import org.apache.storm.utils.Time; -import net.minidev.json.JSONValue; import org.eclipse.jetty.ee10.servlet.FilterHolder; import org.eclipse.jetty.ee10.servlets.CrossOriginFilter; import org.eclipse.jetty.server.Server; @@ -41,21 +55,6 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Arrays; -import java.util.HashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; - class UIHelpersTest { private static final String TOPOLOGY_ID = "Test-Topology-Id"; private static final long TOPOLOGY_MESSAGE_TIMEOUT_SECS = 100L; @@ -75,7 +74,7 @@ class UIHelpersTest { * Setups up bare minimum TopologyPageInfo instance such that we can pass to * UIHelpers.getTopologySummary() without it throwing a NPE. * - * This should provide a base for which other tests can be written, but will + *

      This should provide a base for which other tests can be written, but will * require populating additional values as needed for each test case. */ @BeforeEach @@ -94,9 +93,9 @@ void setup() { topologyStats.set_window_to_failed(new HashMap<>()); // Create empty AggregateStats instances. - final Map idToSpoutAggStats = new HashMap<>(); + final Map idToSpoutAggStats = new HashMap<>(); - final Map idToBoltAggStats = new HashMap<>(); + final Map idToBoltAggStats = new HashMap<>(); // Build up TopologyPageInfo instance topoPageInfo = new TopologyPageInfo(); @@ -158,7 +157,8 @@ void test_getTopologyBoltAggStatsMap_includesLastError() { assertNotNull(result, "Should never return null"); // Validate our Bolt result - final Map boltResult = getBoltStatsFromTopologySummaryResult(result, expectedBoltId); + final Map boltResult = getBoltStatsFromTopologySummaryResult(result, + expectedBoltId); assertNotNull(boltResult, "Should have an entry for bolt"); // Verify each piece @@ -198,7 +198,8 @@ void test_getTopologyBoltAggStatsMap_hasNoLastError() { assertNotNull(result, "Should never return null"); // Validate our Bolt result - final Map boltResult = getBoltStatsFromTopologySummaryResult(result, expectedBoltId); + final Map boltResult = getBoltStatsFromTopologySummaryResult(result, + expectedBoltId); assertNotNull(boltResult, "Should have an entry for bolt"); // Verify each piece @@ -208,7 +209,8 @@ void test_getTopologyBoltAggStatsMap_hasNoLastError() { // Verify error fields exist, but are not populated. // These fields default to empty string. assertTrue(boltResult.containsKey("lastError")); - assertEquals("", boltResult.get("lastError"), "Backwards compat. with API docs say this should be empty string when empty"); + assertEquals("", boltResult.get("lastError"), + "Backwards compat. with API docs say this should be empty string when empty"); assertTrue(boltResult.containsKey("errorHost")); assertEquals("", boltResult.get("errorHost")); assertTrue(boltResult.containsKey("errorWorkerLogLink")); @@ -231,7 +233,7 @@ void test_getTopologyBoltAggStatsMap_hasNoLastError() { void test_getTopologyBoltAggStatsMap_generalFields() { // Define inputs final String expectedBoltId = "MyBoltId"; - final float expectedCapacity = 0.97f; + final float expectedCapacity = 0.97F; final double expectedProcessLatency = 432.0D; final double expectedExecuteLatency = 122.0D; final long expectedExecuted = 153343L; @@ -286,15 +288,18 @@ void test_getTopologyBoltAggStatsMap_generalFields() { assertNotNull(result, "Should never return null"); // Validate our Bolt result - final Map boltResult = getBoltStatsFromTopologySummaryResult(result, expectedBoltId); + final Map boltResult = getBoltStatsFromTopologySummaryResult(result, + expectedBoltId); assertNotNull(boltResult, "Should have an entry for bolt"); // Validate fields assertEquals(expectedBoltId, boltResult.get("boltId")); assertEquals(expectedBoltId, boltResult.get("encodedBoltId")); assertEquals(expectedTransferred, boltResult.get("transferred")); - assertEquals(String.format("%.3f", expectedExecuteLatency), boltResult.get("executeLatency")); - assertEquals(String.format("%.3f", expectedProcessLatency), boltResult.get("processLatency")); + assertEquals(String.format("%.3f", expectedExecuteLatency), boltResult + .get("executeLatency")); + assertEquals(String.format("%.3f", expectedProcessLatency), boltResult + .get("processLatency")); assertEquals(expectedExecuted, boltResult.get("executed")); assertEquals(expectedFailed, boltResult.get("failed")); assertEquals(expectedAcked, boltResult.get("acked")); @@ -310,7 +315,8 @@ void test_getTopologyBoltAggStatsMap_generalFields() { assertEquals("", boltResult.get("requestedGenericResourcesComp")); // We expect there to be no error populated. - assertEquals("", boltResult.get("lastError"), "No error should be reported as empty string"); + assertEquals("", boltResult.get("lastError"), + "No error should be reported as empty string"); } /** @@ -353,7 +359,8 @@ void test_getTopologySpoutAggStatsMap_includesLastError() { assertNotNull(result, "Should never return null"); // Validate our Spout result - final Map spoutResult = getSpoutStatsFromTopologySummaryResult(result, expectedSpoutId); + final Map spoutResult = getSpoutStatsFromTopologySummaryResult(result, + expectedSpoutId); assertNotNull(spoutResult, "Should have an entry for spout"); // Verify each piece @@ -393,7 +400,8 @@ void test_getTopologySpoutAggStatsMap_hasNoLastError() { assertNotNull(result, "Should never return null"); // Validate our Spout result - final Map spoutResult = getSpoutStatsFromTopologySummaryResult(result, expectedSpoutId); + final Map spoutResult = getSpoutStatsFromTopologySummaryResult(result, + expectedSpoutId); assertNotNull(spoutResult, "Should have an entry for spout"); // Verify each piece @@ -403,7 +411,8 @@ void test_getTopologySpoutAggStatsMap_hasNoLastError() { // Verify error fields exist, but are not populated. // These fields default to empty string. assertTrue(spoutResult.containsKey("lastError")); - assertEquals("", spoutResult.get("lastError"), "Backwards compat. with API docs say this should be empty string when empty"); + assertEquals("", spoutResult.get("lastError"), + "Backwards compat. with API docs say this should be empty string when empty"); assertTrue(spoutResult.containsKey("errorHost")); assertEquals("", spoutResult.get("errorHost")); assertTrue(spoutResult.containsKey("errorWorkerLogLink")); @@ -475,14 +484,16 @@ void test_getTopologySpoutAggStatsMap_generalFields() { assertNotNull(result, "Should never return null"); // Validate our Spout result - final Map spoutResult = getSpoutStatsFromTopologySummaryResult(result, expectedSpoutId); + final Map spoutResult = getSpoutStatsFromTopologySummaryResult(result, + expectedSpoutId); assertNotNull(spoutResult, "Should have an entry for spout"); // Validate fields assertEquals(expectedSpoutId, spoutResult.get("spoutId")); assertEquals(expectedSpoutId, spoutResult.get("encodedSpoutId")); assertEquals(expectedTransferred, spoutResult.get("transferred")); - assertEquals(String.format("%.3f", expectedCompleteLatency), spoutResult.get("completeLatency")); + assertEquals(String.format("%.3f", expectedCompleteLatency), spoutResult + .get("completeLatency")); assertEquals(expectedFailed, spoutResult.get("failed")); assertEquals(expectedAcked, spoutResult.get("acked")); assertEquals(expectedEmitted, spoutResult.get("emitted")); @@ -496,15 +507,17 @@ void test_getTopologySpoutAggStatsMap_generalFields() { assertEquals("", spoutResult.get("requestedGenericResourcesComp")); // We expect there to be no error populated. - assertEquals("", spoutResult.get("lastError"), "No error should be reported as empty string"); + assertEquals("", spoutResult.get("lastError"), + "No error should be reported as empty string"); } /** - * Tests that santizeStreamName() does the expected manipulations + * Tests that santizeStreamName() does the expected manipulations. */ @Test public void testSanitizeStreamName() { - // replaces the expected characters with underscores (everything except A-Z, a-z, dot, dash, and underscore) + // replaces the expected characters with underscores (everything except A-Z, a-z, dot, dash, + // and underscore) assertEquals("my-stream_with.all_characterClasses____", UIHelpers.sanitizeStreamName("my-stream:with.all_characterClasses1/\\2")); @@ -517,6 +530,7 @@ public void testSanitizeStreamName() { /** * Add an AggregateStats entry to the TopologyPageInfo instance. + * * @param boltId Id of the bolt to add the entry for. * @param aggregateStats Defines the entry. */ @@ -526,6 +540,7 @@ private void addBoltStats(final String boltId, final ComponentAggregateStats agg /** * Add an AggregateStats entry to the TopologyPageInfo instance. + * * @param spoutId Id of the spout to add the entry for. * @param aggregateStats Defines the entry. */ @@ -535,6 +550,7 @@ private void addSpoutStats(final String spoutId, final ComponentAggregateStats a /** * Builds an empty ComponentAggregateStats instance for bolts. + * * @return empty ComponentAggregateStats instance. */ private ComponentAggregateStats buildBoltAggregateStatsBase() { @@ -553,6 +569,7 @@ private ComponentAggregateStats buildBoltAggregateStatsBase() { /** * Builds an empty ComponentAggregateStats instance for spouts. + * * @return empty ComponentAggregateStats instance. */ private ComponentAggregateStats buildSpoutAggregateStatsBase() { @@ -578,14 +595,17 @@ private ComponentAggregateStats buildSpoutAggregateStatsBase() { * @return Map for the given boltId. * @throws IllegalArgumentException if passed an invalid BoltId. */ - private Map getBoltStatsFromTopologySummaryResult(final Map result, final String boltId) { + private Map getBoltStatsFromTopologySummaryResult(final Map result, final String boltId) { assertNotNull(result.get("bolts"), "Should have non-null 'bolts' property"); - final List> bolts = (List>) result.get("bolts"); + final List> bolts = (List>) result + .get("bolts"); return bolts.stream() .filter((entry) -> boltId.equals(entry.get("boltId"))) .findFirst() - .orElseThrow(() -> new IllegalArgumentException("Unable to find entry for boltId '" + boltId + "'")); + .orElseThrow(() -> new IllegalArgumentException("Unable to find entry for boltId '" + + boltId + "'")); } /** @@ -597,14 +617,17 @@ private Map getBoltStatsFromTopologySummaryResult(final Map getSpoutStatsFromTopologySummaryResult(final Map result, final String spoutId) { + private Map getSpoutStatsFromTopologySummaryResult(final Map result, final String spoutId) { assertNotNull(result.get("spouts"), "Should have non-null 'spouts' property"); - final List> bolts = (List>) result.get("spouts"); + final List> bolts = (List>) result + .get("spouts"); return bolts.stream() .filter((entry) -> spoutId.equals(entry.get("spoutId"))) .findFirst() - .orElseThrow(() -> new IllegalArgumentException("Unable to find entry for spoutId '" + spoutId + "'")); + .orElseThrow(() -> new IllegalArgumentException("Unable to find entry for spoutId '" + + spoutId + "'")); } @Test @@ -708,8 +731,10 @@ public void testGetJsonResponseHeadersCallbackIgnoredWhenJsonpDisabled() { @Test public void testConfigSslKeepsJettyDefaultTlsExclusions(@TempDir Path tempDir) throws Exception { SslContextFactory.Server defaults = new SslContextFactory.Server(); - Set expectedProtocols = new LinkedHashSet<>(Arrays.asList(defaults.getExcludeProtocols())); - Set expectedCiphers = new LinkedHashSet<>(Arrays.asList(defaults.getExcludeCipherSuites())); + Set expectedProtocols = new LinkedHashSet<>(Arrays.asList(defaults + .getExcludeProtocols())); + Set expectedCiphers = new LinkedHashSet<>(Arrays.asList(defaults + .getExcludeCipherSuites())); assertFalse(expectedProtocols.isEmpty()); assertFalse(expectedCiphers.isEmpty()); expectedProtocols.add("SSLv3"); @@ -722,19 +747,24 @@ public void testConfigSslKeepsJettyDefaultTlsExclusions(@TempDir Path tempDir) t null, null, null, false, false, false); ServerConnector connector = (ServerConnector) server.getConnectors()[0]; - SslContextFactory factory = connector.getConnectionFactory(SslConnectionFactory.class).getSslContextFactory(); - assertEquals(expectedProtocols, new LinkedHashSet<>(Arrays.asList(factory.getExcludeProtocols()))); - assertEquals(expectedCiphers, new LinkedHashSet<>(Arrays.asList(factory.getExcludeCipherSuites()))); + SslContextFactory factory = connector.getConnectionFactory(SslConnectionFactory.class) + .getSslContextFactory(); + assertEquals(expectedProtocols, new LinkedHashSet<>(Arrays.asList(factory + .getExcludeProtocols()))); + assertEquals(expectedCiphers, new LinkedHashSet<>(Arrays.asList(factory + .getExcludeCipherSuites()))); } @Test public void testCorsFilterHandleSetsExplicitInitParameters() { FilterHolder filterHolder = UIHelpers.corsFilterHandle(); assertEquals("*", filterHolder.getInitParameter(CrossOriginFilter.ALLOWED_ORIGINS_PARAM)); - assertEquals("GET, POST, PUT", filterHolder.getInitParameter(CrossOriginFilter.ALLOWED_METHODS_PARAM)); + assertEquals("GET, POST, PUT", filterHolder + .getInitParameter(CrossOriginFilter.ALLOWED_METHODS_PARAM)); assertEquals("X-Requested-With, X-Requested-By, Access-Control-Allow-Origin," + " Content-Type, Content-Length, Accept, Origin", filterHolder.getInitParameter(CrossOriginFilter.ALLOWED_HEADERS_PARAM)); - assertEquals("false", filterHolder.getInitParameter(CrossOriginFilter.ALLOW_CREDENTIALS_PARAM)); + assertEquals("false", filterHolder + .getInitParameter(CrossOriginFilter.ALLOW_CREDENTIALS_PARAM)); } } diff --git a/storm-webapp/src/test/java/org/apache/storm/daemon/ui/filters/AuthorizedUserFilterTest.java b/storm-webapp/src/test/java/org/apache/storm/daemon/ui/filters/AuthorizedUserFilterTest.java index 5e9e2c5c1de..61ee58f4326 100644 --- a/storm-webapp/src/test/java/org/apache/storm/daemon/ui/filters/AuthorizedUserFilterTest.java +++ b/storm-webapp/src/test/java/org/apache/storm/daemon/ui/filters/AuthorizedUserFilterTest.java @@ -18,6 +18,14 @@ package org.apache.storm.daemon.ui.filters; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + import jakarta.ws.rs.container.ContainerRequestContext; import jakarta.ws.rs.container.ResourceInfo; import jakarta.ws.rs.core.Response; @@ -31,14 +39,6 @@ import org.junit.jupiter.api.Test; import org.mockito.ArgumentCaptor; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - public class AuthorizedUserFilterTest { /** @@ -58,7 +58,8 @@ public void unannotated() { } } - private static AuthorizedUserFilter filterFor(String methodName, IAuthorizer aclHandler) throws Exception { + private static AuthorizedUserFilter filterFor(String methodName, + IAuthorizer aclHandler) throws Exception { Method method = SampleResource.class.getMethod(methodName); ResourceInfo resourceInfo = mock(ResourceInfo.class); when(resourceInfo.getResourceMethod()).thenReturn(method); @@ -95,7 +96,8 @@ public void unannotatedEndpointIsDenied() throws Exception { @Test public void annotatedEndpointIsCheckedAgainstItsOperation() throws Exception { IAuthorizer aclHandler = mock(IAuthorizer.class); - when(aclHandler.permit(any(ReqContext.class), eq("getNimbusConf"), any())).thenReturn(false); + when(aclHandler.permit(any(ReqContext.class), eq("getNimbusConf"), any())) + .thenReturn(false); AuthorizedUserFilter filter = filterFor("gated", aclHandler); ContainerRequestContext request = mock(ContainerRequestContext.class);