diff --git a/.mvn/.gitkeep b/.mvn/.gitkeep
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/.mvn/jvm.config b/.mvn/jvm.config
new file mode 100644
index 00000000000..3d61a1f27e7
--- /dev/null
+++ b/.mvn/jvm.config
@@ -0,0 +1,11 @@
+--add-exports jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED
+--add-exports jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED
+--add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
+--add-exports jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED
+--add-exports jdk.compiler/com.sun.tools.javac.model=ALL-UNNAMED
+--add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED
+--add-exports jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED
+--add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED
+--add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
+--add-opens jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED
+--add-opens jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED
diff --git a/examples/storm-hdfs-examples/pom.xml b/examples/storm-hdfs-examples/pom.xml
index 40e66a2a582..a307844ef27 100644
--- a/examples/storm-hdfs-examples/pom.xml
+++ b/examples/storm-hdfs-examples/pom.xml
@@ -93,6 +93,16 @@
+
+ org.openrewrite.maven
+ rewrite-maven-plugin
+
+
+
+ org.codehaus.gmavenplus
+ gmavenplus-plugin
+
+ org.apache.maven.pluginsmaven-checkstyle-plugin
diff --git a/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/bolt/HdfsFileTopology.java b/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/bolt/HdfsFileTopology.java
index 01a446c9353..c4087e72801 100644
--- a/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/bolt/HdfsFileTopology.java
+++ b/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/bolt/HdfsFileTopology.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -56,7 +62,8 @@ public static void main(String[] args) throws Exception {
SyncPolicy syncPolicy = new CountSyncPolicy(1000);
// rotate files when they reach 5MB
- FileRotationPolicy rotationPolicy = new TimedRotationPolicy(1.0f, TimedRotationPolicy.TimeUnit.MINUTES);
+ FileRotationPolicy rotationPolicy = new TimedRotationPolicy(1.0f,
+ TimedRotationPolicy.TimeUnit.MINUTES);
FileNameFormat fileNameFormat = new DefaultFileNameFormat()
.withPath("/tmp/foo/")
@@ -91,7 +98,9 @@ public static void main(String[] args) throws Exception {
if (args.length == 3) {
topoName = args[2];
} else if (args.length > 3) {
- System.out.println("Usage: HdfsFileTopology [hdfs url] [hdfs yaml config file] ");
+ System.out
+ .println("Usage: HdfsFileTopology [hdfs url] [hdfs yaml config file] "
+ + "");
return;
}
StormSubmitter.submitTopology(topoName, config, builder.createTopology());
@@ -101,7 +110,7 @@ public static void waitForSeconds(int seconds) {
try {
Thread.sleep(seconds * 1000);
} catch (InterruptedException e) {
- //ignore
+ // ignore
}
}
@@ -145,7 +154,8 @@ public void nextTuple() {
total++;
if (count > 20000) {
count = 0;
- System.out.println("Pending count: " + this.pending.size() + ", total: " + this.total);
+ System.out.println("Pending count: " + this.pending.size() + ", total: "
+ + this.total);
}
Thread.yield();
}
@@ -168,7 +178,8 @@ public static class MyBolt extends BaseRichBolt {
private OutputCollector collector;
@Override
- public void prepare(Map config, TopologyContext context, OutputCollector collector) {
+ public void prepare(Map config, TopologyContext context,
+ OutputCollector collector) {
this.counts = new HashMap();
this.collector = collector;
}
diff --git a/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/bolt/SequenceFileTopology.java b/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/bolt/SequenceFileTopology.java
index 90e0aea0d42..111e6a61807 100644
--- a/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/bolt/SequenceFileTopology.java
+++ b/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/bolt/SequenceFileTopology.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -25,8 +31,8 @@
import org.apache.storm.hdfs.bolt.format.DefaultSequenceFormat;
import org.apache.storm.hdfs.bolt.format.FileNameFormat;
import org.apache.storm.hdfs.bolt.rotation.FileRotationPolicy;
-import org.apache.storm.hdfs.bolt.rotation.FileSizeRotationPolicy;
import org.apache.storm.hdfs.bolt.rotation.FileSizeRotationPolicy.Units;
+import org.apache.storm.hdfs.bolt.rotation.FileSizeRotationPolicy;
import org.apache.storm.hdfs.bolt.sync.CountSyncPolicy;
import org.apache.storm.hdfs.bolt.sync.SyncPolicy;
import org.apache.storm.hdfs.common.rotation.MoveFileAction;
@@ -94,7 +100,9 @@ public static void main(String[] args) throws Exception {
if (args.length == 3) {
topoName = args[2];
} else if (args.length > 3) {
- System.out.println("Usage: SequenceFileTopology [hdfs url] [hdfs yaml config file] ");
+ System.out
+ .println("Usage: SequenceFileTopology [hdfs url] [hdfs yaml config file] "
+ + "");
return;
}
StormSubmitter.submitTopology(topoName, config, builder.createTopology());
@@ -104,14 +112,12 @@ public static void waitForSeconds(int seconds) {
try {
Thread.sleep(seconds * 1000);
} catch (InterruptedException e) {
- //ignore
+ // ignore
}
}
-
public static class SentenceSpout extends BaseRichSpout {
-
private ConcurrentHashMap pending;
private SpoutOutputCollector collector;
private String[] sentences = {
@@ -151,7 +157,8 @@ public void nextTuple() {
total++;
if (count > 20000) {
count = 0;
- System.out.println("Pending count: " + this.pending.size() + ", total: " + this.total);
+ System.out.println("Pending count: " + this.pending.size() + ", total: "
+ + this.total);
}
Thread.yield();
}
@@ -169,14 +176,14 @@ public void fail(Object msgId) {
}
}
-
public static class MyBolt extends BaseRichBolt {
private HashMap counts = null;
private OutputCollector collector;
@Override
- public void prepare(Map config, TopologyContext context, OutputCollector collector) {
+ public void prepare(Map config, TopologyContext context,
+ OutputCollector collector) {
this.counts = new HashMap();
this.collector = collector;
}
@@ -186,7 +193,6 @@ public void execute(Tuple tuple) {
collector.ack(tuple);
}
-
@Override
public void declareOutputFields(OutputFieldsDeclarer declarer) {
// this bolt does not emit anything
diff --git a/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/spout/HdfsSpoutTopology.java b/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/spout/HdfsSpoutTopology.java
index 01d44832a49..3a59a20cb94 100644
--- a/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/spout/HdfsSpoutTopology.java
+++ b/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/spout/HdfsSpoutTopology.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -35,14 +41,14 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
public class HdfsSpoutTopology {
public static final String SPOUT_ID = "hdfsspout";
public static final String BOLT_ID = "constbolt";
/**
- * Copies text file content from sourceDir to destinationDir. Moves source files into sourceDir after its done consuming
+ * Copies text file content from sourceDir to destinationDir. Moves source files into sourceDir
+ * after its done consuming
*/
public static void main(String[] args) throws Exception {
// 0 - validate args
@@ -50,13 +56,21 @@ public static void main(String[] args) throws Exception {
System.err.println("Please check command line arguments.");
System.err.println("Usage :");
System.err.println(
- HdfsSpoutTopology.class.toString() + " topologyName hdfsUri fileFormat sourceDir sourceArchiveDir badDir destinationDir.");
+ HdfsSpoutTopology.class.toString()
+ + " topologyName hdfsUri fileFormat sourceDir sourceArchiveDir badDir "
+ + "destinationDir.");
System.err.println(" topologyName - topology name.");
System.err.println(" hdfsUri - hdfs name node URI");
- System.err.println(" fileFormat - Set to 'TEXT' for reading text files or 'SEQ' for sequence files.");
+ System.err
+ .println(" fileFormat - Set to 'TEXT' for reading text files or 'SEQ' for "
+ + "sequence files.");
System.err.println(" sourceDir - read files from this HDFS dir using HdfsSpout.");
- System.err.println(" archiveDir - after a file in sourceDir is read completely, it is moved to this HDFS location.");
- System.err.println(" badDir - files that cannot be read properly will be moved to this HDFS location.");
+ System.err
+ .println(" archiveDir - after a file in sourceDir is read completely, it is "
+ + "moved to this HDFS location.");
+ System.err
+ .println(" badDir - files that cannot be read properly will be moved to this "
+ + "HDFS location.");
System.err.println(" spoutCount - Num of spout instances.");
System.err.println();
System.exit(-1);
@@ -153,7 +167,8 @@ public ConstBolt() {
}
@Override
- public void prepare(Map conf, TopologyContext context, OutputCollector collector) {
+ public void prepare(Map conf, TopologyContext context,
+ OutputCollector collector) {
this.collector = collector;
}
diff --git a/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/trident/TridentFileTopology.java b/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/trident/TridentFileTopology.java
index b722497183c..5d495508222 100644
--- a/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/trident/TridentFileTopology.java
+++ b/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/trident/TridentFileTopology.java
@@ -41,11 +41,15 @@
public class TridentFileTopology {
public static StormTopology buildTopology(String hdfsUrl) {
- FixedBatchSpout spout = new FixedBatchSpout(new Fields("sentence", "key"), 1000, new Values("the cow jumped over the moon", 1L),
- new Values("the man went to the store and bought some candy", 2L),
- new Values("four score and seven years ago", 3L),
+ FixedBatchSpout spout = new FixedBatchSpout(new Fields("sentence", "key"), 1000,
+ new Values("the cow jumped over the moon", 1L),
+ new Values("the man went to the store and "
+ + "bought some candy", 2L),
+ new Values("four score and seven years ago",
+ 3L),
new Values("how many apples can you eat", 4L),
- new Values("to be or not to be the person", 5L));
+ new Values("to be or not to be the person",
+ 5L));
spout.setCycle(true);
TridentTopology topology = new TridentTopology();
@@ -61,7 +65,8 @@ public static StormTopology buildTopology(String hdfsUrl) {
RecordFormat recordFormat = new DelimitedRecordFormat()
.withFields(hdfsFields);
- FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(5.0f, FileSizeRotationPolicy.Units.MB);
+ FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(5.0f,
+ FileSizeRotationPolicy.Units.MB);
HdfsState.Options options = new HdfsState.HdfsFileOptions()
.withFileNameFormat(fileNameFormat)
@@ -91,7 +96,9 @@ public static void main(String[] args) throws Exception {
if (args.length == 3) {
topoName = args[2];
} else if (args.length > 3) {
- System.out.println("Usage: TridentFileTopology [hdfs url] [hdfs yaml config file] ");
+ System.out
+ .println("Usage: TridentFileTopology [hdfs url] [hdfs yaml config file] "
+ + "");
return;
}
conf.setNumWorkers(3);
diff --git a/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/trident/TridentSequenceTopology.java b/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/trident/TridentSequenceTopology.java
index 74d1d5c8dc5..dddd727e2f5 100644
--- a/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/trident/TridentSequenceTopology.java
+++ b/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/trident/TridentSequenceTopology.java
@@ -41,11 +41,15 @@
public class TridentSequenceTopology {
public static StormTopology buildTopology(String hdfsUrl) {
- FixedBatchSpout spout = new FixedBatchSpout(new Fields("sentence", "key"), 1000, new Values("the cow jumped over the moon", 1L),
- new Values("the man went to the store and bought some candy", 2L),
- new Values("four score and seven years ago", 3L),
+ FixedBatchSpout spout = new FixedBatchSpout(new Fields("sentence", "key"), 1000,
+ new Values("the cow jumped over the moon", 1L),
+ new Values("the man went to the store and "
+ + "bought some candy", 2L),
+ new Values("four score and seven years ago",
+ 3L),
new Values("how many apples can you eat", 4L),
- new Values("to be or not to be the person", 5L));
+ new Values("to be or not to be the person",
+ 5L));
spout.setCycle(true);
TridentTopology topology = new TridentTopology();
@@ -58,7 +62,8 @@ public static StormTopology buildTopology(String hdfsUrl) {
.withPrefix("trident")
.withExtension(".seq");
- FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(5.0f, FileSizeRotationPolicy.Units.MB);
+ FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(5.0f,
+ FileSizeRotationPolicy.Units.MB);
HdfsState.Options seqOpts = new HdfsState.SequenceFileOptions()
.withFileNameFormat(fileNameFormat)
@@ -88,7 +93,8 @@ public static void main(String[] args) throws Exception {
if (args.length == 3) {
topoName = args[2];
} else if (args.length > 3) {
- System.out.println("Usage: TridentSequenceTopology []");
+ System.out
+ .println("Usage: TridentSequenceTopology []");
return;
}
diff --git a/examples/storm-iceberg-examples/src/main/java/org/apache/storm/iceberg/examples/BoundedTupleSpout.java b/examples/storm-iceberg-examples/src/main/java/org/apache/storm/iceberg/examples/BoundedTupleSpout.java
index 95c4596f79b..20c7ceefa0e 100644
--- a/examples/storm-iceberg-examples/src/main/java/org/apache/storm/iceberg/examples/BoundedTupleSpout.java
+++ b/examples/storm-iceberg-examples/src/main/java/org/apache/storm/iceberg/examples/BoundedTupleSpout.java
@@ -60,7 +60,8 @@ public BoundedTupleSpout(long totalTuples, Fields outputFields, ValuesFactory va
}
@Override
- public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
+ public void open(Map conf, TopologyContext context,
+ SpoutOutputCollector collector) {
this.collector = collector;
// Each task walks its own stride of the sequence, so the spout can be parallelised without
// any two tasks emitting the same row.
diff --git a/examples/storm-jdbc-examples/pom.xml b/examples/storm-jdbc-examples/pom.xml
index 95525a3cc26..5a3e7ab82e2 100644
--- a/examples/storm-jdbc-examples/pom.xml
+++ b/examples/storm-jdbc-examples/pom.xml
@@ -85,6 +85,16 @@
+
+ org.openrewrite.maven
+ rewrite-maven-plugin
+
+
+
+ org.codehaus.gmavenplus
+ gmavenplus-plugin
+
+ org.apache.maven.pluginsmaven-checkstyle-plugin
diff --git a/examples/storm-jdbc-examples/src/main/java/org/apache/storm/jdbc/spout/UserSpout.java b/examples/storm-jdbc-examples/src/main/java/org/apache/storm/jdbc/spout/UserSpout.java
index 6854722191d..59c383ca887 100644
--- a/examples/storm-jdbc-examples/src/main/java/org/apache/storm/jdbc/spout/UserSpout.java
+++ b/examples/storm-jdbc-examples/src/main/java/org/apache/storm/jdbc/spout/UserSpout.java
@@ -7,9 +7,9 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
+ *
Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
@@ -19,11 +19,9 @@
package org.apache.storm.jdbc.spout;
import com.google.common.collect.Lists;
-
import java.util.List;
import java.util.Map;
import java.util.Random;
-
import org.apache.storm.spout.SpoutOutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.IRichSpout;
@@ -52,7 +50,8 @@ public boolean isDistributed() {
}
@Override
- public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
+ public void open(Map conf, TopologyContext context,
+ SpoutOutputCollector collector) {
this.collector = collector;
}
diff --git a/examples/storm-jdbc-examples/src/main/java/org/apache/storm/jdbc/topology/AbstractUserTopology.java b/examples/storm-jdbc-examples/src/main/java/org/apache/storm/jdbc/topology/AbstractUserTopology.java
index aa7f68609a2..d741f675b67 100644
--- a/examples/storm-jdbc-examples/src/main/java/org/apache/storm/jdbc/topology/AbstractUserTopology.java
+++ b/examples/storm-jdbc-examples/src/main/java/org/apache/storm/jdbc/topology/AbstractUserTopology.java
@@ -7,9 +7,9 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
+ *
Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
@@ -20,11 +20,9 @@
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
-
import java.sql.Types;
import java.util.List;
import java.util.Map;
-
import org.apache.storm.Config;
import org.apache.storm.StormSubmitter;
import org.apache.storm.generated.StormTopology;
@@ -47,7 +45,8 @@ public abstract class AbstractUserTopology {
"drop table if exists user",
"drop table if exists department",
"drop table if exists user_department",
- "create table if not exists user (user_id integer, user_name varchar(100), dept_name varchar(100), create_date date)",
+ "create table if not exists user (user_id integer, user_name varchar(100), dept_name "
+ + "varchar(100), create_date date)",
"create table if not exists department (dept_id integer, dept_name varchar(100))",
"create table if not exists user_department (user_id integer, dept_id integer)",
"insert into department values (1, 'R&D')",
@@ -72,22 +71,24 @@ public abstract class AbstractUserTopology {
/**
* A main method template to extend.
+ *
* @param args main method arguments
* @throws Exception any expection occuring durch cluster setup or operation
*/
public void execute(String[] args) throws Exception {
if (args.length != 4 && args.length != 5) {
- System.out.println("Usage: " + this.getClass().getSimpleName() + " "
+ System.out.println("Usage: " + this.getClass().getSimpleName()
+ + " "
+ " [topology name]");
System.exit(-1);
}
Map map = Maps.newHashMap();
- map.put("dataSourceClassName", args[0]); //com.mysql.jdbc.jdbc2.optional.MysqlDataSource
- map.put("dataSource.url", args[1]); //jdbc:mysql://localhost/test
- map.put("dataSource.user", args[2]); //root
+ map.put("dataSourceClassName", args[0]); // com.mysql.jdbc.jdbc2.optional.MysqlDataSource
+ map.put("dataSource.url", args[1]); // jdbc:mysql://localhost/test
+ map.put("dataSource.user", args[2]); // root
if (args.length == 4) {
- map.put("dataSource.password", args[3]); //password
+ map.put("dataSource.password", args[3]); // password
}
Config config = new Config();
diff --git a/examples/storm-jdbc-examples/src/main/java/org/apache/storm/jdbc/topology/UserPersistenceTopology.java b/examples/storm-jdbc-examples/src/main/java/org/apache/storm/jdbc/topology/UserPersistenceTopology.java
index 7d4129341f6..3d4bfbb8530 100644
--- a/examples/storm-jdbc-examples/src/main/java/org/apache/storm/jdbc/topology/UserPersistenceTopology.java
+++ b/examples/storm-jdbc-examples/src/main/java/org/apache/storm/jdbc/topology/UserPersistenceTopology.java
@@ -7,9 +7,9 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
+ *
Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
@@ -19,10 +19,8 @@
package org.apache.storm.jdbc.topology;
import com.google.common.collect.Lists;
-
import java.sql.Types;
import java.util.List;
-
import org.apache.storm.generated.StormTopology;
import org.apache.storm.jdbc.bolt.JdbcInsertBolt;
import org.apache.storm.jdbc.bolt.JdbcLookupBolt;
@@ -31,7 +29,6 @@
import org.apache.storm.jdbc.mapper.SimpleJdbcMapper;
import org.apache.storm.topology.TopologyBuilder;
-
public class UserPersistenceTopology extends AbstractUserTopology {
private static final String USER_SPOUT = "USER_SPOUT";
private static final String LOOKUP_BOLT = "LOOKUP_BOLT";
@@ -43,9 +40,10 @@ public static void main(String[] args) throws Exception {
@Override
public StormTopology getTopology() {
- JdbcLookupBolt departmentLookupBolt = new JdbcLookupBolt(connectionProvider, SELECT_QUERY, this.jdbcLookupMapper);
+ JdbcLookupBolt departmentLookupBolt = new JdbcLookupBolt(connectionProvider, SELECT_QUERY,
+ this.jdbcLookupMapper);
- //must specify column schema when providing custom query.
+ // must specify column schema when providing custom query.
List schemaColumns = Lists.newArrayList(new Column("create_date", Types.DATE),
new Column("dept_name", Types.VARCHAR),
new Column("user_id", Types.INTEGER),
@@ -53,7 +51,8 @@ public StormTopology getTopology() {
JdbcMapper mapper = new SimpleJdbcMapper(schemaColumns);
JdbcInsertBolt userPersistenceBolt = new JdbcInsertBolt(connectionProvider, mapper)
- .withInsertQuery("insert into user (create_date, dept_name, user_id, user_name) values (?,?,?,?)");
+ .withInsertQuery("insert into user (create_date, dept_name, user_id, user_name) "
+ + "values (?,?,?,?)");
// userSpout ==> jdbcBolt
TopologyBuilder builder = new TopologyBuilder();
diff --git a/examples/storm-jdbc-examples/src/main/java/org/apache/storm/jdbc/topology/UserPersistenceTridentTopology.java b/examples/storm-jdbc-examples/src/main/java/org/apache/storm/jdbc/topology/UserPersistenceTridentTopology.java
index 1c38c351efd..7e3274f5021 100644
--- a/examples/storm-jdbc-examples/src/main/java/org/apache/storm/jdbc/topology/UserPersistenceTridentTopology.java
+++ b/examples/storm-jdbc-examples/src/main/java/org/apache/storm/jdbc/topology/UserPersistenceTridentTopology.java
@@ -7,9 +7,9 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
+ *
Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
@@ -19,9 +19,7 @@
package org.apache.storm.jdbc.topology;
import com.google.common.collect.Lists;
-
import java.sql.Types;
-
import org.apache.storm.generated.StormTopology;
import org.apache.storm.jdbc.common.Column;
import org.apache.storm.jdbc.mapper.SimpleJdbcLookupMapper;
diff --git a/examples/storm-jms-examples/pom.xml b/examples/storm-jms-examples/pom.xml
index 41cf4b9bd71..47f765b7fe2 100644
--- a/examples/storm-jms-examples/pom.xml
+++ b/examples/storm-jms-examples/pom.xml
@@ -105,6 +105,16 @@
+
+ org.openrewrite.maven
+ rewrite-maven-plugin
+
+
+
+ org.codehaus.gmavenplus
+ gmavenplus-plugin
+
+ org.apache.maven.pluginsmaven-checkstyle-plugin
diff --git a/examples/storm-jms-examples/src/main/java/org/apache/storm/jms/example/ExampleJmsTopology.java b/examples/storm-jms-examples/src/main/java/org/apache/storm/jms/example/ExampleJmsTopology.java
index aea290b752c..c0cd47fcd70 100644
--- a/examples/storm-jms-examples/src/main/java/org/apache/storm/jms/example/ExampleJmsTopology.java
+++ b/examples/storm-jms-examples/src/main/java/org/apache/storm/jms/example/ExampleJmsTopology.java
@@ -46,6 +46,7 @@ public class ExampleJmsTopology {
/**
* The main method.
+ *
* @param args takes the topology name as first argument
* @throws Exception any expection occuring durch cluster setup or operation
*/
@@ -72,7 +73,8 @@ public static void main(String[] args) throws Exception {
builder.setSpout(JMS_QUEUE_SPOUT, queueSpout, 5);
// intermediate bolt, subscribes to jms spout, anchors on tuples, and auto-acks
builder.setBolt(INTERMEDIATE_BOLT,
- new GenericBolt("INTERMEDIATE_BOLT", true, true, new Fields("json")), 3).shuffleGrouping(
+ new GenericBolt("INTERMEDIATE_BOLT", true, true, new Fields("json")), 3)
+ .shuffleGrouping(
JMS_QUEUE_SPOUT);
// bolt that subscribes to the intermediate bolt, and auto-acks
@@ -110,7 +112,8 @@ public Message toMessage(Session session, ITuple input) throws JMSException {
builder.setSpout(JMS_TOPIC_SPOUT, topicSpout);
- builder.setBolt(ANOTHER_BOLT, new GenericBolt("ANOTHER_BOLT", true, true), 1).shuffleGrouping(
+ builder.setBolt(ANOTHER_BOLT, new GenericBolt("ANOTHER_BOLT", true, true), 1)
+ .shuffleGrouping(
JMS_TOPIC_SPOUT);
Config conf = new Config();
diff --git a/examples/storm-jms-examples/src/main/java/org/apache/storm/jms/example/SpringJmsProvider.java b/examples/storm-jms-examples/src/main/java/org/apache/storm/jms/example/SpringJmsProvider.java
index 334a98e3fd7..57f8d41a418 100644
--- a/examples/storm-jms-examples/src/main/java/org/apache/storm/jms/example/SpringJmsProvider.java
+++ b/examples/storm-jms-examples/src/main/java/org/apache/storm/jms/example/SpringJmsProvider.java
@@ -52,8 +52,10 @@ public class SpringJmsProvider implements JmsProvider {
* @param connectionFactoryBean - the JMS connection factory bean name
* @param destinationBean - the JMS destination bean name
*/
- public SpringJmsProvider(String appContextClasspathResource, String connectionFactoryBean, String destinationBean) {
- ApplicationContext context = new ClassPathXmlApplicationContext(appContextClasspathResource);
+ public SpringJmsProvider(String appContextClasspathResource, String connectionFactoryBean,
+ String destinationBean) {
+ ApplicationContext context =
+ new ClassPathXmlApplicationContext(appContextClasspathResource);
this.connectionFactory = (ConnectionFactory) context.getBean(connectionFactoryBean);
this.destination = (Destination) context.getBean(destinationBean);
}
diff --git a/examples/storm-kafka-client-examples/pom.xml b/examples/storm-kafka-client-examples/pom.xml
index 4700be77148..f29819951e8 100644
--- a/examples/storm-kafka-client-examples/pom.xml
+++ b/examples/storm-kafka-client-examples/pom.xml
@@ -103,6 +103,16 @@
+
+ org.openrewrite.maven
+ rewrite-maven-plugin
+
+
+
+ org.codehaus.gmavenplus
+ gmavenplus-plugin
+
+ org.apache.maven.pluginsmaven-checkstyle-plugin
diff --git a/examples/storm-kafka-client-examples/src/main/java/org/apache/storm/kafka/bolt/KafkaProducerTopology.java b/examples/storm-kafka-client-examples/src/main/java/org/apache/storm/kafka/bolt/KafkaProducerTopology.java
index f57d98d4d03..f4a1b373659 100644
--- a/examples/storm-kafka-client-examples/src/main/java/org/apache/storm/kafka/bolt/KafkaProducerTopology.java
+++ b/examples/storm-kafka-client-examples/src/main/java/org/apache/storm/kafka/bolt/KafkaProducerTopology.java
@@ -35,13 +35,14 @@ public class KafkaProducerTopology {
*
* @param brokerUrl Kafka broker URL
* @param topicName Topic to which publish sentences
- * @return A Storm topology that produces random UUIDs using a {@link LambdaSpout} and uses a {@link KafkaBolt} to publish the UUIDs to
+ * @return A Storm topology that produces random UUIDs using a {@link LambdaSpout} and uses a
+ * {@link KafkaBolt} to publish the UUIDs to
* the kafka topic specified
*/
public static StormTopology newTopology(String brokerUrl, String topicName) {
final TopologyBuilder builder = new TopologyBuilder();
builder.setSpout("spout", () -> {
- Utils.sleep(1000); //Throttle this spout a bit to avoid maxing out CPU
+ Utils.sleep(1000); // Throttle this spout a bit to avoid maxing out CPU
return UUID.randomUUID().toString();
});
@@ -60,14 +61,18 @@ public static StormTopology newTopology(String brokerUrl, String topicName) {
/**
* Create the Storm config.
- * @return the Storm config for the topology that publishes random UUIDs to Kafka using a Kafka bolt.
+ *
+ * @return the Storm config for the topology that publishes random UUIDs to Kafka using a Kafka
+ * bolt.
*/
private static Properties newProps(final String brokerUrl, final String topicName) {
return new Properties() {
{
put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, brokerUrl);
- put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer");
- put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer");
+ put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG,
+ "org.apache.kafka.common.serialization.StringSerializer");
+ put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG,
+ "org.apache.kafka.common.serialization.StringSerializer");
put(ProducerConfig.CLIENT_ID_CONFIG, topicName);
}
};
diff --git a/examples/storm-kafka-client-examples/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutTestBolt.java b/examples/storm-kafka-client-examples/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutTestBolt.java
index 2bc348d7b3c..5ef2b9d3b5a 100644
--- a/examples/storm-kafka-client-examples/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutTestBolt.java
+++ b/examples/storm-kafka-client-examples/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutTestBolt.java
@@ -32,7 +32,8 @@ public class KafkaSpoutTestBolt extends BaseRichBolt {
private OutputCollector collector;
@Override
- public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) {
+ public void prepare(Map topoConf, TopologyContext context,
+ OutputCollector collector) {
this.collector = collector;
}
diff --git a/examples/storm-kafka-client-examples/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutTopologyMainNamedTopics.java b/examples/storm-kafka-client-examples/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutTopologyMainNamedTopics.java
index beb794871be..e43fbdd79e1 100644
--- a/examples/storm-kafka-client-examples/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutTopologyMainNamedTopics.java
+++ b/examples/storm-kafka-client-examples/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutTopologyMainNamedTopics.java
@@ -53,13 +53,19 @@ protected void runMain(String[] args) throws Exception {
Config tpConf = getConfig();
- // Producers. This is just to get some data in Kafka, normally you would be getting this data from elsewhere
- StormSubmitter.submitTopology(TOPIC_0 + "-producer", tpConf, KafkaProducerTopology.newTopology(brokerUrl, TOPIC_0));
- StormSubmitter.submitTopology(TOPIC_1 + "-producer", tpConf, KafkaProducerTopology.newTopology(brokerUrl, TOPIC_1));
- StormSubmitter.submitTopology(TOPIC_2 + "-producer", tpConf, KafkaProducerTopology.newTopology(brokerUrl, TOPIC_2));
+ // Producers. This is just to get some data in Kafka, normally you would be getting this
+ // data from elsewhere
+ StormSubmitter.submitTopology(TOPIC_0 + "-producer", tpConf, KafkaProducerTopology
+ .newTopology(brokerUrl, TOPIC_0));
+ StormSubmitter.submitTopology(TOPIC_1 + "-producer", tpConf, KafkaProducerTopology
+ .newTopology(brokerUrl, TOPIC_1));
+ StormSubmitter.submitTopology(TOPIC_2 + "-producer", tpConf, KafkaProducerTopology
+ .newTopology(brokerUrl, TOPIC_2));
- //Consumer. Sets up a topology that reads the given Kafka spouts and logs the received messages
- StormSubmitter.submitTopology("storm-kafka-client-spout-test", tpConf, getTopologyKafkaSpout(getKafkaSpoutConfig(brokerUrl)));
+ // Consumer. Sets up a topology that reads the given Kafka spouts and logs the received
+ // messages
+ StormSubmitter.submitTopology("storm-kafka-client-spout-test", tpConf,
+ getTopologyKafkaSpout(getKafkaSpoutConfig(brokerUrl)));
}
protected Config getConfig() {
@@ -74,7 +80,8 @@ protected StormTopology getTopologyKafkaSpout(KafkaSpoutConfig s
tp.setBolt("kafka_bolt", new KafkaSpoutTestBolt())
.shuffleGrouping("kafka_spout", TOPIC_0_1_STREAM)
.shuffleGrouping("kafka_spout", TOPIC_2_STREAM);
- tp.setBolt("kafka_bolt_1", new KafkaSpoutTestBolt()).shuffleGrouping("kafka_spout", TOPIC_2_STREAM);
+ tp.setBolt("kafka_bolt_1", new KafkaSpoutTestBolt()).shuffleGrouping("kafka_spout",
+ TOPIC_2_STREAM);
return tp.createTopology();
}
diff --git a/examples/storm-kafka-client-examples/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutTopologyMainWildcardTopics.java b/examples/storm-kafka-client-examples/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutTopologyMainWildcardTopics.java
index a5740782c94..e6b1b0e9e78 100644
--- a/examples/storm-kafka-client-examples/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutTopologyMainWildcardTopics.java
+++ b/examples/storm-kafka-client-examples/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutTopologyMainWildcardTopics.java
@@ -28,7 +28,8 @@
import org.apache.storm.tuple.Values;
/**
- * This example is similar to {@link KafkaSpoutTopologyMainNamedTopics}, but demonstrates subscribing to Kafka topics with a regex.
+ * This example is similar to {@link KafkaSpoutTopologyMainNamedTopics}, but demonstrates
+ * subscribing to Kafka topics with a regex.
*/
public class KafkaSpoutTopologyMainWildcardTopics extends KafkaSpoutTopologyMainNamedTopics {
@@ -52,7 +53,8 @@ protected KafkaSpoutConfig getKafkaSpoutConfig(String bootstrapS
return KafkaSpoutConfig.builder(bootstrapServers, TOPIC_WILDCARD_PATTERN)
.setProp(ConsumerConfig.GROUP_ID_CONFIG, "kafkaSpoutTestGroup")
.setRetry(getRetryService())
- .setRecordTranslator((r) -> new Values(r.topic(), r.partition(), r.offset(), r.key(), r.value()),
+ .setRecordTranslator((r) -> new Values(r.topic(), r.partition(), r.offset(), r.key(), r
+ .value()),
new Fields("topic", "partition", "offset", "key", "value"), STREAM)
.setOffsetCommitPeriodMs(10_000)
.setFirstPollOffsetStrategy(EARLIEST)
diff --git a/examples/storm-kafka-client-examples/src/main/java/org/apache/storm/kafka/trident/TridentKafkaClientTopologyNamedTopics.java b/examples/storm-kafka-client-examples/src/main/java/org/apache/storm/kafka/trident/TridentKafkaClientTopologyNamedTopics.java
index 3c92a22d4fe..74624c57133 100644
--- a/examples/storm-kafka-client-examples/src/main/java/org/apache/storm/kafka/trident/TridentKafkaClientTopologyNamedTopics.java
+++ b/examples/storm-kafka-client-examples/src/main/java/org/apache/storm/kafka/trident/TridentKafkaClientTopologyNamedTopics.java
@@ -40,7 +40,8 @@
/**
* This example sets up a few topologies to put random strings in Kafka topics via the KafkaBolt,
- * and shows how to set up a Trident topology that reads from some Kafka topics using the KafkaSpout.
+ * and shows how to set up a Trident topology that reads from some Kafka topics using the
+ * KafkaSpout.
*/
public class TridentKafkaClientTopologyNamedTopics {
@@ -57,7 +58,8 @@ private KafkaTridentSpoutTransactional newKafkaTridentSpoutTrans
return new KafkaTridentSpoutTransactional<>(spoutConfig);
}
- private static final Func, List
+
+ org.openrewrite.maven
+ rewrite-maven-plugin
+
+
+
+ org.codehaus.gmavenplus
+ gmavenplus-plugin
+
+ org.apache.maven.pluginsmaven-checkstyle-plugin
diff --git a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/CaptureLoad.java b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/CaptureLoad.java
index 00a6a32f4e9..9434ed7163b 100644
--- a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/CaptureLoad.java
+++ b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/CaptureLoad.java
@@ -7,9 +7,9 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
+ *
Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
@@ -70,7 +70,8 @@ private static List extractBoltValues(List summaries,
if (summaries != null) {
for (ExecutorSummary summ : summaries) {
if (summ != null && summ.is_set_stats()) {
- Map> data = func.apply(summ.get_stats().get_specific().get_bolt());
+ Map> data = func.apply(summ.get_stats()
+ .get_specific().get_bolt());
if (data != null) {
List subvalues = data.values().stream()
.map((subMap) -> subMap.get(id))
@@ -84,7 +85,8 @@ private static List extractBoltValues(List summaries,
return ret;
}
- static TopologyLoadConf captureTopology(Nimbus.Iface client, TopologySummary topologySummary) throws Exception {
+ static TopologyLoadConf captureTopology(Nimbus.Iface client,
+ TopologySummary topologySummary) throws Exception {
String topologyName = topologySummary.get_name();
LOG.info("Capturing {}...", topologyName);
String topologyId = topologySummary.get_id();
@@ -92,10 +94,11 @@ static TopologyLoadConf captureTopology(Nimbus.Iface client, TopologySummary top
TopologyPageInfo tpinfo = client.getTopologyPageInfo(topologyId, ":all-time", false);
@SuppressWarnings("checkstyle:VariableDeclarationUsageDistance")
StormTopology topo = client.getUserTopology(topologyId);
- //Done capturing topology information...
+ // Done capturing topology information...
Map savedTopoConf = new HashMap<>();
- Map topoConf = (Map) JSONValue.parse(client.getTopologyConf(topologyId));
+ Map topoConf = (Map) JSONValue.parse(client
+ .getTopologyConf(topologyId));
for (String key : TopologyLoadConf.IMPORTANT_CONF_KEYS) {
Object o = topoConf.get(key);
if (o != null) {
@@ -103,10 +106,11 @@ static TopologyLoadConf captureTopology(Nimbus.Iface client, TopologySummary top
LOG.info("with config {}: {}", key, o);
}
}
- //Lets use the number of actually scheduled workers as a way to bridge RAS and non-RAS
+ // Lets use the number of actually scheduled workers as a way to bridge RAS and non-RAS
int numWorkers = tpinfo.get_num_workers();
if (savedTopoConf.containsKey(Config.TOPOLOGY_WORKERS)) {
- numWorkers = Math.max(numWorkers, ((Number) savedTopoConf.get(Config.TOPOLOGY_WORKERS)).intValue());
+ numWorkers = Math.max(numWorkers, ((Number) savedTopoConf.get(Config.TOPOLOGY_WORKERS))
+ .intValue());
}
savedTopoConf.put(Config.TOPOLOGY_WORKERS, numWorkers);
@@ -115,7 +119,7 @@ static TopologyLoadConf captureTopology(Nimbus.Iface client, TopologySummary top
List inputStreams = new ArrayList<>();
Map outStreams = new HashMap<>();
- //Bolts
+ // Bolts
if (topo.get_bolts() != null) {
for (Map.Entry boltSpec : topo.get_bolts().entrySet()) {
String boltComp = boltSpec.getKey();
@@ -161,7 +165,8 @@ static TopologyLoadConf captureTopology(Nimbus.Iface client, TopologySummary top
if (cpu != null) {
bd.withCpuLoad(cpu);
}
- Double mem = resources.get(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB);
+ Double mem = resources
+ .get(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB);
if (mem != null) {
bd.withMemoryLoad(mem);
}
@@ -169,7 +174,7 @@ static TopologyLoadConf captureTopology(Nimbus.Iface client, TopologySummary top
}
}
- //Spouts
+ // Spouts
if (topo.get_spouts() != null) {
for (Map.Entry spoutSpec : topo.get_spouts().entrySet()) {
String spoutComp = spoutSpec.getKey();
@@ -202,7 +207,8 @@ static TopologyLoadConf captureTopology(Nimbus.Iface client, TopologySummary top
if (cpu != null) {
sd.withCpuLoad(cpu);
}
- Double mem = resources.get(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB);
+ Double mem = resources
+ .get(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB);
if (mem != null) {
sd.withMemoryLoad(mem);
}
@@ -210,7 +216,7 @@ static TopologyLoadConf captureTopology(Nimbus.Iface client, TopologySummary top
}
}
- //Stats...
+ // Stats...
Map> byComponent = new HashMap<>();
for (ExecutorSummary executor : info.get_executors()) {
String component = executor.get_component_id();
@@ -223,20 +229,21 @@ static TopologyLoadConf captureTopology(Nimbus.Iface client, TopologySummary top
}
List streams = new ArrayList<>(inputStreams.size());
- //Compute the stats for the different input streams
+ // Compute the stats for the different input streams
for (InputStream.Builder builder : inputStreams) {
- GlobalStreamId streamId = new GlobalStreamId(builder.getFromComponent(), builder.getId());
+ GlobalStreamId streamId = new GlobalStreamId(builder.getFromComponent(), builder
+ .getId());
List summaries = byComponent.get(builder.getToComponent());
- //Execute and process latency...
+ // Execute and process latency...
builder.withProcessTime(new NormalDistStats(
extractBoltValues(summaries, streamId, BoltStats::get_process_ms_avg)));
builder.withExecTime(new NormalDistStats(
extractBoltValues(summaries, streamId, BoltStats::get_execute_ms_avg)));
- //InputStream is done
+ // InputStream is done
streams.add(builder.build());
}
- //There is a bug in some versions that returns 0 for the uptime.
+ // There is a bug in some versions that returns 0 for the uptime.
// To work around it we should get it an alternative (working) way.
Map workerToUptime = new HashMap<>();
for (WorkerSummary ws : tpinfo.get_workers()) {
@@ -255,18 +262,19 @@ static TopologyLoadConf captureTopology(Nimbus.Iface client, TopologySummary top
int uptime = summary.get_uptime_secs();
LOG.debug("UPTIME {}", uptime);
if (uptime <= 0) {
- //Likely it is because of a bug, so try to get it another way
+ // Likely it is because of a bug, so try to get it another way
String key = summary.get_host() + ":" + summary.get_port();
uptime = workerToUptime.getOrDefault(key, 1);
LOG.debug("Getting uptime for worker {}, {}", key, uptime);
}
- for (Map.Entry> statEntry : summary.get_stats().get_emitted().entrySet()) {
+ for (Map.Entry> statEntry : summary.get_stats()
+ .get_emitted().entrySet()) {
String timeWindow = statEntry.getKey();
long timeSecs = uptime;
try {
timeSecs = Long.valueOf(timeWindow);
} catch (NumberFormatException e) {
- //Ignored...
+ // Ignored...
}
timeSecs = Math.min(timeSecs, uptime);
Long count = statEntry.getValue().get(id.get_streamId());
@@ -281,7 +289,7 @@ static TopologyLoadConf captureTopology(Nimbus.Iface client, TopologySummary top
}
builder.withRate(new NormalDistStats(emittedRate));
- //The OutputStream is done
+ // The OutputStream is done
LoadCompConf.Builder comp = boltBuilders.get(id.get_componentId());
if (comp == null) {
comp = spoutBuilders.get(id.get_componentId());
@@ -302,6 +310,7 @@ static TopologyLoadConf captureTopology(Nimbus.Iface client, TopologySummary top
/**
* Main entry point for CaptureLoad command.
+ *
* @param args the arguments to the command
* @throws Exception on any error
*/
@@ -367,13 +376,16 @@ public static void main(String[] args) throws Exception {
}
}
- //ResourceUtils.java is not a available on the classpath to let us parse out the resources we want.
+ // ResourceUtils.java is not a available on the classpath to let us parse out the resources we
+ // want.
// So we have copied and pasted some of the needed methods here. (with a few changes to logging)
- static Map> getBoltsResources(StormTopology topology, Map topologyConf) {
+ static Map> getBoltsResources(StormTopology topology, Map topologyConf) {
Map> boltResources = new HashMap<>();
if (topology.get_bolts() != null) {
for (Map.Entry bolt : topology.get_bolts().entrySet()) {
- Map topologyResources = parseResources(bolt.getValue().get_common().get_json_conf());
+ Map topologyResources = parseResources(bolt.getValue().get_common()
+ .get_json_conf());
checkInitialization(topologyResources, bolt.getValue().toString(), topologyConf);
boltResources.put(bolt.getKey(), topologyResources);
}
@@ -381,11 +393,13 @@ static Map> getBoltsResources(StormTopology topology
return boltResources;
}
- static Map> getSpoutsResources(StormTopology topology, Map topologyConf) {
+ static Map> getSpoutsResources(StormTopology topology, Map topologyConf) {
Map> spoutResources = new HashMap<>();
if (topology.get_spouts() != null) {
for (Map.Entry spout : topology.get_spouts().entrySet()) {
- Map topologyResources = parseResources(spout.getValue().get_common().get_json_conf());
+ Map topologyResources = parseResources(spout.getValue().get_common()
+ .get_json_conf());
checkInitialization(topologyResources, spout.getValue().toString(), topologyConf);
spoutResources.put(spout.getKey(), topologyResources);
}
@@ -403,16 +417,21 @@ static Map parseResources(String input) {
JSONObject jsonObject = (JSONObject) obj;
if (jsonObject.containsKey(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB)) {
Double topoMemOnHeap = ObjectReader
- .getDouble(jsonObject.get(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB), null);
- topologyResources.put(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB, topoMemOnHeap);
+ .getDouble(jsonObject
+ .get(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB), null);
+ topologyResources.put(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB,
+ topoMemOnHeap);
}
if (jsonObject.containsKey(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB)) {
Double topoMemOffHeap = ObjectReader
- .getDouble(jsonObject.get(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB), null);
- topologyResources.put(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB, topoMemOffHeap);
+ .getDouble(jsonObject
+ .get(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB), null);
+ topologyResources.put(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB,
+ topoMemOffHeap);
}
if (jsonObject.containsKey(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT)) {
- Double topoCpu = ObjectReader.getDouble(jsonObject.get(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT),
+ Double topoCpu = ObjectReader.getDouble(jsonObject
+ .get(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT),
null);
topologyResources.put(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT, topoCpu);
}
@@ -428,12 +447,14 @@ static Map parseResources(String input) {
/**
* Checks if the topology's resource requirements are initialized.
* Will modify topologyResources by adding the appropriate defaults
+ *
* @param topologyResources map of resouces requirements
* @param componentId component for which initialization is being conducted
* @param topologyConf topology configuration
* @throws Exception on any error
*/
- public static void checkInitialization(Map topologyResources, String componentId, Map topologyConf) {
+ public static void checkInitialization(Map topologyResources,
+ String componentId, Map topologyConf) {
StringBuilder msgBuilder = new StringBuilder();
for (String resourceName : topologyResources.keySet()) {
@@ -448,13 +469,15 @@ public static void checkInitialization(Map topologyResources, St
}
}
- private static String checkInitResource(Map topologyResources, Map topologyConf, String resourceName) {
+ private static String checkInitResource(Map topologyResources, Map topologyConf, String resourceName) {
StringBuilder msgBuilder = new StringBuilder();
if (topologyResources.containsKey(resourceName)) {
Double resourceValue = (Double) topologyConf.getOrDefault(resourceName, null);
if (resourceValue != null) {
topologyResources.put(resourceName, resourceValue);
- msgBuilder.append(resourceName.substring(resourceName.lastIndexOf(".")) + " has been set to " + resourceValue);
+ msgBuilder.append(resourceName.substring(resourceName.lastIndexOf("."))
+ + " has been set to " + resourceValue);
}
}
diff --git a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/EstimateThroughput.java b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/EstimateThroughput.java
index 2b1570d957e..54386b70e9a 100644
--- a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/EstimateThroughput.java
+++ b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/EstimateThroughput.java
@@ -7,9 +7,9 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
+ *
Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
@@ -44,6 +44,7 @@ public class EstimateThroughput {
/**
* Main entry point for estimate throughput command.
+ *
* @param args the command line arguments.
* @throws Exception on any error.
*/
@@ -79,7 +80,8 @@ public static void main(String[] args) throws Exception {
for (TopologySummary topologySummary : client.getTopologySummaries()) {
if (topologyNames.isEmpty() || topologyNames.contains(topologySummary.get_name())) {
- TopologyLoadConf capturedConf = CaptureLoad.captureTopology(client, topologySummary);
+ TopologyLoadConf capturedConf = CaptureLoad.captureTopology(client,
+ topologySummary);
if (capturedConf.looksLikeTrident()) {
trident.add(capturedConf);
} else {
@@ -90,10 +92,12 @@ public static void main(String[] args) throws Exception {
System.out.println("TOPOLOGY\tTOTAL MESSAGES/sec\tESTIMATED INPUT MESSAGES/sec");
for (TopologyLoadConf tl : regular) {
- System.out.println(tl.name + "\t" + tl.getAllEmittedAggregate() + "\t" + tl.getSpoutEmittedAggregate());
+ System.out.println(tl.name + "\t" + tl.getAllEmittedAggregate() + "\t" + tl
+ .getSpoutEmittedAggregate());
}
for (TopologyLoadConf tl : trident) {
- System.out.println(tl.name + "\t" + tl.getAllEmittedAggregate() + "\t" + tl.getTridentEstimatedEmittedAggregate());
+ System.out.println(tl.name + "\t" + tl.getAllEmittedAggregate() + "\t" + tl
+ .getTridentEstimatedEmittedAggregate());
}
exitStatus = 0;
} catch (Exception e) {
diff --git a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/ExecAndProcessLatencyEngine.java b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/ExecAndProcessLatencyEngine.java
index c2dd81f71a4..6e63ca23f14 100644
--- a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/ExecAndProcessLatencyEngine.java
+++ b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/ExecAndProcessLatencyEngine.java
@@ -7,9 +7,9 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
+ *
Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
@@ -57,19 +57,21 @@ public void prepare() {
/**
* Sleep for a set number of nano seconds.
+ *
* @param start the start time of the sleep
* @param sleepAmount how many nano seconds after start when we should stop.
*/
public void sleepNano(long start, long sleepAmount) {
long endTime = start + sleepAmount;
- // A small control algorithm to adjust the amount of time that we sleep to make it more accurate
+ // A small control algorithm to adjust the amount of time that we sleep to make it more
+ // accurate
long newEnd = endTime - parkOffset.get();
long diff = newEnd - start;
- //There are some different levels of accuracy here, and we want to deal with all of them
+ // There are some different levels of accuracy here, and we want to deal with all of them
if (diff <= 1_000) {
- //We are done, nothing that short is going to work here
+ // We are done, nothing that short is going to work here
} else if (diff < NANO_IN_MS) {
- //Busy wait...
+ // Busy wait...
long sum = 0;
while (System.nanoTime() < newEnd) {
for (long i = 0; i < 1_000_000; i++) {
@@ -77,7 +79,7 @@ public void sleepNano(long start, long sleepAmount) {
}
}
} else {
- //More accurate that thread.sleep, but still not great
+ // More accurate that thread.sleep, but still not great
LockSupport.parkNanos(newEnd - System.nanoTime());
}
parkOffset.addAndGet((System.nanoTime() - endTime) / 2);
@@ -94,16 +96,22 @@ public void sleepUntilNano(long endTime) {
/**
* Simulate both process and exec times.
+ *
* @param executorIndex the index of this executor. It is used to skew the latencies.
* @param startTimeNs when the executor started in nano-seconds.
* @param in the metrics for the input stream (or null if you don't want to use them).
- * @param r what to run when the process latency is up. Note that this may run on a separate thread after this method call has
+ * @param r what to run when the process latency is up. Note that this may run on a separate
+ * thread after this method call has
* completed.
*/
- public void simulateProcessAndExecTime(int executorIndex, long startTimeNs, InputStream in, Runnable r) {
- long extraTimeNs = skewedPattern == null ? 0 : toNano(skewedPattern.getExtraSlowness(executorIndex));
- long endExecNs = startTimeNs + extraTimeNs + (in == null ? 0 : ExecAndProcessLatencyEngine.toNano(in.execTime.nextRandom(rand)));
- long endProcNs = startTimeNs + extraTimeNs + (in == null ? 0 : ExecAndProcessLatencyEngine.toNano(in.processTime.nextRandom(rand)));
+ public void simulateProcessAndExecTime(int executorIndex, long startTimeNs, InputStream in,
+ Runnable r) {
+ long extraTimeNs = skewedPattern == null ? 0 : toNano(skewedPattern
+ .getExtraSlowness(executorIndex));
+ long endExecNs = startTimeNs + extraTimeNs + (in == null ? 0 : ExecAndProcessLatencyEngine
+ .toNano(in.execTime.nextRandom(rand)));
+ long endProcNs = startTimeNs + extraTimeNs + (in == null ? 0 : ExecAndProcessLatencyEngine
+ .toNano(in.processTime.nextRandom(rand)));
if ((endProcNs - 1_000_000) < endExecNs) {
sleepUntilNano(endProcNs);
diff --git a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/GenLoad.java b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/GenLoad.java
index 8ca16eadece..7bfc4ad747f 100644
--- a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/GenLoad.java
+++ b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/GenLoad.java
@@ -7,9 +7,9 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
+ *
Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
@@ -57,6 +57,7 @@ public class GenLoad {
/**
* Main entry point for GenLoad application.
+ *
* @param args the command line args.
* @throws Exception on any error.
*/
@@ -70,7 +71,8 @@ public static void main(String[] args) throws Exception {
.longOpt("test-time")
.argName("MINS")
.hasArg()
- .desc("How long to run the tests for in mins (defaults to " + TEST_EXECUTE_TIME_DEFAULT + ")")
+ .desc("How long to run the tests for in mins (defaults to " + TEST_EXECUTE_TIME_DEFAULT
+ + ")")
.build());
options.addOption(Option.builder()
.longOpt("parallel")
@@ -79,8 +81,10 @@ public static void main(String[] args) throws Exception {
.desc("How much to scale the topology up or down in parallelism. "
+ "The new parallelism will round up to the next whole number. "
+ "If a topology + component is supplied only that component will be scaled. "
- + "If topo or component is blank or a '*' all topologies or components matched will be scaled. "
- + "Only 1 scaling rule, the most specific, will be applied to a component. Providing a topology name is considered more "
+ + "If topo or component is blank or a '*' all topologies or components matched "
+ + "will be scaled. "
+ + "Only 1 scaling rule, the most specific, will be applied to a component. "
+ + "Providing a topology name is considered more "
+ "specific than not providing one."
+ "(defaults to 1.0 no scaling)")
.build());
@@ -90,8 +94,10 @@ public static void main(String[] args) throws Exception {
.hasArg()
.desc("How much to scale the topology up or down in throughput. "
+ "If a topology + component is supplied only that component will be scaled. "
- + "If topo or component is blank or a '*' all topologies or components matched will be scaled. "
- + "Only 1 scaling rule, the most specific, will be applied to a component. Providing a topology name is considered more "
+ + "If topo or component is blank or a '*' all topologies or components matched "
+ + "will be scaled. "
+ + "Only 1 scaling rule, the most specific, will be applied to a component. "
+ + "Providing a topology name is considered more "
+ "specific than not providing one."
+ "(defaults to 1.0 no scaling)")
.build());
@@ -103,8 +109,10 @@ public static void main(String[] args) throws Exception {
.longOpt("imbalance")
.argName("MS(:COUNT)?:TOPO:COMP")
.hasArg()
- .desc("The number of ms that the first COUNT of TOPO:COMP will wait before processing. This creates an imbalance "
- + "that helps test load aware groupings. By default there is no imbalance. If no count is given it defaults to 1")
+ .desc("The number of ms that the first COUNT of TOPO:COMP will wait before "
+ + "processing. This creates an imbalance "
+ + "that helps test load aware groupings. By default there is no imbalance. If no "
+ + "count is given it defaults to 1")
.build());
options.addOption(Option.builder()
.longOpt("debug")
@@ -129,7 +137,8 @@ public static void main(String[] args) throws Exception {
for (String stringParallel : cmd.getOptionValues("parallel")) {
Matcher m = MULTI_PATTERN.matcher(stringParallel);
if (!m.matches()) {
- throw new ParseException("--parallel " + stringParallel + " is not in the format MULTIPLIER(:TOPO:COMP)?");
+ throw new ParseException("--parallel " + stringParallel
+ + " is not in the format MULTIPLIER(:TOPO:COMP)?");
}
double parallel = Double.parseDouble(m.group("value"));
String topo = m.group("topo");
@@ -151,7 +160,8 @@ public static void main(String[] args) throws Exception {
for (String stringThroughput : cmd.getOptionValues("throughput")) {
Matcher m = MULTI_PATTERN.matcher(stringThroughput);
if (!m.matches()) {
- throw new ParseException("--throughput " + stringThroughput + " is not in the format MULTIPLIER(:TOPO:COMP)?");
+ throw new ParseException("--throughput " + stringThroughput
+ + " is not in the format MULTIPLIER(:TOPO:COMP)?");
}
double throughput = Double.parseDouble(m.group("value"));
String topo = m.group("topo");
@@ -171,13 +181,16 @@ public static void main(String[] args) throws Exception {
}
if (cmd.hasOption("imbalance")) {
for (String stringImbalance : cmd.getOptionValues("imbalance")) {
- //We require there to be both a topology and a component in this case, so parse it out as such.
+ // We require there to be both a topology and a component in this case, so parse
+ // it out as such.
String [] parts = stringImbalance.split(":");
if (parts.length < 3 || parts.length > 4) {
- throw new ParseException(stringImbalance + " does not appear to match the expected pattern");
+ throw new ParseException(stringImbalance
+ + " does not appear to match the expected pattern");
} else if (parts.length == 3) {
- topoSpecificImbalance.put(parts[1] + ":" + parts[2], SlowExecutorPattern.fromString(parts[0]));
- } else { //== 4
+ topoSpecificImbalance.put(parts[1] + ":" + parts[2], SlowExecutorPattern
+ .fromString(parts[0]));
+ } else { // == 4
topoSpecificImbalance.put(parts[2] + ":" + parts[3],
SlowExecutorPattern.fromString(parts[0] + ":" + parts[1]));
}
@@ -197,11 +210,14 @@ public static void main(String[] args) throws Exception {
metrics.put("parallel_adjust", globalParallel);
metrics.put("throughput_adjust", globalThroughput);
metrics.put("local_or_shuffle", cmd.hasOption("local-or-shuffle"));
- metrics.put("topo_parallel", topoSpecificParallel.entrySet().stream().map((entry) -> entry.getValue() + ":" + entry.getKey())
+ metrics.put("topo_parallel", topoSpecificParallel.entrySet().stream().map((entry) -> entry
+ .getValue() + ":" + entry.getKey())
.collect(Collectors.toList()));
- metrics.put("topo_throuhgput", topoSpecificThroughput.entrySet().stream().map((entry) -> entry.getValue() + ":" + entry.getKey())
+ metrics.put("topo_throuhgput", topoSpecificThroughput.entrySet().stream()
+ .map((entry) -> entry.getValue() + ":" + entry.getKey())
.collect(Collectors.toList()));
- metrics.put("slow_execs", topoSpecificImbalance.entrySet().stream().map((entry) -> entry.getValue() + ":" + entry.getKey())
+ metrics.put("slow_execs", topoSpecificImbalance.entrySet().stream().map((entry) -> entry
+ .getValue() + ":" + entry.getKey())
.collect(Collectors.toList()));
Config conf = new Config();
@@ -255,15 +271,16 @@ private static TopologyLoadConf readTopology(String topoFile) throws IOException
private static int uniquifier = 0;
- private static String parseAndSubmit(TopologyLoadConf tlc, String url) throws IOException, InvalidTopologyException,
+ private static String parseAndSubmit(TopologyLoadConf tlc,
+ String url) throws IOException, InvalidTopologyException,
AuthorizationException, AlreadyAliveException {
- //First we need some configs
+ // First we need some configs
Config conf = new Config();
if (tlc.topoConf != null) {
conf.putAll(tlc.topoConf);
}
- //For some reason on the new code if ackers is null we get 0???
+ // For some reason on the new code if ackers is null we get 0???
Object ackers = conf.get(Config.TOPOLOGY_ACKER_EXECUTORS);
Object workers = conf.get(Config.TOPOLOGY_WORKERS);
if (ackers == null || ((Number) ackers).intValue() <= 0) {
@@ -276,17 +293,18 @@ private static String parseAndSubmit(TopologyLoadConf tlc, String url) throws IO
conf.registerMetricsConsumer(HttpForwardingMetricsConsumer.class, url, 1);
Map workerMetrics = new HashMap<>();
if (!NimbusClient.isLocalOverride()) {
- //sigar uses JNI and does not work in local mode
+ // sigar uses JNI and does not work in local mode
workerMetrics.put("CPU", "org.apache.storm.metrics.sigar.CPUMetric");
}
conf.put(Config.TOPOLOGY_WORKER_METRICS, workerMetrics);
conf.put(Config.TOPOLOGY_BUILTIN_METRICS_BUCKET_SIZE_SECS, 10);
- //Lets build a topology.
+ // Lets build a topology.
TopologyBuilder builder = new TopologyBuilder();
for (LoadCompConf spoutConf : tlc.spouts) {
System.out.println("ADDING SPOUT " + spoutConf.id);
- SpoutDeclarer sd = builder.setSpout(spoutConf.id, new LoadSpout(spoutConf), spoutConf.parallelism);
+ SpoutDeclarer sd = builder.setSpout(spoutConf.id, new LoadSpout(spoutConf),
+ spoutConf.parallelism);
if (spoutConf.memoryLoad > 0) {
sd.setMemoryLoad(spoutConf.memoryLoad);
}
@@ -317,7 +335,8 @@ private static String parseAndSubmit(TopologyLoadConf tlc, String url) throws IO
for (InputStream in : tlc.streams) {
BoltDeclarer declarer = boltDeclarers.get(in.toComponent);
if (declarer == null) {
- throw new IllegalArgumentException("to bolt " + in.toComponent + " does not exist");
+ throw new IllegalArgumentException("to bolt " + in.toComponent
+ + " does not exist");
}
LoadBolt lb = bolts.get(in.toComponent);
lb.add(in);
diff --git a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/GroupingType.java b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/GroupingType.java
index a4e0c1af7b5..57a7debc0e9 100644
--- a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/GroupingType.java
+++ b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/GroupingType.java
@@ -7,9 +7,9 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
+ *
Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
@@ -71,6 +71,7 @@ public void assign(BoltDeclarer declarer, InputStream stream) {
/**
* Parse a String config value and covert it into the enum.
+ *
* @param conf the string config.
* @return the parsed grouping type or SHUFFLE if conf is null.
* @throws IllegalArgumentException if parsing does not work.
diff --git a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/HttpForwardingMetricsConsumer.java b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/HttpForwardingMetricsConsumer.java
index f316c76300b..0ca7fbe238e 100644
--- a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/HttpForwardingMetricsConsumer.java
+++ b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/HttpForwardingMetricsConsumer.java
@@ -37,10 +37,13 @@
* conf.registerMetricsConsumer(HttpForwardingMetricsConsumer.class, "http://example.com:8080/metrics/my-topology/", 1);
* ```
*
- *
The body of the post is data serialized using {@link org.apache.storm.serialization.KryoValuesSerializer}, with the data passed in
- * as a list of `[TaskInfo, Collection<DataPoint>]`. More things may be appended to the end of the list in the future.
+ *
The body of the post is data serialized using {@link
+ * org.apache.storm.serialization.KryoValuesSerializer}, with the data passed in
+ * as a list of `[TaskInfo, Collection<DataPoint>]`. More things may be appended to the end of
+ * the list in the future.
*
- *
The values can be deserialized using the org.apache.storm.serialization.KryoValuesDeserializer, and a correct config + classpath.
+ *
The values can be deserialized using the
+ * org.apache.storm.serialization.KryoValuesDeserializer, and a correct config + classpath.
*
*
@see org.apache.storm.serialization.KryoValuesSerializer
*/
@@ -51,7 +54,8 @@ public class HttpForwardingMetricsConsumer implements IMetricsConsumer {
private transient String topologyId;
@Override
- public void prepare(Map topoConf, Object registrationArgument, TopologyContext context, IErrorReporter errorReporter) {
+ public void prepare(Map topoConf, Object registrationArgument,
+ TopologyContext context, IErrorReporter errorReporter) {
try {
url = new URL((String) registrationArgument);
this.errorReporter = errorReporter;
@@ -72,7 +76,7 @@ public void handleDataPoints(TaskInfo taskInfo, Collection dataPoints
serializer.serializeInto(Arrays.asList(taskInfo, dataPoints, topologyId), out);
out.flush();
}
- //The connection is not sent unless a response is requested
+ // The connection is not sent unless a response is requested
int response = con.getResponseCode();
} catch (Exception e) {
throw new RuntimeException(e);
@@ -80,5 +84,5 @@ public void handleDataPoints(TaskInfo taskInfo, Collection dataPoints
}
@Override
- public void cleanup() { }
+ public void cleanup() {}
}
diff --git a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/HttpForwardingMetricsServer.java b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/HttpForwardingMetricsServer.java
index 74317137fe3..95333ac9da3 100644
--- a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/HttpForwardingMetricsServer.java
+++ b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/HttpForwardingMetricsServer.java
@@ -55,16 +55,19 @@ protected KryoValuesDeserializer initialValue() {
private class MetricsCollectionServlet extends HttpServlet {
@Override
- protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
+ protected void doPost(HttpServletRequest request,
+ HttpServletResponse response) throws ServletException, IOException {
Input in = new Input(request.getInputStream());
List metrics = des.get().deserializeFrom(in);
- handle((TaskInfo) metrics.get(0), (Collection) metrics.get(1), (String) metrics.get(2));
+ handle((TaskInfo) metrics.get(0), (Collection) metrics.get(1),
+ (String) metrics.get(2));
response.setStatus(HttpServletResponse.SC_OK);
}
}
/**
* Constructor.
+ *
* @param conf the configuration for storm.
*/
public HttpForwardingMetricsServer(Map conf) {
@@ -74,11 +77,13 @@ public HttpForwardingMetricsServer(Map conf) {
}
}
- //This needs to be thread safe
- public abstract void handle(TaskInfo taskInfo, Collection dataPoints, String topologyId);
+ // This needs to be thread safe
+ public abstract void handle(TaskInfo taskInfo, Collection dataPoints,
+ String topologyId);
/**
* Start the server.
+ *
* @param port the port it shuld listen on, or null/<= 0 to pick a free ephemeral port.
*/
public void serve(Integer port) {
@@ -96,7 +101,8 @@ public void serve(Integer port) {
this.port = port;
url = "http://" + InetAddress.getLocalHost().getHostName() + ":" + this.port + "/";
- ServletContextHandler context = new ServletContextHandler(ServletContextHandler.SESSIONS);
+ ServletContextHandler context =
+ new ServletContextHandler(ServletContextHandler.SESSIONS);
context.setContextPath("/");
server.setHandler(context);
diff --git a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/InputStream.java b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/InputStream.java
index 19802d9e0fa..ece22b88dd5 100644
--- a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/InputStream.java
+++ b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/InputStream.java
@@ -7,9 +7,9 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
+ *
Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
@@ -40,19 +40,22 @@ public class InputStream implements Serializable {
public final NormalDistStats execTime;
public final NormalDistStats processTime;
public final GroupingType groupingType;
- //Cached GlobalStreamId
+ // Cached GlobalStreamId
private GlobalStreamId gsid = null;
/**
* Create an output stream from a config.
+ *
* @param conf the config to read from.
* @return the read OutputStream.
*/
public static InputStream fromConf(Map conf) {
String component = (String) conf.get("from");
String toComp = (String) conf.get("to");
- NormalDistStats execTime = NormalDistStats.fromConf((Map) conf.get("execTime"));
- NormalDistStats processTime = NormalDistStats.fromConf((Map) conf.get("processTime"));
+ NormalDistStats execTime = NormalDistStats.fromConf((Map) conf
+ .get("execTime"));
+ NormalDistStats processTime = NormalDistStats.fromConf((Map) conf
+ .get("processTime"));
Map grouping = (Map) conf.get("grouping");
GroupingType groupingType = GroupingType.fromConf((String) grouping.get("type"));
String streamId = (String) grouping.getOrDefault("streamId", "default");
@@ -61,6 +64,7 @@ public static InputStream fromConf(Map conf) {
/**
* Convert this to a conf.
+ *
* @return the conf.
*/
public Map toConf() {
@@ -142,6 +146,7 @@ public Builder withGroupingType(GroupingType groupingType) {
/**
* Add the grouping type based off of the thrift Grouping class.
+ *
* @param grouping the Grouping to extract the grouping type from
* @return this
*/
@@ -152,7 +157,7 @@ public Builder withGroupingType(Grouping grouping) {
switch (thriftType) {
case FIELDS:
- //Global Grouping is fields with an empty list
+ // Global Grouping is fields with an empty list
if (grouping.get_fields().isEmpty()) {
group = GroupingType.GLOBAL;
} else {
@@ -172,7 +177,7 @@ public Builder withGroupingType(Grouping grouping) {
group = GroupingType.LOCAL_OR_SHUFFLE;
break;
case CUSTOM_SERIALIZED:
- //This might be a partial key grouping..
+ // This might be a partial key grouping..
byte[] data = grouping.get_custom_serialized();
try (ByteArrayInputStream bis = new ByteArrayInputStream(data);
ObjectInputStream ois = new ObjectInputStream(bis);) {
@@ -182,29 +187,33 @@ public Builder withGroupingType(Grouping grouping) {
break;
}
} catch (Exception e) {
- //ignored
+ // ignored
}
- //Fall through if not supported
+ // Fall through if not supported
default:
- LOG.warn("{} is not supported for replay of a topology. Using SHUFFLE", thriftType);
+ LOG.warn("{} is not supported for replay of a topology. Using SHUFFLE",
+ thriftType);
break;
}
return withGroupingType(group);
}
public InputStream build() {
- return new InputStream(fromComponent, toComponent, id, execTime, processTime, groupingType);
+ return new InputStream(fromComponent, toComponent, id, execTime, processTime,
+ groupingType);
}
}
/**
* Create a new input stream to a bolt.
+ *
* @param fromComponent the source component of the stream.
* @param id the id of the stream
* @param execTime exec time stats
* @param processTime process time stats
*/
- public InputStream(String fromComponent, String toComponent, String id, NormalDistStats execTime,
+ public InputStream(String fromComponent, String toComponent, String id,
+ NormalDistStats execTime,
NormalDistStats processTime, GroupingType groupingType) {
this.fromComponent = fromComponent;
this.toComponent = toComponent;
@@ -228,6 +237,7 @@ public InputStream(String fromComponent, String toComponent, String id, NormalDi
/**
* Get the global stream id for this input stream.
+ *
* @return the GlobalStreamId for this input stream.
*/
public synchronized GlobalStreamId gsid() {
@@ -239,25 +249,30 @@ public synchronized GlobalStreamId gsid() {
/**
* Remap the names of components.
+ *
* @param remappedComponents old name to new name of components.
* @param remappedStreams old ID to new ID of streams.
* @return a modified version of this with names remapped.
*/
- public InputStream remap(Map remappedComponents, Map remappedStreams) {
+ public InputStream remap(Map remappedComponents, Map remappedStreams) {
String remapTo = remappedComponents.get(toComponent);
String remapFrom = remappedComponents.get(fromComponent);
GlobalStreamId remapStreamId = remappedStreams.get(gsid());
- return new InputStream(remapFrom, remapTo, remapStreamId.get_streamId(), execTime, processTime, groupingType);
+ return new InputStream(remapFrom, remapTo, remapStreamId.get_streamId(), execTime,
+ processTime, groupingType);
}
/**
* Replace all SHUFFLE groupings with LOCAL_OR_SHUFFLE.
+ *
* @return a modified copy of this
*/
public InputStream replaceShuffleWithLocalOrShuffle() {
if (groupingType != GroupingType.SHUFFLE) {
return this;
}
- return new InputStream(fromComponent, toComponent, id, execTime, processTime, GroupingType.LOCAL_OR_SHUFFLE);
+ return new InputStream(fromComponent, toComponent, id, execTime, processTime,
+ GroupingType.LOCAL_OR_SHUFFLE);
}
}
diff --git a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/LoadBolt.java b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/LoadBolt.java
index 06f1e5a25ac..388c759ec9e 100644
--- a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/LoadBolt.java
+++ b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/LoadBolt.java
@@ -7,9 +7,9 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
+ *
Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
@@ -58,7 +58,8 @@ public void add(InputStream inputStream) {
}
@Override
- public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) {
+ public void prepare(Map topoConf, TopologyContext context,
+ OutputCollector collector) {
outputStreams = Collections.unmodifiableList(outputStreamStats.stream()
.map((ss) -> new OutputStreamEngine(ss)).collect(Collectors.toList()));
this.collector = collector;
diff --git a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/LoadCompConf.java b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/LoadCompConf.java
index 33460a8d882..f2b490524ae 100644
--- a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/LoadCompConf.java
+++ b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/LoadCompConf.java
@@ -7,9 +7,9 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
+ *
Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
@@ -39,6 +39,7 @@ public class LoadCompConf {
/**
* Parse the LoadCompConf from a config Map.
+ *
* @param conf the map holding the config for a LoadCompConf.
* @return the parsed object.
*/
@@ -57,13 +58,15 @@ public static LoadCompConf fromConf(Map conf) {
SlowExecutorPattern slp = null;
if (conf.containsKey("slowExecutorPattern")) {
- slp = SlowExecutorPattern.fromConf((Map) conf.get("slowExecutorPattern"));
+ slp = SlowExecutorPattern.fromConf((Map) conf
+ .get("slowExecutorPattern"));
}
return new LoadCompConf(id, parallelism, streams, memoryMb, cpuPercent, slp);
}
/**
* Build a config map for this object.
+ *
* @return the config map.
*/
public Map toConf() {
@@ -92,23 +95,28 @@ public Map toConf() {
/**
* Chenge the name of components and streams according to the parameters passed in.
+ *
* @param remappedComponents original component name to new component name.
* @param remappedStreams original stream id to new stream id.
* @return a copy of this with the values remapped.
*/
- public LoadCompConf remap(Map remappedComponents, Map remappedStreams) {
+ public LoadCompConf remap(Map remappedComponents, Map remappedStreams) {
String remappedId = remappedComponents.get(id);
List remappedOutStreams = (streams == null) ? null :
streams.stream()
.map((orig) -> orig.remap(id, remappedStreams))
.collect(Collectors.toList());
- return new LoadCompConf(remappedId, parallelism, remappedOutStreams, cpuLoad, memoryLoad, slp);
+ return new LoadCompConf(remappedId, parallelism, remappedOutStreams, cpuLoad, memoryLoad,
+ slp);
}
/**
* Scale the parallelism of this component by v. The aggregate throughput will be the same.
- * The parallelism will be rounded up to the next largest whole number. Parallelism will always be at least 1.
+ * The parallelism will be rounded up to the next largest whole number. Parallelism will always
+ * be at least 1.
+ *
* @param v 1.0 is not change 0.5 is drop the parallelism by half.
* @return a copy of this with the parallelism adjusted.
*/
@@ -117,24 +125,29 @@ public LoadCompConf scaleParallel(double v) {
}
/**
- * Set the parallelism of this component, and adjust the throughput so in aggregate it stays the same.
+ * Set the parallelism of this component, and adjust the throughput so in aggregate it stays the
+ * same.
+ *
* @param newParallelism the new parallelism to set.
* @return a copy of this with the adjustments made.
*/
public LoadCompConf setParallel(int newParallelism) {
- //We need to adjust the throughput accordingly (so that it stays the same in aggregate)
+ // We need to adjust the throughput accordingly (so that it stays the same in aggregate)
double throughputAdjustment = ((double) parallelism) / newParallelism;
- return new LoadCompConf(id, newParallelism, streams, cpuLoad, memoryLoad, slp).scaleThroughput(throughputAdjustment);
+ return new LoadCompConf(id, newParallelism, streams, cpuLoad, memoryLoad, slp)
+ .scaleThroughput(throughputAdjustment);
}
/**
* Scale the throughput of this component.
+ *
* @param v 1.0 is unchanged 0.5 will cut the throughput in half.
* @return a copy of this with the adjustments made.
*/
public LoadCompConf scaleThroughput(double v) {
if (streams != null) {
- List newStreams = streams.stream().map((s) -> s.scaleThroughput(v)).collect(Collectors.toList());
+ List newStreams = streams.stream().map((s) -> s.scaleThroughput(v))
+ .collect(Collectors.toList());
return new LoadCompConf(id, parallelism, newStreams, cpuLoad, memoryLoad, slp);
} else {
return this;
@@ -143,6 +156,7 @@ public LoadCompConf scaleThroughput(double v) {
/**
* Override the SlowExecutorPattern with a new one.
+ *
* @param slp the new pattern or null if you don't want it to change
* @return a copy of this with the adjustments made.
*/
@@ -156,6 +170,7 @@ public LoadCompConf overrideSlowExecutorPattern(SlowExecutorPattern slp) {
/**
* Compute the total amount of all messages emitted in all streams per second.
+ *
* @return the sum of all messages emitted per second.
*/
public double getAllEmittedAggregate() {
@@ -202,6 +217,7 @@ public List getStreams() {
/**
* Add in a single OutputStream to this component.
+ *
* @param stream the stream to add
* @return this
*/
@@ -240,11 +256,13 @@ public LoadCompConf build() {
/**
* Create a new LoadCompConf with the given values.
+ *
* @param id the id of the component.
* @param parallelism tha parallelism of the component.
* @param streams the output streams of the component.
*/
- public LoadCompConf(String id, int parallelism, List streams, double cpuLoad, double memoryLoad,
+ public LoadCompConf(String id, int parallelism, List streams, double cpuLoad,
+ double memoryLoad,
SlowExecutorPattern slp) {
this.id = id;
if (id == null) {
diff --git a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/LoadMetricsServer.java b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/LoadMetricsServer.java
index f02c407093a..b3ccd06ac9a 100644
--- a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/LoadMetricsServer.java
+++ b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/LoadMetricsServer.java
@@ -7,9 +7,9 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
+ *
Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
@@ -113,12 +113,14 @@ public static class Measurements {
/**
* Constructor.
+ *
* @param histo latency histogram.
* @param userMs user CPU in ms.
* @param sysMs system CPU in ms.
* @param gcMs GC CPU in ms.
*/
- public Measurements(long uptimeSecs, long acked, long timeWindow, long failed, Histogram histo,
+ public Measurements(long uptimeSecs, long acked, long timeWindow, long failed,
+ Histogram histo,
double userMs, double sysMs, double gcMs, long memBytes, Set topologyIds,
long workers, long executors, long hosts, Map congested, long skippedMaxSpoutMs,
double uiCompleteLatency) {
@@ -164,6 +166,7 @@ public Measurements() {
/**
* Add other to this.
+ *
* @param other meaurements to add in.
*/
public void add(Measurements other) {
@@ -319,7 +322,7 @@ private static class NoCloseOutputStream extends FilterOutputStream {
@Override
public void close() {
- //NOOP on purpose
+ // NOOP on purpose
}
}
@@ -332,7 +335,8 @@ abstract static class FileReporter implements MetricResultsReporter {
this(null, Collections.emptyMap(), allExtractors);
}
- FileReporter(String path, Map query, Map allExtractors)
+ FileReporter(String path, Map query, Map allExtractors)
throws FileNotFoundException {
boolean append = Boolean.parseBoolean(query.getOrDefault("append", "false"));
boolean tee = Boolean.parseBoolean(query.getOrDefault("tee", "false"));
@@ -356,14 +360,14 @@ abstract static class FileReporter implements MetricResultsReporter {
includesSysOutOrError = true;
}
this.out = new PrintStream(out);
- //Copy it in case we want to modify it
+ // Copy it in case we want to modify it
this.allExtractors = new LinkedHashMap<>(allExtractors);
this.includesSysOutOrError = includesSysOutOrError;
}
@Override
public void start() {
- //NOOP
+ // NOOP
}
@Override
@@ -420,7 +424,7 @@ public void finish(List allTime) throws Exception {
private static final Map NAMED_EXTRACTORS;
static {
- //Perhaps there is a better way to do this???
+ // Perhaps there is a better way to do this???
LinkedHashMap tmp = new LinkedHashMap<>();
tmp.put("start_time", new MetricExtractor((m, unit) -> m.startTime(), "s"));
tmp.put("end_time", new MetricExtractor((m, unit) -> m.endTime(), "s"));
@@ -429,7 +433,8 @@ public void finish(List allTime) throws Exception {
tmp.put("99%ile", new MetricExtractor((m, unit) -> m.getLatencyAtPercentile(99.0, unit)));
tmp.put("99.9%ile", new MetricExtractor((m, unit) -> m.getLatencyAtPercentile(99.9, unit)));
tmp.put("cores", new MetricExtractor(
- (m, unit) -> (m.getSysTime(TimeUnit.SECONDS) + m.getUserTime(TimeUnit.SECONDS)) / m.getTimeWindow(),
+ (m, unit) -> (m.getSysTime(TimeUnit.SECONDS) + m.getUserTime(TimeUnit.SECONDS)) / m
+ .getTimeWindow(),
""));
tmp.put("mem", new MetricExtractor((m, unit) -> m.getMemMb(), "MB"));
tmp.put("failed", new MetricExtractor((m, unit) -> m.getFailed(), ""));
@@ -457,9 +462,12 @@ public void finish(List allTime) throws Exception {
+ " " + System.getProperty("java.version"), ""));
tmp.put("os_arch", new MetricExtractor((m, unit) -> System.getProperty("os.arch"), ""));
tmp.put("os_name", new MetricExtractor((m, unit) -> System.getProperty("os.name"), ""));
- tmp.put("os_version", new MetricExtractor((m, unit) -> System.getProperty("os.version"), ""));
- tmp.put("config_override", new MetricExtractor((m, unit) -> Utils.readCommandLineOpts(), ""));
- tmp.put("ui_complete_latency", new MetricExtractor((m, unit) -> m.getUiCompleteLatency(unit)));
+ tmp.put("os_version", new MetricExtractor((m, unit) -> System.getProperty("os.version"),
+ ""));
+ tmp.put("config_override", new MetricExtractor((m, unit) -> Utils.readCommandLineOpts(),
+ ""));
+ tmp.put("ui_complete_latency", new MetricExtractor((m, unit) -> m
+ .getUiCompleteLatency(unit)));
NAMED_EXTRACTORS = Collections.unmodifiableMap(tmp);
}
@@ -504,20 +512,24 @@ abstract static class ColumnsFileReporter extends FileReporter {
protected final int precision;
protected String doubleFormat;
- ColumnsFileReporter(String path, Map query, Map extractorsMap)
+ ColumnsFileReporter(String path, Map query, Map extractorsMap)
throws FileNotFoundException {
this(path, query, extractorsMap, null);
}
- ColumnsFileReporter(String path, Map query, Map extractorsMap,
+ ColumnsFileReporter(String path, Map query, Map extractorsMap,
String defaultPreceision) throws FileNotFoundException {
super(path, query, extractorsMap);
targetUnit = UNIT_MAP.get(query.getOrDefault("time", "MILLISECONDS").toUpperCase());
if (targetUnit == null) {
- throw new IllegalArgumentException(query.get("time") + " is not a supported time unit");
+ throw new IllegalArgumentException(query.get("time")
+ + " is not a supported time unit");
}
if (query.containsKey("columns")) {
- List extractors = handleExtractorCleanup(Arrays.asList(query.get("columns").split("\\s*,\\s*")));
+ List extractors = handleExtractorCleanup(Arrays.asList(query.get("columns")
+ .split("\\s*,\\s*")));
HashSet notFound = new HashSet<>(extractors);
notFound.removeAll(allExtractors.keySet());
@@ -526,17 +538,19 @@ abstract static class ColumnsFileReporter extends FileReporter {
}
this.extractors = extractors;
} else {
- //Wrapping it makes it mutable
+ // Wrapping it makes it mutable
extractors = new ArrayList<>(Arrays.asList("start_time", "end_time", "rate",
"mean", "99%ile", "99.9%ile", "cores", "mem", "failed", "ids", "congested"));
}
if (query.containsKey("extraColumns")) {
List moreExtractors =
- handleExtractorCleanup(Arrays.asList(query.get("extraColumns").split("\\s*,\\s*")));
+ handleExtractorCleanup(Arrays.asList(query.get("extraColumns")
+ .split("\\s*,\\s*")));
for (String extractor : moreExtractors) {
if (!allExtractors.containsKey(extractor)) {
- throw new IllegalArgumentException(extractor + " is not a supported column");
+ throw new IllegalArgumentException(extractor
+ + " is not a supported column");
}
if (!extractors.contains(extractor)) {
extractors.add(extractor);
@@ -564,8 +578,10 @@ protected List handleExtractorCleanup(List orig) {
allExtractors.put(extractor, new MetricExtractor((m, t) -> confValue, ""));
ret.add(extractor);
} else if (extractor.endsWith("%ile")) {
- double number = Double.valueOf(extractor.substring(0, extractor.length() - "%ile".length()));
- allExtractors.put(extractor, new MetricExtractor((m, t) -> m.getLatencyAtPercentile(number, t)));
+ double number = Double.valueOf(extractor.substring(0, extractor
+ .length() - "%ile".length()));
+ allExtractors.put(extractor, new MetricExtractor((m, t) -> m
+ .getLatencyAtPercentile(number, t)));
ret.add(extractor);
} else if ("*".equals(extractor)) {
ret.addAll(allExtractors.keySet());
@@ -585,21 +601,23 @@ protected String format(Object o) {
}
}
-
static class FixedWidthReporter extends ColumnsFileReporter {
public final String longFormat;
public final String stringFormat;
- FixedWidthReporter(String path, Map query, Map extractorsMap)
+ FixedWidthReporter(String path, Map query, Map extractorsMap)
throws FileNotFoundException {
super(path, query, extractorsMap, "3");
- int columnWidth = Integer.parseInt(query.getOrDefault("columnWidth", "15")) - 1; //Always have a space in between
+ int columnWidth = Integer.parseInt(query.getOrDefault("columnWidth",
+ "15")) - 1; // Always have a space in between
doubleFormat = "%," + columnWidth + "." + precision + "f";
longFormat = "%," + columnWidth + "d";
stringFormat = "%" + columnWidth + "s";
}
- FixedWidthReporter(Map allExtractors) throws FileNotFoundException {
+ FixedWidthReporter(Map allExtractors) throws FileNotFoundException {
this(null, Collections.emptyMap(), allExtractors);
}
@@ -652,7 +670,8 @@ public void reportWindow(Measurements m, List allTime) {
static class SepValReporter extends ColumnsFileReporter {
private final String separator;
- SepValReporter(String separator, String path, Map query, Map extractorsMap)
+ SepValReporter(String separator, String path, Map query, Map extractorsMap)
throws FileNotFoundException {
super(path, query, extractorsMap);
this.separator = separator;
@@ -707,13 +726,15 @@ static class LegacyReporter extends FileReporter {
targetUnitOverride = null;
}
- LegacyReporter(String path, Map query, Map allExtractors)
+ LegacyReporter(String path, Map query, Map allExtractors)
throws FileNotFoundException {
super(path, query, allExtractors);
if (query.containsKey("time")) {
targetUnitOverride = UNIT_MAP.get(query.get("time").toUpperCase());
if (targetUnitOverride == null) {
- throw new IllegalArgumentException(query.get("time") + " is not a supported time unit");
+ throw new IllegalArgumentException(query.get("time")
+ + " is not a supported time unit");
}
} else {
targetUnitOverride = null;
@@ -749,10 +770,11 @@ public void reportWindow(Measurements m, List allTime) {
/**
* Add Command line options for configuring the output of this.
+ *
* @param options command line options to update
*/
public static void addCommandLineOptions(Options options) {
- //We want to be able to select the measurement interval
+ // We want to be able to select the measurement interval
// reporting window (We don't need 3 different reports)
// We want to be able to specify format (and configs specific to the format)
// With perhaps defaults overall
@@ -760,7 +782,8 @@ public static void addCommandLineOptions(Options options) {
.longOpt("report-interval")
.hasArg()
.argName("SECS")
- .desc("How long in between reported metrics. Will be rounded up to the next 10 sec boundary.\n"
+ .desc("How long in between reported metrics. Will be rounded up to the next 10 sec "
+ + "boundary.\n"
+ "default " + DEFAULT_REPORT_INTERVAL)
.build());
@@ -768,7 +791,8 @@ public static void addCommandLineOptions(Options options) {
.longOpt("report-window")
.hasArg()
.argName("SECS")
- .desc("How long of a rolling window should be in each report. Will be rounded up to the next report interval boundary.\n"
+ .desc("How long of a rolling window should be in each report. Will be rounded up to "
+ + "the next report interval boundary.\n"
+ "default " + DEFAULT_WINDOW_INTERVAL)
.build());
@@ -781,7 +805,8 @@ public static void addCommandLineOptions(Options options) {
+ "LEGACY - (write things out in the legacy format)\n"
+ "TSV - tab separated values\n"
+ "CSV - comma separated values\n"
- + "PATH and OPTIONS are each optional but must be marked with a ':' or '?' separator respectively.")
+ + "PATH and OPTIONS are each optional but must be marked with a ':' or '?' "
+ + "separator respectively.")
.build());
}
@@ -798,7 +823,8 @@ public static void addCommandLineOptions(Options options) {
private final AtomicLong gcMs = new AtomicLong(0);
private final AtomicLong skippedMaxSpoutMs = new AtomicLong(0);
private final ConcurrentHashMap memoryBytes = new ConcurrentHashMap<>();
- private final AtomicReference> congested = new AtomicReference<>(new ConcurrentHashMap<>());
+ private final AtomicReference> congested =
+ new AtomicReference<>(new ConcurrentHashMap<>());
private final List reporters;
private long prevAcked = 0;
private long prevFailed = 0;
@@ -808,7 +834,8 @@ public static void addCommandLineOptions(Options options) {
private final LinkedList allCombined = new LinkedList<>();
- LoadMetricsServer(Map conf, CommandLine commandLine, Map parameterMetrics) throws URISyntaxException,
+ LoadMetricsServer(Map conf, CommandLine commandLine, Map parameterMetrics) throws URISyntaxException,
FileNotFoundException {
super(conf);
Map allExtractors = new LinkedHashMap<>(NAMED_EXTRACTORS);
@@ -829,7 +856,8 @@ public static void addCommandLineOptions(Options options) {
for (String reporterString : commandLine.getOptionValues("reporter")) {
Matcher m = REPORTER_PATTERN.matcher(reporterString);
if (!m.matches()) {
- throw new IllegalArgumentException(reporterString + " does not look like it is a reporter");
+ throw new IllegalArgumentException(reporterString
+ + " does not look like it is a reporter");
}
String type = m.group("type");
String path = m.group("path");
@@ -898,12 +926,14 @@ private void finishMetricsOutput() throws Exception {
/**
* Monitor the list of topologies for the given time frame.
+ *
* @param execTimeMins how long to monitor for
* @param client the client to use when monitoring
* @param topoNames the names of the topologies to monitor
* @throws Exception on any error
*/
- public void monitorFor(double execTimeMins, Nimbus.Iface client, Collection topoNames) throws Exception {
+ public void monitorFor(double execTimeMins, Nimbus.Iface client,
+ Collection topoNames) throws Exception {
startMetricsOutput();
long iterations = (long) ((execTimeMins * 60) / reportIntervalSecs);
for (int i = 0; i < iterations; i++) {
@@ -952,7 +982,8 @@ private void outputMetrics(Nimbus.Iface client, Collection names) throws
}
}
}
- Double latency = tpi.get_topology_stats().get_window_to_complete_latencies_ms().get(":all-time");
+ Double latency = tpi.get_topology_stats().get_window_to_complete_latencies_ms()
+ .get(":all-time");
Long latAcked = tpi.get_topology_stats().get_window_to_acked().get(":all-time");
if (latency != null && latAcked != null) {
totalLatCount += latAcked;
@@ -980,8 +1011,10 @@ private void outputMetrics(Nimbus.Iface client, Collection names) throws
long skippedMaxSpout = skippedMaxSpoutMs.getAndSet(0);
long memBytes = readMemory();
- allCombined.add(new Measurements(uptime, ackedThisTime, thisTime, failedThisTime, copy, user, sys, gc, memBytes,
- ids, workers.size(), executors, hosts.size(), congested.getAndSet(new ConcurrentHashMap<>()), skippedMaxSpout,
+ allCombined.add(new Measurements(uptime, ackedThisTime, thisTime, failedThisTime, copy,
+ user, sys, gc, memBytes,
+ ids, workers.size(), executors, hosts.size(), congested
+ .getAndSet(new ConcurrentHashMap<>()), skippedMaxSpout,
totalLatMs / totalLatCount));
Measurements inWindow = Measurements.combine(allCombined, null, windowLength);
for (MetricResultsReporter reporter : reporters) {
@@ -991,7 +1024,8 @@ private void outputMetrics(Nimbus.Iface client, Collection names) throws
@Override
@SuppressWarnings("unchecked")
- public void handle(IMetricsConsumer.TaskInfo taskInfo, Collection dataPoints, String topologyId) {
+ public void handle(IMetricsConsumer.TaskInfo taskInfo,
+ Collection dataPoints, String topologyId) {
String worker = taskInfo.srcWorkerHost + ":" + taskInfo.srcWorkerPort;
for (IMetricsConsumer.DataPoint dp : dataPoints) {
if (dp.name.startsWith("comp-lat-histo") && dp.value instanceof Histogram) {
@@ -1045,7 +1079,8 @@ public void handle(IMetricsConsumer.TaskInfo taskInfo, Collection= 0.8) {
congested.get().put(
topologyId + ":" + taskInfo.srcComponentId + ":" + taskInfo.srcTaskId,
diff --git a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/LoadSpout.java b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/LoadSpout.java
index 1fc006eb800..6db5e9e6b37 100644
--- a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/LoadSpout.java
+++ b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/LoadSpout.java
@@ -7,9 +7,9 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
+ *
Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
@@ -44,7 +44,7 @@ private static class OutputStreamEngineWithHisto extends OutputStreamEngine {
OutputStreamEngineWithHisto(OutputStream stats, TopologyContext context) {
super(stats);
histogram = new HistogramMetric(3600000000000L, 3);
- //TODO: perhaps we can adjust the frequency later...
+ // TODO: perhaps we can adjust the frequency later...
context.registerMetric("comp-lat-histo-" + stats.id, histogram, 10);
}
}
@@ -70,13 +70,14 @@ public void done() {
private final List streamStats;
private List streams;
private SpoutOutputCollector collector;
- //This is an attempt to give all of the streams an equal opportunity to emit something.
+ // This is an attempt to give all of the streams an equal opportunity to emit something.
private long nextStreamCounter = 0;
private final int numStreams;
private final Queue replays = new ArrayDeque<>();
/**
* Create a simple load spout with just a set rate per second on the default stream.
+ *
* @param ratePerSecond the rate to send messages at.
*/
public LoadSpout(double ratePerSecond) {
@@ -94,9 +95,11 @@ public LoadSpout(LoadCompConf conf) {
}
@Override
- public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
+ public void open(Map conf, TopologyContext context,
+ SpoutOutputCollector collector) {
streams = Collections.unmodifiableList(streamStats.stream()
- .map((ss) -> new OutputStreamEngineWithHisto(ss, context)).collect(Collectors.toList()));
+ .map((ss) -> new OutputStreamEngineWithHisto(ss, context)).collect(Collectors
+ .toList()));
this.collector = collector;
}
diff --git a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/NormalDistStats.java b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/NormalDistStats.java
index 12dc6c5efc2..1987e3ae84f 100644
--- a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/NormalDistStats.java
+++ b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/NormalDistStats.java
@@ -7,9 +7,9 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
+ *
Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
@@ -40,6 +40,7 @@ public class NormalDistStats implements Serializable {
/**
* Read the stats from a config.
+ *
* @param conf the config.
* @return the corresponding stats.
*/
@@ -49,6 +50,7 @@ public static NormalDistStats fromConf(Map conf) {
/**
* Read the stats from a config.
+ *
* @param conf the config.
* @param def the default mean.
* @return the corresponding stats.
@@ -66,6 +68,7 @@ public static NormalDistStats fromConf(Map conf, Double def) {
/**
* Return this as a config.
+ *
* @return the config version of this.
*/
public Map toConf() {
@@ -78,11 +81,13 @@ public Map toConf() {
}
/**
- * Create an instance of this from a list of values. The metrics will be computed from the values.
+ * Create an instance of this from a list of values. The metrics will be computed from the
+ * values.
+ *
* @param values the values to compute metrics from.
*/
public NormalDistStats(List values) {
- //Compute the stats for these and save them
+ // Compute the stats for these and save them
double min = values.isEmpty() ? 0.0 : values.get(0);
double max = values.isEmpty() ? 0.0 : values.get(0);
double sum = 0.0;
@@ -110,6 +115,7 @@ public NormalDistStats(List values) {
/**
* A Constructor for the pre computed stats.
+ *
* @param mean the mean of the values.
* @param stddev the standard deviation of the values.
* @param min the min of the values.
@@ -124,6 +130,7 @@ public NormalDistStats(double mean, double stddev, double min, double max) {
/**
* Generate a random number that follows the statistical distribution.
+ *
* @param rand the random number generator to use
* @return the next number that should follow the statistical distribution.
*/
@@ -137,8 +144,10 @@ public String toString() {
}
/**
- * Scale the stats by v. This is not scaling everything proportionally. We don't want the stddev to increase
+ * Scale the stats by v. This is not scaling everything proportionally. We don't want the stddev
+ * to increase
* so instead we scale the mean and shift everything up or down by the same amount.
+ *
* @param v the amount to scale by 1.0 is nothing 0.5 is half.
* @return a copy of this with the needed adjustments.
*/
diff --git a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/OutputStream.java b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/OutputStream.java
index 33f894f1200..7acea95d098 100644
--- a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/OutputStream.java
+++ b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/OutputStream.java
@@ -7,9 +7,9 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
+ *
Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
@@ -27,13 +27,14 @@
* A set of measurements about a stream so we can statistically reproduce it.
*/
public class OutputStream implements Serializable {
- //The global stream id is this + the from component it must be a part of.
+ // The global stream id is this + the from component it must be a part of.
public final String id;
public final NormalDistStats rate;
public final boolean areKeysSkewed;
/**
* Create an output stream from a config.
+ *
* @param conf the config to read from.
* @return the read OutputStream.
*/
@@ -46,6 +47,7 @@ public static OutputStream fromConf(Map conf) {
/**
* Convert this to a conf.
+ *
* @return the conf.
*/
public Map toConf() {
@@ -104,6 +106,7 @@ public OutputStream build() {
/**
* Create a new stream with stats.
+ *
* @param id the id of the stream
* @param rate the rate of tuples being emitted on this stream
* @param areKeysSkewed true if keys are skewed else false. For skewed keys
diff --git a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/OutputStreamEngine.java b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/OutputStreamEngine.java
index d8b52034ec2..5830ece1d4c 100644
--- a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/OutputStreamEngine.java
+++ b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/OutputStreamEngine.java
@@ -7,9 +7,9 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
+ *
Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
@@ -23,8 +23,8 @@
/**
* Provides an API to simulate the output of a stream.
- *
- * Right now it is just rate, but in the future we expect to do data skew as well...
+ *
+ *
Right now it is just rate, but in the future we expect to do data skew as well...
*
*/
public class OutputStreamEngine {
@@ -33,7 +33,8 @@ public class OutputStreamEngine {
private static final String[] KEYS = new String[2048];
static {
- //We get a new random number and seed it to make sure that runs are consistent where possible.
+ // We get a new random number and seed it to make sure that runs are consistent where
+ // possible.
Random r = new Random(KEYS.length);
for (int i = 0; i < KEYS.length; i++) {
KEYS[i] = String.valueOf(r.nextDouble());
@@ -51,13 +52,14 @@ public class OutputStreamEngine {
/**
* Create an engine that can simulate the given stats.
+ *
* @param stats the stats to follow
*/
public OutputStreamEngine(OutputStream stats) {
this.stats = stats;
rand = ThreadLocalRandom.current();
selectNewRate();
- //Start emitting right now
+ // Start emitting right now
nextEmitTime = System.nanoTime();
nextRateRandomizeTime = nextEmitTime + UPDATE_RATE_PERIOD_NS;
emitsLeft = emitAmount;
@@ -70,7 +72,7 @@ private void selectNewRate() {
periodNano = Math.max(1, (long) (NANO_PER_SEC / ratePerSecond));
emitAmount = Math.max(1, (long) ((ratePerSecond / NANO_PER_SEC) * periodNano));
} else {
- //if it is is 0 or less it really is 1 per 10 seconds.
+ // if it is is 0 or less it really is 1 per 10 seconds.
periodNano = (long) NANO_PER_SEC * 10;
emitAmount = 1;
}
@@ -78,6 +80,7 @@ private void selectNewRate() {
/**
* Should we emit or not.
+ *
* @return the start time of the message, or null of nothing should be emitted.
*/
public Long shouldEmit() {
@@ -88,7 +91,7 @@ public Long shouldEmit() {
}
if (nextRateRandomizeTime <= time) {
- //Once every UPDATE_RATE_PERIOD_NS
+ // Once every UPDATE_RATE_PERIOD_NS
selectNewRate();
nextRateRandomizeTime = nextEmitTime + UPDATE_RATE_PERIOD_NS;
}
@@ -102,14 +105,17 @@ public Long shouldEmit() {
/**
* Get the next key to emit.
+ *
* @return the key that should be emitted.
*/
public String nextKey() {
int keyIndex;
if (stats.areKeysSkewed) {
- //We set the stddev of the skewed keys to be 1/5 of the length, but then we use the absolute value
+ // We set the stddev of the skewed keys to be 1/5 of the length, but then we use the
+ // absolute value
// of that so everything is skewed towards 0
- keyIndex = Math.min(KEYS.length - 1, Math.abs((int) (rand.nextGaussian() * KEYS.length / 5)));
+ keyIndex = Math.min(KEYS.length - 1, Math.abs((int) (rand
+ .nextGaussian() * KEYS.length / 5)));
} else {
keyIndex = rand.nextInt(KEYS.length);
}
diff --git a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/ScopedTopologySet.java b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/ScopedTopologySet.java
index f7e79121453..20cde608aee 100644
--- a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/ScopedTopologySet.java
+++ b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/ScopedTopologySet.java
@@ -7,9 +7,9 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
+ *
Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
@@ -45,6 +45,7 @@ public class ScopedTopologySet extends HashSet implements AutoCloseable
/**
* Constructor.
+ *
* @param client the client used to kill the topologies when this exist.
*/
public ScopedTopologySet(Nimbus.Iface client) {
diff --git a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/SlowExecutorPattern.java b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/SlowExecutorPattern.java
index d2c3ac502c3..cd9d7a966a5 100644
--- a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/SlowExecutorPattern.java
+++ b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/SlowExecutorPattern.java
@@ -7,9 +7,9 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
+ *
Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
@@ -26,22 +26,26 @@
import org.apache.storm.utils.ObjectReader;
/**
- * A repeating pattern of skewedness in processing times. This is used to simulate an executor that slows down.
+ * A repeating pattern of skewedness in processing times. This is used to simulate an executor that
+ * slows down.
*/
public class SlowExecutorPattern implements Serializable {
- private static final Pattern PARSER = Pattern.compile("\\s*(?[^:]+)\\s*(?::\\s*(?[0-9]+))?\\s*");
+ private static final Pattern PARSER = Pattern
+ .compile("\\s*(?[^:]+)\\s*(?::\\s*(?[0-9]+))?\\s*");
public final double maxSlownessMs;
public final int count;
/**
* Parses a string (command line) representation of "<SLOWNESS>(:<COUNT>)?".
+ *
* @param strRepresentation the string representation to parse
* @return the corresponding SlowExecutorPattern.
*/
public static SlowExecutorPattern fromString(String strRepresentation) {
Matcher m = PARSER.matcher(strRepresentation);
if (!m.matches()) {
- throw new IllegalArgumentException(strRepresentation + " is not in the form (:)?");
+ throw new IllegalArgumentException(strRepresentation
+ + " is not in the form (:)?");
}
double slownessMs = Double.valueOf(m.group("slowness"));
String c = m.group("count");
@@ -51,6 +55,7 @@ public static SlowExecutorPattern fromString(String strRepresentation) {
/**
* Creates a SlowExecutorPattern from a Map config.
+ *
* @param conf the conf to parse.
* @return the corresponding SlowExecutorPattern.
*/
@@ -62,6 +67,7 @@ public static SlowExecutorPattern fromConf(Map conf) {
/**
* Convert this to a Config map.
+ *
* @return the corresponding Config map to this.
*/
public Map toConf() {
diff --git a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/ThroughputVsLatency.java b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/ThroughputVsLatency.java
index 17aa8cf9441..b730ce6f395 100644
--- a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/ThroughputVsLatency.java
+++ b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/ThroughputVsLatency.java
@@ -7,9 +7,9 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
+ *
Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
@@ -69,6 +69,7 @@ public static class FastRandomSentenceSpout extends LoadSpout {
/**
* Constructor.
+ *
* @param ratePerSecond the rate to emite tuples at.
*/
public FastRandomSentenceSpout(long ratePerSecond) {
@@ -142,6 +143,7 @@ public void declareOutputFields(OutputFieldsDeclarer declarer) {
/**
* The main entry point for ThroughputVsLatency.
+ *
* @param args the command line args
* @throws Exception on any error.
*/
@@ -155,13 +157,15 @@ public static void main(String[] args) throws Exception {
.longOpt("test-time")
.argName("MINS")
.hasArg()
- .desc("How long to run the tests for in mins (defaults to " + TEST_EXECUTE_TIME_DEFAULT + ")")
+ .desc("How long to run the tests for in mins (defaults to " + TEST_EXECUTE_TIME_DEFAULT
+ + ")")
.build());
options.addOption(Option.builder()
.longOpt("rate")
.argName("SENTENCES/SEC")
.hasArg()
- .desc("How many sentences per second to run. (defaults to " + DEFAULT_RATE_PER_SECOND + ")")
+ .desc("How many sentences per second to run. (defaults to " + DEFAULT_RATE_PER_SECOND
+ + ")")
.build());
options.addOption(Option.builder()
.longOpt("name")
@@ -185,7 +189,8 @@ public static void main(String[] args) throws Exception {
.longOpt("splitter-imbalance")
.argName("MS(:COUNT)?")
.hasArg()
- .desc("The number of ms that the first COUNT splitters will wait before processing. This creates an imbalance "
+ .desc("The number of ms that the first COUNT splitters will wait before processing. "
+ + "This creates an imbalance "
+ "that helps test load aware groupings (defaults to 0:1)")
.build());
options.addOption(Option.builder()
@@ -256,7 +261,7 @@ public static void main(String[] args) throws Exception {
conf.registerMetricsConsumer(HttpForwardingMetricsConsumer.class, url, 1);
Map workerMetrics = new HashMap<>();
if (!NimbusClient.isLocalOverride()) {
- //sigar uses JNI and does not work in local mode
+ // sigar uses JNI and does not work in local mode
workerMetrics.put("CPU", "org.apache.storm.metrics.sigar.CPUMetric");
}
conf.put(Config.TOPOLOGY_WORKER_METRICS, workerMetrics);
@@ -264,10 +269,12 @@ public static void main(String[] args) throws Exception {
TopologyBuilder builder = new TopologyBuilder();
- builder.setSpout("spout", new FastRandomSentenceSpout((long) ratePerSecond / numSpouts), numSpouts);
+ builder.setSpout("spout", new FastRandomSentenceSpout((long) ratePerSecond / numSpouts),
+ numSpouts);
builder.setBolt("split", new SplitSentence(slowness), numSplits)
.shuffleGrouping("spout");
- builder.setBolt("count", new WordCount(), numCounts).fieldsGrouping("split", new Fields("word"));
+ builder.setBolt("count", new WordCount(), numCounts).fieldsGrouping("split",
+ new Fields("word"));
int exitStatus = -1;
try (ScopedTopologySet topologyNames = new ScopedTopologySet(client.getClient())) {
diff --git a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/TopologyLoadConf.java b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/TopologyLoadConf.java
index 3765f21d13d..a425c314e54 100644
--- a/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/TopologyLoadConf.java
+++ b/examples/storm-loadgen/src/main/java/org/apache/storm/loadgen/TopologyLoadConf.java
@@ -7,9 +7,9 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
+ *
Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
@@ -47,7 +47,8 @@
*/
public class TopologyLoadConf {
private static final Logger LOG = LoggerFactory.getLogger(TopologyLoadConf.class);
- static final Set IMPORTANT_CONF_KEYS = Collections.unmodifiableSet(new HashSet(Arrays.asList(
+ static final Set IMPORTANT_CONF_KEYS = Collections.unmodifiableSet(new HashSet(Arrays
+ .asList(
Config.TOPOLOGY_WORKERS,
Config.TOPOLOGY_ACKER_EXECUTORS,
Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT,
@@ -83,6 +84,7 @@ public class TopologyLoadConf {
/**
* Parse the TopologyLoadConf from a file in YAML format.
+ *
* @param file the file to read from
* @return the parsed conf
* @throws IOException if there is an issue reading the file.
@@ -95,6 +97,7 @@ public static TopologyLoadConf fromConf(File file) throws IOException {
/**
* Parse the TopologyLoadConf from a config map.
+ *
* @param conf the config with the TopologyLoadConf in it
* @return the parsed instance.
*/
@@ -130,6 +133,7 @@ public static TopologyLoadConf fromConf(Map conf) {
/**
* Write this out to a file in YAML format.
+ *
* @param file the file to write to.
* @throws IOException if there is an error writing to the file.
*/
@@ -142,6 +146,7 @@ public void writeTo(File file) throws IOException {
/**
* Convert this into a YAML String.
+ *
* @return this as a YAML String.
*/
public String toYamlString() {
@@ -153,6 +158,7 @@ public String toYamlString() {
/**
* Covert this into a Map config.
+ *
* @return this as a Map config.
*/
public Map toConf() {
@@ -182,6 +188,7 @@ public Map toConf() {
/**
* Constructor.
+ *
* @param name the name of the topology.
* @param topoConf the config for the topology
* @param spouts the spouts for the topology
@@ -230,11 +237,12 @@ public TopologyLoadConf withName(String baseName) {
/**
* The first one that is not null.
+ *
* @param rest all the other somethings
* @param whatever type you want.
* @return the first one that is not null
*/
- static V or(V...rest) {
+ static V or(V... rest) {
for (V i : rest) {
if (i != null) {
return i;
@@ -243,7 +251,8 @@ static V or(V...rest) {
return null;
}
- LoadCompConf scaleCompParallel(LoadCompConf comp, double v, Map topoSpecificParallel) {
+ LoadCompConf scaleCompParallel(LoadCompConf comp, double v, Map topoSpecificParallel) {
LoadCompConf ret = comp;
double scale = or(topoSpecificParallel.get(name + ":" + comp.id),
topoSpecificParallel.get(name + ":*"),
@@ -255,7 +264,8 @@ LoadCompConf scaleCompParallel(LoadCompConf comp, double v, Map
return ret;
}
- LoadCompConf scaleCompThroughput(LoadCompConf comp, double v, Map topoSpecificParallel) {
+ LoadCompConf scaleCompThroughput(LoadCompConf comp, double v, Map topoSpecificParallel) {
LoadCompConf ret = comp;
double scale = or(topoSpecificParallel.get(name + ":" + comp.id),
topoSpecificParallel.get(name + ":*"),
@@ -267,7 +277,8 @@ LoadCompConf scaleCompThroughput(LoadCompConf comp, double v, Map topoSpecific) {
+ private LoadCompConf overrideCompSlowExec(LoadCompConf comp, Map topoSpecific) {
LoadCompConf ret = comp;
SlowExecutorPattern slp = topoSpecific.get(name + ":" + comp.id);
if (slp != null) {
@@ -277,23 +288,29 @@ private LoadCompConf overrideCompSlowExec(LoadCompConf comp, Map topoSpecific) {
if (v == 1.0 && (topoSpecific == null || topoSpecific.isEmpty())) {
return this;
}
- List scaledSpouts = spouts.stream().map((s) -> scaleCompParallel(s, v, topoSpecific))
+ List scaledSpouts = spouts.stream().map((s) -> scaleCompParallel(s, v,
+ topoSpecific))
.collect(Collectors.toList());
- List scaledBolts = bolts.stream().map((s) -> scaleCompParallel(s, v, topoSpecific))
+ List scaledBolts = bolts.stream().map((s) -> scaleCompParallel(s, v,
+ topoSpecific))
.collect(Collectors.toList());
return new TopologyLoadConf(name, topoConf, scaledSpouts, scaledBolts, streams);
}
/**
* Scale the throughput of the entire topology by a percentage.
+ *
* @param v the amount to scale it by 1.0 is nothing 0.5 cuts it in half and 2.0 doubles it.
* @return a copy of this with the needed adjustments made.
*/
@@ -301,15 +318,18 @@ public TopologyLoadConf scaleThroughput(double v, Map topoSpecif
if (v == 1.0 && (topoSpecific == null || topoSpecific.isEmpty())) {
return this;
}
- List scaledSpouts = spouts.stream().map((s) -> scaleCompThroughput(s, v, topoSpecific))
+ List scaledSpouts = spouts.stream().map((s) -> scaleCompThroughput(s, v,
+ topoSpecific))
.collect(Collectors.toList());
- List scaledBolts = bolts.stream().map((s) -> scaleCompThroughput(s, v, topoSpecific))
+ List scaledBolts = bolts.stream().map((s) -> scaleCompThroughput(s, v,
+ topoSpecific))
.collect(Collectors.toList());
return new TopologyLoadConf(name, topoConf, scaledSpouts, scaledBolts, streams);
}
/**
* Override the SlowExecutorPattern for given components.
+ *
* @param topoSpecific what we are going to use to override.
* @return a copy of this with the needed adjustments made.
*/
@@ -317,15 +337,18 @@ public TopologyLoadConf overrideSlowExecs(Map topoS
if (topoSpecific == null || topoSpecific.isEmpty()) {
return this;
}
- List modedSpouts = spouts.stream().map((s) -> overrideCompSlowExec(s, topoSpecific))
+ List modedSpouts = spouts.stream().map((s) -> overrideCompSlowExec(s,
+ topoSpecific))
.collect(Collectors.toList());
- List modedBolts = bolts.stream().map((b) -> overrideCompSlowExec(b, topoSpecific))
+ List modedBolts = bolts.stream().map((b) -> overrideCompSlowExec(b,
+ topoSpecific))
.collect(Collectors.toList());
return new TopologyLoadConf(name, topoConf, modedSpouts, modedBolts, streams);
}
/**
* Create a new version of this topology with identifiable information removed.
+ *
* @return the anonymized version of the TopologyLoadConf.
*/
public TopologyLoadConf anonymize() {
@@ -362,16 +385,17 @@ public TopologyLoadConf anonymize() {
}
GlobalStreamId orig = in.gsid();
if (!remappedStreams.containsKey(orig)) {
- //Even if the topology is not valid we still need to remap it all
- String remappedComp = remappedComponents.computeIfAbsent(in.fromComponent, (key) -> {
- LOG.warn("stream's {} from is not defined {}", in.id, in.fromComponent);
- return getUniqueBoltName();
- });
+ // Even if the topology is not valid we still need to remap it all
+ String remappedComp = remappedComponents.computeIfAbsent(in.fromComponent,
+ (key) -> {
+ LOG.warn("stream's {} from is not defined {}", in.id, in.fromComponent);
+ return getUniqueBoltName();
+ });
remappedStreams.put(orig, new GlobalStreamId(remappedComp, getUniqueStreamName()));
}
}
- //Now we need to map them all back again
+ // Now we need to map them all back again
List remappedSpouts = spouts.stream()
.map((orig) -> orig.remap(remappedComponents, remappedStreams))
.collect(Collectors.toList());
@@ -381,11 +405,12 @@ public TopologyLoadConf anonymize() {
List remappedInputStreams = streams.stream()
.map((orig) -> orig.remap(remappedComponents, remappedStreams))
.collect(Collectors.toList());
- return new TopologyLoadConf(getUniqueTopoName(), anonymizeTopoConf(topoConf), remappedSpouts, remappedBolts, remappedInputStreams);
+ return new TopologyLoadConf(getUniqueTopoName(), anonymizeTopoConf(topoConf),
+ remappedSpouts, remappedBolts, remappedInputStreams);
}
private static Map anonymizeTopoConf(Map topoConf) {
- //Only keep important conf keys
+ // Only keep important conf keys
Map ret = new HashMap<>();
for (Map.Entry entry : topoConf.entrySet()) {
String key = entry.getKey();
@@ -416,13 +441,15 @@ private static Object cleanupChildOpts(Object value) {
for (String subValue : (Collection) value) {
ret.add((String) cleanupChildOpts(subValue));
}
- return ret.stream().filter((item) -> item != null && !item.isEmpty()).collect(Collectors.toList());
+ return ret.stream().filter((item) -> item != null && !item.isEmpty()).collect(Collectors
+ .toList());
}
}
/**
* Try to see if this looks like a trident topology.
* NOTE: this will not work for anonymized configs
+ *
* @return true if it does else false.
*/
public boolean looksLikeTrident() {
@@ -448,6 +475,7 @@ public boolean looksLikeTrident() {
/**
* Get the messages emitted per second in aggregate across all streams in the topology.
+ *
* @return messages per second.
*/
public double getAllEmittedAggregate() {
@@ -460,6 +488,7 @@ public double getAllEmittedAggregate() {
/**
* Get the messages emitted per second in aggregate for all of the spouts in the topology.
+ *
* @return messages per second.
*/
public double getSpoutEmittedAggregate() {
@@ -471,12 +500,14 @@ public double getSpoutEmittedAggregate() {
}
/**
- * Try and guess at the actual number of messages emitted per second by a trident topology, not the number of batches.
+ * Try and guess at the actual number of messages emitted per second by a trident topology, not
+ * the number of batches.
* This does not work on an anonymized conf.
+ *
* @return messages per second or 0 if this does not look like a trident topology.
*/
public double getTridentEstimatedEmittedAggregate() {
- //In this case we are ignoring the coord stuff, and only looking at
+ // In this case we are ignoring the coord stuff, and only looking at
double ret = 0;
if (looksLikeTrident()) {
List all = new ArrayList<>(bolts);
@@ -499,7 +530,8 @@ public double getTridentEstimatedEmittedAggregate() {
}
public TopologyLoadConf replaceShuffleWithLocalOrShuffle() {
- List modified = streams.stream().map((in) -> in.replaceShuffleWithLocalOrShuffle()).collect(Collectors.toList());
+ List modified = streams.stream().map((in) -> in
+ .replaceShuffleWithLocalOrShuffle()).collect(Collectors.toList());
return new TopologyLoadConf(name, topoConf, spouts, bolts, modified);
}
}
diff --git a/examples/storm-loadgen/src/test/java/org/apache/storm/loadgen/LoadCompConfTest.java b/examples/storm-loadgen/src/test/java/org/apache/storm/loadgen/LoadCompConfTest.java
index ed10692cdc4..14b6f22c71a 100644
--- a/examples/storm-loadgen/src/test/java/org/apache/storm/loadgen/LoadCompConfTest.java
+++ b/examples/storm-loadgen/src/test/java/org/apache/storm/loadgen/LoadCompConfTest.java
@@ -7,9 +7,9 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
+ *
Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
@@ -18,24 +18,25 @@
package org.apache.storm.loadgen;
-import org.junit.jupiter.api.Test;
-
import static org.junit.jupiter.api.Assertions.assertEquals;
+import org.junit.jupiter.api.Test;
+
public class LoadCompConfTest {
@Test
public void scaleParallel() {
LoadCompConf orig = new LoadCompConf.Builder()
.withId("SOME_SPOUT")
.withParallelism(1)
- .withStream(new OutputStream("default", new NormalDistStats(500.0, 100.0, 300.0, 600.0), false))
+ .withStream(new OutputStream("default", new NormalDistStats(500.0, 100.0, 300.0, 600.0),
+ false))
.build();
assertEquals(500.0, orig.getAllEmittedAggregate(), 0.001);
LoadCompConf scaled = orig.scaleParallel(2);
- //Parallelism is double
+ // Parallelism is double
assertEquals(2, scaled.parallelism);
assertEquals("SOME_SPOUT", scaled.id);
- //But throughput is the same
+ // But throughput is the same
assertEquals(500.0, scaled.getAllEmittedAggregate(), 0.001);
}
@@ -44,14 +45,15 @@ public void scaleThroughput() {
LoadCompConf orig = new LoadCompConf.Builder()
.withId("SOME_SPOUT")
.withParallelism(1)
- .withStream(new OutputStream("default", new NormalDistStats(500.0, 100.0, 300.0, 600.0), false))
+ .withStream(new OutputStream("default", new NormalDistStats(500.0, 100.0, 300.0, 600.0),
+ false))
.build();
assertEquals(500.0, orig.getAllEmittedAggregate(), 0.001);
LoadCompConf scaled = orig.scaleThroughput(2.0);
- //Parallelism is same
+ // Parallelism is same
assertEquals(1, scaled.parallelism);
assertEquals("SOME_SPOUT", scaled.id);
- //But throughput is the same
+ // But throughput is the same
assertEquals(1000.0, scaled.getAllEmittedAggregate(), 0.001);
}
-}
\ No newline at end of file
+}
diff --git a/examples/storm-loadgen/src/test/java/org/apache/storm/loadgen/LoadMetricsServerTest.java b/examples/storm-loadgen/src/test/java/org/apache/storm/loadgen/LoadMetricsServerTest.java
index 7f549be1c0c..6ef5c832ed1 100644
--- a/examples/storm-loadgen/src/test/java/org/apache/storm/loadgen/LoadMetricsServerTest.java
+++ b/examples/storm-loadgen/src/test/java/org/apache/storm/loadgen/LoadMetricsServerTest.java
@@ -7,9 +7,9 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
+ *
Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
@@ -18,12 +18,12 @@
package org.apache.storm.loadgen;
+import static org.apache.storm.loadgen.LoadMetricsServer.convert;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
import java.util.concurrent.TimeUnit;
import org.junit.jupiter.api.Test;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.apache.storm.loadgen.LoadMetricsServer.convert;
-
public class LoadMetricsServerTest {
@Test
public void convertTest() {
@@ -34,4 +34,4 @@ public void convertTest() {
}
}
}
-}
\ No newline at end of file
+}
diff --git a/examples/storm-loadgen/src/test/java/org/apache/storm/loadgen/NormalDistStatsTest.java b/examples/storm-loadgen/src/test/java/org/apache/storm/loadgen/NormalDistStatsTest.java
index 3512bbae13d..2387456b9c3 100644
--- a/examples/storm-loadgen/src/test/java/org/apache/storm/loadgen/NormalDistStatsTest.java
+++ b/examples/storm-loadgen/src/test/java/org/apache/storm/loadgen/NormalDistStatsTest.java
@@ -7,9 +7,9 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
+ *
Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
@@ -18,10 +18,10 @@
package org.apache.storm.loadgen;
-import org.junit.jupiter.api.Test;
-
import static org.junit.jupiter.api.Assertions.assertEquals;
+import org.junit.jupiter.api.Test;
+
public class NormalDistStatsTest {
public static void assertNDSEquals(NormalDistStats a, NormalDistStats b) {
assertEquals(a.mean, b.mean, 0.0001, "mean");
@@ -40,4 +40,4 @@ public void scaleBy() {
assertNDSEquals(expectedHalf, orig.scaleBy(0.5));
}
-}
\ No newline at end of file
+}
diff --git a/examples/storm-loadgen/src/test/java/org/apache/storm/loadgen/OutputStreamTest.java b/examples/storm-loadgen/src/test/java/org/apache/storm/loadgen/OutputStreamTest.java
index 3980e2a7ab6..15f58e0ca74 100644
--- a/examples/storm-loadgen/src/test/java/org/apache/storm/loadgen/OutputStreamTest.java
+++ b/examples/storm-loadgen/src/test/java/org/apache/storm/loadgen/OutputStreamTest.java
@@ -7,9 +7,9 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
+ *
Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
@@ -18,14 +18,15 @@
package org.apache.storm.loadgen;
-import org.junit.jupiter.api.Test;
-
import static org.junit.jupiter.api.Assertions.assertEquals;
+import org.junit.jupiter.api.Test;
+
public class OutputStreamTest {
@Test
public void scaleThroughput() {
- OutputStream orig = new OutputStream("ID", new NormalDistStats(100.0, 1.0, 99.0, 101.0), false);
+ OutputStream orig = new OutputStream("ID", new NormalDistStats(100.0, 1.0, 99.0, 101.0),
+ false);
OutputStream scaled = orig.scaleThroughput(2.0);
assertEquals(orig.id, scaled.id);
assertEquals(orig.areKeysSkewed, scaled.areKeysSkewed);
@@ -34,4 +35,4 @@ public void scaleThroughput() {
assertEquals(scaled.rate.min, 199.0, 0.0001);
assertEquals(scaled.rate.max, 201.0, 0.0001);
}
-}
\ No newline at end of file
+}
diff --git a/examples/storm-perf/pom.xml b/examples/storm-perf/pom.xml
index 7e9377cbf10..69c435b9e2e 100644
--- a/examples/storm-perf/pom.xml
+++ b/examples/storm-perf/pom.xml
@@ -86,6 +86,16 @@
${storm.topology}
+
+ org.openrewrite.maven
+ rewrite-maven-plugin
+
+
+
+ org.codehaus.gmavenplus
+ gmavenplus-plugin
+
+ org.apache.maven.pluginsmaven-checkstyle-plugin
diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/BackPressureTopo.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/BackPressureTopo.java
index 5fb70220eb9..a3b0da5ac94 100644
--- a/examples/storm-perf/src/main/java/org/apache/storm/perf/BackPressureTopo.java
+++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/BackPressureTopo.java
@@ -34,7 +34,6 @@
import org.apache.storm.utils.Utils;
import org.slf4j.LoggerFactory;
-
public class BackPressureTopo {
private static final String SPOUT_ID = "ConstSpout";
@@ -80,7 +79,8 @@ public static void main(String[] args) throws Exception {
return;
}
// Submit topology to storm cluster
- Helper.runOnClusterAndPrintMetrics(runTime, "BackPressureTopo", topoConf, getTopology(topoConf));
+ Helper.runOnClusterAndPrintMetrics(runTime, "BackPressureTopo", topoConf,
+ getTopology(topoConf));
}
private static class ThrottledBolt extends BaseRichBolt {
@@ -93,7 +93,8 @@ private static class ThrottledBolt extends BaseRichBolt {
}
@Override
- public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) {
+ public void prepare(Map topoConf, TopologyContext context,
+ OutputCollector collector) {
this.collector = collector;
}
@@ -104,7 +105,7 @@ public void execute(Tuple tuple) {
try {
Thread.sleep(sleepMs);
} catch (InterruptedException e) {
- //.. ignore
+ // .. ignore
}
}
diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/ConstSpoutIdBoltNullBoltTopo.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/ConstSpoutIdBoltNullBoltTopo.java
index c83763d6e26..b30f689377f 100644
--- a/examples/storm-perf/src/main/java/org/apache/storm/perf/ConstSpoutIdBoltNullBoltTopo.java
+++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/ConstSpoutIdBoltNullBoltTopo.java
@@ -29,8 +29,10 @@
import org.apache.storm.utils.Utils;
/**
- * ConstSpout -> IdBolt -> DevNullBolt This topology measures speed of messaging between spouts->bolt and bolt->bolt ConstSpout :
- * Continuously emits a constant string IdBolt : clones and emits input tuples DevNullBolt : discards incoming tuples.
+ * ConstSpout -> IdBolt -> DevNullBolt This topology measures speed of messaging between
+ * spouts->bolt and bolt->bolt ConstSpout :
+ * Continuously emits a constant string IdBolt : clones and emits input tuples DevNullBolt :
+ * discards incoming tuples.
*/
public class ConstSpoutIdBoltNullBoltTopo {
@@ -67,11 +69,11 @@ static StormTopology getTopology(Map conf) {
int numBolt2 = Helper.getInt(conf, BOLT2_COUNT, 1);
builder.setBolt(BOLT2_ID, bolt2, numBolt2)
.localOrShuffleGrouping(BOLT1_ID);
- System.err.printf("====> Using : numSpouts = %d , numBolt1 = %d, numBolt2=%d\n", numSpouts, numBolt1, numBolt2);
+ System.err.printf("====> Using : numSpouts = %d , numBolt1 = %d, numBolt2=%d\n", numSpouts,
+ numBolt1, numBolt2);
return builder.createTopology();
}
-
public static void main(String[] args) throws Exception {
int runTime = -1;
Config topoConf = new Config();
diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/ConstSpoutNullBoltTopo.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/ConstSpoutNullBoltTopo.java
index 63ef51b45c7..34b4704ee23 100755
--- a/examples/storm-perf/src/main/java/org/apache/storm/perf/ConstSpoutNullBoltTopo.java
+++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/ConstSpoutNullBoltTopo.java
@@ -87,11 +87,15 @@ public static void main(String[] args) throws Exception {
Config topoConf = new Config();
// Configured for achieving max throughput in single worker mode (empirically found).
// For reference : numbers taken on MacBook Pro mid 2015
- // -- ACKer=0: ~8 mill/sec (batchSz=2k & recvQsize=50k). 6.7 mill/sec (batchSz=1 & recvQsize=1k)
- // -- ACKer=1: ~1 mill/sec, lat= ~1 microsec (batchSz=1 & bolt.wait.strategy=Park bolt.wait.park.micros=0)
- // -- ACKer=1: ~1.3 mill/sec, lat= ~11 micros (batchSz=1 & receive.buffer.size=1k, bolt.wait & bp.wait =
+ // -- ACKer=0: ~8 mill/sec (batchSz=2k & recvQsize=50k). 6.7 mill/sec (batchSz=1 &
+ // recvQsize=1k)
+ // -- ACKer=1: ~1 mill/sec, lat= ~1 microsec (batchSz=1 & bolt.wait.strategy=Park
+ // bolt.wait.park.micros=0)
+ // -- ACKer=1: ~1.3 mill/sec, lat= ~11 micros (batchSz=1 & receive.buffer.size=1k, bolt.wait
+ // & bp.wait =
// Progressive[defaults])
- // -- ACKer=1: ~1.6 mill/sec, lat= ~300 micros (batchSz=500 & bolt.wait.strategy=Park bolt.wait.park.micros=0)
+ // -- ACKer=1: ~1.6 mill/sec, lat= ~300 micros (batchSz=500 & bolt.wait.strategy=Park
+ // bolt.wait.park.micros=0)
topoConf.put(Config.TOPOLOGY_SPOUT_RECVQ_SKIPS, 8);
topoConf.put(Config.TOPOLOGY_PRODUCER_BATCH_SIZE, 500);
topoConf.put(Config.TOPOLOGY_EXECUTOR_RECEIVE_BUFFER_SIZE, 50_000);
diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/ConstSpoutOnlyTopo.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/ConstSpoutOnlyTopo.java
index 1b012fe2fb8..44eb2cb37fd 100755
--- a/examples/storm-perf/src/main/java/org/apache/storm/perf/ConstSpoutOnlyTopo.java
+++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/ConstSpoutOnlyTopo.java
@@ -35,7 +35,6 @@ public class ConstSpoutOnlyTopo {
public static final String TOPOLOGY_NAME = "ConstSpoutOnlyTopo";
public static final String SPOUT_ID = "constSpout";
-
static StormTopology getTopology() {
// 1 - Setup Const Spout --------
diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/FileReadWordCountSpoutCompressionTopo.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/FileReadWordCountSpoutCompressionTopo.java
index 51c43057ef4..a15aa0f7f78 100644
--- a/examples/storm-perf/src/main/java/org/apache/storm/perf/FileReadWordCountSpoutCompressionTopo.java
+++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/FileReadWordCountSpoutCompressionTopo.java
@@ -50,7 +50,6 @@ public class FileReadWordCountSpoutCompressionTopo {
public static final int DEFAULT_SPLIT_BOLT_NUM = 2;
public static final int DEFAULT_COUNT_BOLT_NUM = 2;
-
static StormTopology getTopology(Map config) {
final int spoutNum = Helper.getInt(config, SPOUT_NUM, DEFAULT_SPOUT_NUM);
@@ -62,8 +61,10 @@ static StormTopology getTopology(Map config) {
// sampledata/longrandomwords.txt contains sentences with at least 1500 bytes
builder.setSpout(SPOUT_ID, new FileReadSpout(inputFile), spoutNum)
.addConfiguration(Config.TOPOLOGY_TUPLE_COMPRESSION_ENABLE, true);
- builder.setBolt(SPLIT_ID, new SplitSentenceBolt(), spBoltNum).localOrShuffleGrouping(SPOUT_ID);
- builder.setBolt(COUNT_ID, new CountBolt(), cntBoltNum).fieldsGrouping(SPLIT_ID, new Fields(SplitSentenceBolt.FIELDS));
+ builder.setBolt(SPLIT_ID, new SplitSentenceBolt(), spBoltNum)
+ .localOrShuffleGrouping(SPOUT_ID);
+ builder.setBolt(COUNT_ID, new CountBolt(), cntBoltNum).fieldsGrouping(SPLIT_ID,
+ new Fields(SplitSentenceBolt.FIELDS));
return builder.createTopology();
}
@@ -78,7 +79,8 @@ public static void main(String[] args) throws Exception {
topoConf.putAll(Utils.findAndReadConfigFile(args[1]));
}
topoConf.put(Config.TOPOLOGY_PRODUCER_BATCH_SIZE, 1000);
- topoConf.put(Config.TOPOLOGY_BOLT_WAIT_STRATEGY, "org.apache.storm.policy.WaitStrategyPark");
+ topoConf.put(Config.TOPOLOGY_BOLT_WAIT_STRATEGY,
+ "org.apache.storm.policy.WaitStrategyPark");
topoConf.put(Config.TOPOLOGY_BOLT_WAIT_PARK_MICROSEC, 0);
topoConf.put(Config.TOPOLOGY_DISABLE_LOADAWARE_MESSAGING, true);
topoConf.put(Config.TOPOLOGY_STATS_SAMPLE_RATE, 0.0005);
diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/FileReadWordCountTopo.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/FileReadWordCountTopo.java
index 78eaab7eff7..a0f9c5f8e54 100644
--- a/examples/storm-perf/src/main/java/org/apache/storm/perf/FileReadWordCountTopo.java
+++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/FileReadWordCountTopo.java
@@ -50,7 +50,6 @@ public class FileReadWordCountTopo {
public static final int DEFAULT_SPLIT_BOLT_NUM = 2;
public static final int DEFAULT_COUNT_BOLT_NUM = 2;
-
static StormTopology getTopology(Map config) {
final int spoutNum = Helper.getInt(config, SPOUT_NUM, DEFAULT_SPOUT_NUM);
@@ -60,8 +59,10 @@ static StormTopology getTopology(Map config) {
TopologyBuilder builder = new TopologyBuilder();
builder.setSpout(SPOUT_ID, new FileReadSpout(inputFile), spoutNum);
- builder.setBolt(SPLIT_ID, new SplitSentenceBolt(), spBoltNum).localOrShuffleGrouping(SPOUT_ID);
- builder.setBolt(COUNT_ID, new CountBolt(), cntBoltNum).fieldsGrouping(SPLIT_ID, new Fields(SplitSentenceBolt.FIELDS));
+ builder.setBolt(SPLIT_ID, new SplitSentenceBolt(), spBoltNum)
+ .localOrShuffleGrouping(SPOUT_ID);
+ builder.setBolt(COUNT_ID, new CountBolt(), cntBoltNum).fieldsGrouping(SPLIT_ID,
+ new Fields(SplitSentenceBolt.FIELDS));
return builder.createTopology();
}
@@ -76,7 +77,8 @@ public static void main(String[] args) throws Exception {
topoConf.putAll(Utils.findAndReadConfigFile(args[1]));
}
topoConf.put(Config.TOPOLOGY_PRODUCER_BATCH_SIZE, 1000);
- topoConf.put(Config.TOPOLOGY_BOLT_WAIT_STRATEGY, "org.apache.storm.policy.WaitStrategyPark");
+ topoConf.put(Config.TOPOLOGY_BOLT_WAIT_STRATEGY,
+ "org.apache.storm.policy.WaitStrategyPark");
topoConf.put(Config.TOPOLOGY_BOLT_WAIT_PARK_MICROSEC, 0);
topoConf.put(Config.TOPOLOGY_DISABLE_LOADAWARE_MESSAGING, true);
topoConf.put(Config.TOPOLOGY_STATS_SAMPLE_RATE, 0.0005);
diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/HdfsSpoutNullBoltTopo.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/HdfsSpoutNullBoltTopo.java
index 9876cac9f3f..bb55ee2bdf0 100644
--- a/examples/storm-perf/src/main/java/org/apache/storm/perf/HdfsSpoutNullBoltTopo.java
+++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/HdfsSpoutNullBoltTopo.java
@@ -92,13 +92,15 @@ public static void main(String[] args) throws Exception {
Config topoConf = new Config();
topoConf.putAll(Utils.findAndReadConfigFile(args[1]));
topoConf.put(Config.TOPOLOGY_PRODUCER_BATCH_SIZE, 1000);
- topoConf.put(Config.TOPOLOGY_BOLT_WAIT_STRATEGY, "org.apache.storm.policy.WaitStrategyPark");
+ topoConf.put(Config.TOPOLOGY_BOLT_WAIT_STRATEGY,
+ "org.apache.storm.policy.WaitStrategyPark");
topoConf.put(Config.TOPOLOGY_BOLT_WAIT_PARK_MICROSEC, 0);
topoConf.put(Config.TOPOLOGY_DISABLE_LOADAWARE_MESSAGING, true);
topoConf.put(Config.TOPOLOGY_STATS_SAMPLE_RATE, 0.0005);
topoConf.putAll(Utils.readCommandLineOpts());
// Submit to Storm cluster
- Helper.runOnClusterAndPrintMetrics(durationSec, TOPOLOGY_NAME, topoConf, getTopology(topoConf));
+ Helper.runOnClusterAndPrintMetrics(durationSec, TOPOLOGY_NAME, topoConf,
+ getTopology(topoConf));
}
}
diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/JitterAwareGroupingTopology.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/JitterAwareGroupingTopology.java
index 7bc53476f3c..c12eec83023 100644
--- a/examples/storm-perf/src/main/java/org/apache/storm/perf/JitterAwareGroupingTopology.java
+++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/JitterAwareGroupingTopology.java
@@ -52,14 +52,17 @@
*
*
{@code JitteryWorkerBolt} tasks have task-index-dependent latency jitter: task 0 is
* perfectly steady and each higher index is progressively jitterier, because the per-tuple noise
- * width grows with the task index over a constant floor. This deliberately exercises the signal
+ * width grows with the task index over a constant floor. This deliberately exercises the
+ * signal
* the grouping ranks on — RFC-1889 jitter (the EWMA of {@code |Δlatency|}), which measures dispersion,
* not level. A per-task mean offset would not work: it cancels in the estimator's
* consecutive-difference, leaving every task with identical jitter. With upstream feedback enabled,
- * {@link JitterAwareStreamGrouping} steers more tuples toward the steadiest (lowest-jitter) tasks; run it
+ * {@link JitterAwareStreamGrouping} steers more tuples toward the steadiest (lowest-jitter) tasks;
+ * run it
* against the {@code loadaware} baseline mode to measure the effect.
*
- *
Run the baseline and the jitter-aware run back-to-back to compare. Select the grouping with the
+ *
Run the baseline and the jitter-aware run back-to-back to compare. Select the grouping with
+ * the
* {@code grouping.mode} flag:
*
* # Baseline: plain LoadAwareShuffleGrouping (no upstream feedback needed).
@@ -139,8 +142,10 @@ static StormTopology getTopology(Map conf) {
/**
* Picks the {@code splitter -> worker} grouping from {@link #GROUPING_MODE}. Defaults to the
* feedback-driven {@link JitterAwareStreamGrouping}; {@code grouping.mode=loadaware} selects the plain
- * {@link LoadAwareShuffleGrouping} baseline so the two can be benchmarked back-to-back. Both implement
- * {@link org.apache.storm.grouping.LoadAwareCustomStreamGrouping} and require the locality-aware
+ * {@link LoadAwareShuffleGrouping} baseline so the two can be benchmarked back-to-back. Both
+ * implement
+ * {@link org.apache.storm.grouping.LoadAwareCustomStreamGrouping} and require the
+ * locality-aware
* configuration set in {@link #main}.
*/
static CustomStreamGrouping selectGrouping(Map conf) {
@@ -166,17 +171,24 @@ public static void main(String[] args) throws Exception {
}
topoConf.put(Config.TOPOLOGY_STATS_EWMA_ENABLE, true);
- // max.spout.pending counts SPOUT tuples (sentences), but each fans out to ~7 word-tuples at the
- // SplitterBolt, so the in-flight backlog at the slow workers is ~7x this number. Keep it low enough
- // that steady-state complete latency stays well under topology.message.timeout.secs with ~0 fails;
- // 4000 oversubscribed the synthetic workers and pinned latency at the timeout. Tune per cluster:
+ // max.spout.pending counts SPOUT tuples (sentences), but each fans out to ~7 word-tuples at
+ // the
+ // SplitterBolt, so the in-flight backlog at the slow workers is ~7x this number. Keep it
+ // low enough
+ // that steady-state complete latency stays well under topology.message.timeout.secs with ~0
+ // fails;
+ // 4000 oversubscribed the synthetic workers and pinned latency at the timeout. Tune per
+ // cluster:
// complete_latency ~= (pending * fanout) / aggregate_worker_ack_rate (Little's law).
topoConf.putIfAbsent(Config.TOPOLOGY_MAX_SPOUT_PENDING, 500);
topoConf.putIfAbsent(Config.TOPOLOGY_UPSTREAM_FEEDBACK_FREQ_SECS, 10);
- // Both grouping modes resolve to a LoadAwareShuffleGrouping (directly, or as the jitter grouping's
- // fallback), whose prepare() requires these locality-aware keys. Normally supplied by defaults.yaml;
- // set defensively so the topology is self-contained. Placed before the CLI merge so -c can override.
+ // Both grouping modes resolve to a LoadAwareShuffleGrouping (directly, or as the jitter
+ // grouping's
+ // fallback), whose prepare() requires these locality-aware keys. Normally supplied by
+ // defaults.yaml;
+ // set defensively so the topology is self-contained. Placed before the CLI merge so -c can
+ // override.
topoConf.putIfAbsent(Config.STORM_NETWORK_TOPOGRAPHY_PLUGIN,
"org.apache.storm.networktopography.DefaultRackDNSToSwitchMapping");
topoConf.putIfAbsent(Config.TOPOLOGY_LOCALITYAWARE_HIGHER_BOUND, 0.8);
@@ -205,7 +217,8 @@ private static class GenSpout extends BaseRichSpout {
}
@Override
- public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
+ public void open(Map conf, TopologyContext context,
+ SpoutOutputCollector collector) {
this.collector = collector;
try {
this.lines = FileReadSpout.readLines(new FileInputStream(filePath));
@@ -237,7 +250,8 @@ private static class SplitterBolt extends BaseRichBolt {
private OutputCollector collector;
@Override
- public void prepare(Map conf, TopologyContext context, OutputCollector collector) {
+ public void prepare(Map conf, TopologyContext context,
+ OutputCollector collector) {
this.collector = collector;
}
@@ -257,12 +271,14 @@ public void declareOutputFields(OutputFieldsDeclarer declarer) {
}
/**
- * Counts words and parks for a constant floor plus task-index-proportional random noise, so each
+ * Counts words and parks for a constant floor plus task-index-proportional random noise, so
+ * each
* task's RFC-1889 execute-jitter (the EWMA of {@code |Δ execute-latency|}) differs by construction.
*
*
Task {@code i} parks for {@code baseDelayUs} µs plus uniform noise in {@code [0, i * baseDelayUs]}
* µs per tuple. Crucially, the constant floor cancels in the consecutive-difference the jitter
- * estimator takes, so only the noise width drives jitter — not a per-task mean offset (which
+ * estimator takes, so only the noise width drives jitter — not a per-task mean offset
+ * (which
* would cancel and leave every task with identical jitter). Steady-state jitter is {@code width / 3}.
* For a 4-task setup with the default {@code baseDelayUs = 2000}:
*
@@ -271,11 +287,14 @@ public void declareOutputFields(OutputFieldsDeclarer declarer) {
*
Task 2: ~2–6 ms — jitter ≈ 1.33 ms
*
Task 3: ~2–8 ms — jitter ≈ 2 ms (jitteriest)
*
- * Widths are millisecond-scale so the EWMA jitter gauge (millisecond resolution) records distinct
- * values per task. {@link JitterAwareStreamGrouping} then steers tuples toward the lowest-jitter
+ * Widths are millisecond-scale so the EWMA jitter gauge (millisecond resolution) records
+ * distinct
+ * values per task. {@link JitterAwareStreamGrouping} then steers tuples toward the
+ * lowest-jitter
* task (task 0) when feedback is enabled.
*
- *
{@link LockSupport#parkNanos} is used instead of a spin loop so jittery tasks yield the CPU
+ *
{@link LockSupport#parkNanos} is used instead of a spin loop so jittery tasks yield the
+ * CPU
* and do not starve the steady task's executor thread.
*/
private static class JitteryWorkerBolt extends BaseRichBolt {
@@ -290,14 +309,18 @@ private static class JitteryWorkerBolt extends BaseRichBolt {
}
@Override
- public void prepare(Map conf, TopologyContext context, OutputCollector collector) {
+ public void prepare(Map conf, TopologyContext context,
+ OutputCollector collector) {
this.collector = collector;
long baseDelayNs = baseDelayUs * 1_000L;
- // Constant floor parked by every task: keeps each task doing real work, but cancels in the
+ // Constant floor parked by every task: keeps each task doing real work, but cancels in
+ // the
// consecutive-difference the RFC-1889 jitter estimator takes, so it adds no jitter.
this.baseFloorNs = baseDelayNs;
- // Noise WIDTH grows with task index, so execute-jitter (EWMA of |Δlatency|) genuinely differs
- // per task: task 0 is perfectly steady (zero jitter), higher indices are progressively jitterier.
+ // Noise WIDTH grows with task index, so execute-jitter (EWMA of |Δlatency|) genuinely
+ // differs
+ // per task: task 0 is perfectly steady (zero jitter), higher indices are progressively
+ // jitterier.
this.jitterWidthNs = baseDelayNs * context.getThisTaskIndex();
}
@@ -307,7 +330,8 @@ public void execute(Tuple tuple) {
counts.merge(word, 1, Integer::sum);
int count = counts.get(word);
- long noiseNs = jitterWidthNs == 0L ? 0L : (long) (ThreadLocalRandom.current().nextDouble() * jitterWidthNs);
+ long noiseNs = jitterWidthNs == 0L ? 0L : (long) (ThreadLocalRandom.current()
+ .nextDouble() * jitterWidthNs);
long sleepNs = baseFloorNs + noiseNs;
if (sleepNs > 0) {
LockSupport.parkNanos(sleepNs);
@@ -332,7 +356,8 @@ private static class SinkBolt extends BaseRichBolt {
private OutputCollector collector;
@Override
- public void prepare(Map conf, TopologyContext context, OutputCollector collector) {
+ public void prepare(Map conf, TopologyContext context,
+ OutputCollector collector) {
this.collector = collector;
}
diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/KafkaClientHdfsTopo.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/KafkaClientHdfsTopo.java
index 5eb61b2e4e3..defaef39ef5 100755
--- a/examples/storm-perf/src/main/java/org/apache/storm/perf/KafkaClientHdfsTopo.java
+++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/KafkaClientHdfsTopo.java
@@ -58,7 +58,6 @@ public class KafkaClientHdfsTopo {
public static final String HDFS_PATH = "hdfs.dir";
public static final String HDFS_BATCH = "hdfs.batch";
-
public static final int DEFAULT_SPOUT_NUM = 1;
public static final int DEFAULT_BOLT_NUM = 1;
public static final int DEFAULT_HDFS_BATCH = 1000;
@@ -68,7 +67,6 @@ public class KafkaClientHdfsTopo {
public static final String SPOUT_ID = "kafkaSpout";
public static final String BOLT_ID = "hdfsBolt";
-
static StormTopology getTopology(Map config) {
final int spoutNum = getInt(config, SPOUT_NUM, DEFAULT_SPOUT_NUM);
@@ -80,7 +78,8 @@ static StormTopology getTopology(Map config) {
String bootstrapHosts = getStr(config, KAFKA_BOOTSTRAP_HOSTS);
String topicName = getStr(config, KAFKA_TOPIC);
- KafkaSpoutConfig spoutConfig = KafkaSpoutConfig.builder(bootstrapHosts, topicName)
+ KafkaSpoutConfig spoutConfig = KafkaSpoutConfig.builder(bootstrapHosts,
+ topicName)
.setFirstPollOffsetStrategy(
FirstPollOffsetStrategy.EARLIEST)
.build();
@@ -91,9 +90,11 @@ static StormTopology getTopology(Map config) {
String hdfsUrls = getStr(config, HDFS_URI);
RecordFormat format = new LineWriter("value");
SyncPolicy syncPolicy = new CountSyncPolicy(hdfsBatch);
- FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(1.0f, FileSizeRotationPolicy.Units.GB);
+ FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(1.0f,
+ FileSizeRotationPolicy.Units.GB);
- FileNameFormat fileNameFormat = new DefaultFileNameFormat().withPath(getStr(config, HDFS_PATH));
+ FileNameFormat fileNameFormat = new DefaultFileNameFormat().withPath(getStr(config,
+ HDFS_PATH));
// Instantiate the HdfsBolt
HdfsBolt bolt = new HdfsBolt()
@@ -113,7 +114,6 @@ static StormTopology getTopology(Map config) {
return builder.createTopology();
}
-
public static int getInt(Map map, Object key, int def) {
return ObjectReader.getInt(Utils.get(map, key, def));
}
@@ -122,9 +122,9 @@ public static String getStr(Map map, Object key) {
return (String) map.get(key);
}
-
/**
- * Copies text file content from sourceDir to destinationDir. Moves source files into sourceDir after its done consuming.
+ * Copies text file content from sourceDir to destinationDir. Moves source files into sourceDir
+ * after its done consuming.
*/
public static void main(String[] args) throws Exception {
@@ -138,13 +138,15 @@ public static void main(String[] args) throws Exception {
topoConf.put(Config.TOPOLOGY_PRODUCER_BATCH_SIZE, 1000);
topoConf.put(Config.TOPOLOGY_DISABLE_LOADAWARE_MESSAGING, true);
topoConf.put(Config.TOPOLOGY_STATS_SAMPLE_RATE, 0.0005);
- topoConf.put(Config.TOPOLOGY_BOLT_WAIT_STRATEGY, "org.apache.storm.policy.WaitStrategyPark");
+ topoConf.put(Config.TOPOLOGY_BOLT_WAIT_STRATEGY,
+ "org.apache.storm.policy.WaitStrategyPark");
topoConf.put(Config.TOPOLOGY_BOLT_WAIT_PARK_MICROSEC, 0);
topoConf.putAll(Utils.readCommandLineOpts());
// Submit topology to Storm cluster
Integer durationSec = Integer.parseInt(args[0]);
- Helper.runOnClusterAndPrintMetrics(durationSec, TOPOLOGY_NAME, topoConf, getTopology(topoConf));
+ Helper.runOnClusterAndPrintMetrics(durationSec, TOPOLOGY_NAME, topoConf,
+ getTopology(topoConf));
}
public static class LineWriter implements RecordFormat {
diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/KafkaClientSpoutNullBoltTopo.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/KafkaClientSpoutNullBoltTopo.java
index 4ac1ed8e86e..ec346cce869 100644
--- a/examples/storm-perf/src/main/java/org/apache/storm/perf/KafkaClientSpoutNullBoltTopo.java
+++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/KafkaClientSpoutNullBoltTopo.java
@@ -22,15 +22,16 @@
import org.apache.storm.generated.StormTopology;
import org.apache.storm.kafka.spout.FirstPollOffsetStrategy;
import org.apache.storm.kafka.spout.KafkaSpout;
-import org.apache.storm.kafka.spout.KafkaSpoutConfig;
import org.apache.storm.kafka.spout.KafkaSpoutConfig.ProcessingGuarantee;
+import org.apache.storm.kafka.spout.KafkaSpoutConfig;
import org.apache.storm.perf.bolt.DevNullBolt;
import org.apache.storm.perf.utils.Helper;
import org.apache.storm.topology.TopologyBuilder;
import org.apache.storm.utils.Utils;
/**
- * Benchmark topology for measuring spout read/emit/ack performance. The spout reads and emits tuples. The bolt acks and discards received
+ * Benchmark topology for measuring spout read/emit/ack performance. The spout reads and emits
+ * tuples. The bolt acks and discards received
* tuples.
*/
public class KafkaClientSpoutNullBoltTopo {
@@ -62,14 +63,17 @@ public static StormTopology getTopology(Map config) {
final int boltNum = Helper.getInt(config, BOLT_NUM, DEFAULT_BOLT_NUM);
// 1 - Setup Kafka Spout --------
- String bootstrapServers = Optional.ofNullable(Helper.getStr(config, BOOTSTRAP_SERVERS)).orElse("127.0.0.1:9092");
- String kafkaTopic = Optional.ofNullable(Helper.getStr(config, KAFKA_TOPIC)).orElse("storm-perf-null-bolt-topic");
+ String bootstrapServers = Optional.ofNullable(Helper.getStr(config, BOOTSTRAP_SERVERS))
+ .orElse("127.0.0.1:9092");
+ String kafkaTopic = Optional.ofNullable(Helper.getStr(config, KAFKA_TOPIC))
+ .orElse("storm-perf-null-bolt-topic");
ProcessingGuarantee processingGuarantee = ProcessingGuarantee.valueOf(
Optional.ofNullable(Helper.getStr(config, PROCESSING_GUARANTEE))
.orElse(ProcessingGuarantee.AT_LEAST_ONCE.name()));
int offsetCommitPeriodMs = Helper.getInt(config, OFFSET_COMMIT_PERIOD_MS, 30_000);
- KafkaSpoutConfig kafkaSpoutConfig = KafkaSpoutConfig.builder(bootstrapServers, kafkaTopic)
+ KafkaSpoutConfig kafkaSpoutConfig = KafkaSpoutConfig
+ .builder(bootstrapServers, kafkaTopic)
.setProcessingGuarantee(processingGuarantee)
.setOffsetCommitPeriodMs(offsetCommitPeriodMs)
.setFirstPollOffsetStrategy(
@@ -109,7 +113,8 @@ public static void main(String[] args) throws Exception {
}
// Submit to Storm cluster
- Helper.runOnClusterAndPrintMetrics(durationSec, TOPOLOGY_NAME, topoConf, getTopology(topoConf));
+ Helper.runOnClusterAndPrintMetrics(durationSec, TOPOLOGY_NAME, topoConf,
+ getTopology(topoConf));
}
}
diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/LowThroughputTopo.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/LowThroughputTopo.java
index 71ffd7caeae..de3fb0cdf7e 100644
--- a/examples/storm-perf/src/main/java/org/apache/storm/perf/LowThroughputTopo.java
+++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/LowThroughputTopo.java
@@ -49,7 +49,8 @@ static StormTopology getTopology(Map conf) {
Long sleepMs = ObjectReader.getLong(conf.get(SLEEP_MS));
// 1 - Setup Spout --------
- ThrottledSpout spout = new ThrottledSpout(sleepMs).withOutputFields(ThrottledSpout.DEFAULT_FIELD_NAME);
+ ThrottledSpout spout = new ThrottledSpout(sleepMs)
+ .withOutputFields(ThrottledSpout.DEFAULT_FIELD_NAME);
// 2 - Setup DevNull Bolt --------
LatencyPrintBolt bolt = new LatencyPrintBolt();
@@ -83,7 +84,8 @@ public static void main(String[] args) throws Exception {
}
topoConf.putAll(Utils.readCommandLineOpts());
// Submit topology to storm cluster
- Helper.runOnClusterAndPrintMetrics(runTime, "LowThroughputTopo", topoConf, getTopology(topoConf));
+ Helper.runOnClusterAndPrintMetrics(runTime, "LowThroughputTopo", topoConf,
+ getTopology(topoConf));
}
private static class ThrottledSpout extends BaseRichSpout {
diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/SimplifiedWordCountTopo.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/SimplifiedWordCountTopo.java
index 0a9148c593d..24d6175c57b 100644
--- a/examples/storm-perf/src/main/java/org/apache/storm/perf/SimplifiedWordCountTopo.java
+++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/SimplifiedWordCountTopo.java
@@ -42,7 +42,6 @@ public class SimplifiedWordCountTopo {
public static final int DEFAULT_SPOUT_NUM = 1;
public static final int DEFAULT_COUNT_BOLT_NUM = 1;
-
static StormTopology getTopology(Map config) {
final int spoutNum = Helper.getInt(config, SPOUT_NUM, DEFAULT_SPOUT_NUM);
@@ -51,7 +50,8 @@ static StormTopology getTopology(Map config) {
TopologyBuilder builder = new TopologyBuilder();
builder.setSpout(SPOUT_ID, new WordGenSpout(inputFile), spoutNum);
- builder.setBolt(COUNT_ID, new CountBolt(), cntBoltNum).fieldsGrouping(SPOUT_ID, new Fields(WordGenSpout.FIELDS));
+ builder.setBolt(COUNT_ID, new CountBolt(), cntBoltNum).fieldsGrouping(SPOUT_ID,
+ new Fields(WordGenSpout.FIELDS));
return builder.createTopology();
}
@@ -74,7 +74,8 @@ public static void main(String[] args) throws Exception {
topoConf.put(Config.TOPOLOGY_PRODUCER_BATCH_SIZE, 1000);
topoConf.put(Config.TOPOLOGY_DISABLE_LOADAWARE_MESSAGING, true);
topoConf.put(Config.TOPOLOGY_STATS_SAMPLE_RATE, 0.0005);
- topoConf.put(Config.TOPOLOGY_BOLT_WAIT_STRATEGY, "org.apache.storm.policy.WaitStrategyPark");
+ topoConf.put(Config.TOPOLOGY_BOLT_WAIT_STRATEGY,
+ "org.apache.storm.policy.WaitStrategyPark");
topoConf.put(Config.TOPOLOGY_BOLT_WAIT_PARK_MICROSEC, 0);
topoConf.putAll(Utils.readCommandLineOpts());
diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/StrGenSpoutHdfsBoltTopo.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/StrGenSpoutHdfsBoltTopo.java
index 0c53db97c6f..41bde2f37f5 100755
--- a/examples/storm-perf/src/main/java/org/apache/storm/perf/StrGenSpoutHdfsBoltTopo.java
+++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/StrGenSpoutHdfsBoltTopo.java
@@ -16,7 +16,6 @@
* limitations under the License
*/
-
package org.apache.storm.perf;
import java.util.Map;
@@ -63,7 +62,6 @@ public class StrGenSpoutHdfsBoltTopo {
public static final String SPOUT_ID = "GenSpout";
public static final String BOLT_ID = "hdfsBolt";
-
static StormTopology getTopology(Map topoConf) {
final int hdfsBatch = Helper.getInt(topoConf, HDFS_BATCH, DEFAULT_HDFS_BATCH);
@@ -75,12 +73,14 @@ static StormTopology getTopology(Map topoConf) {
String hdfsUrl = Helper.getStr(topoConf, HDFS_URI);
RecordFormat format = new LineWriter("str");
SyncPolicy syncPolicy = new CountSyncPolicy(hdfsBatch);
- FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(1.0f, FileSizeRotationPolicy.Units.GB);
+ FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(1.0f,
+ FileSizeRotationPolicy.Units.GB);
final int spoutNum = Helper.getInt(topoConf, SPOUT_NUM, DEFAULT_SPOUT_NUM);
final int boltNum = Helper.getInt(topoConf, BOLT_NUM, DEFAULT_BOLT_NUM);
// Use default, Storm-generated file names
- FileNameFormat fileNameFormat = new DefaultFileNameFormat().withPath(Helper.getStr(topoConf, HDFS_PATH));
+ FileNameFormat fileNameFormat = new DefaultFileNameFormat().withPath(Helper.getStr(topoConf,
+ HDFS_PATH));
// Instantiate the HdfsBolt
HdfsBolt bolt = new HdfsBolt()
@@ -101,13 +101,12 @@ static StormTopology getTopology(Map topoConf) {
return builder.createTopology();
}
-
/**
* Spout generates random strings and HDFS bolt writes them to a text file.
*/
public static void main(String[] args) throws Exception {
String confFile = "conf/HdfsSpoutTopo.yaml";
- int runTime = -1; //Run until Ctrl-C
+ int runTime = -1; // Run until Ctrl-C
if (args.length > 0) {
runTime = Integer.parseInt(args[0]);
}
@@ -124,7 +123,8 @@ public static void main(String[] args) throws Exception {
Map topoConf = Utils.findAndReadConfigFile(confFile);
topoConf.put(Config.TOPOLOGY_PRODUCER_BATCH_SIZE, 1000);
- topoConf.put(Config.TOPOLOGY_BOLT_WAIT_STRATEGY, "org.apache.storm.policy.WaitStrategyPark");
+ topoConf.put(Config.TOPOLOGY_BOLT_WAIT_STRATEGY,
+ "org.apache.storm.policy.WaitStrategyPark");
topoConf.put(Config.TOPOLOGY_BOLT_WAIT_PARK_MICROSEC, 0);
topoConf.put(Config.TOPOLOGY_DISABLE_LOADAWARE_MESSAGING, true);
topoConf.put(Config.TOPOLOGY_STATS_SAMPLE_RATE, 0.0005);
@@ -133,7 +133,6 @@ public static void main(String[] args) throws Exception {
Helper.runOnClusterAndPrintMetrics(runTime, TOPOLOGY_NAME, topoConf, getTopology(topoConf));
}
-
public static class LineWriter implements RecordFormat {
private static final long serialVersionUID = 7524288317405514146L;
private String lineDelimiter = System.lineSeparator();
diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/ThroughputMeter.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/ThroughputMeter.java
index 3d8e736224e..8a7b439b712 100644
--- a/examples/storm-perf/src/main/java/org/apache/storm/perf/ThroughputMeter.java
+++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/ThroughputMeter.java
@@ -32,6 +32,7 @@ public ThroughputMeter(String name) {
/**
* Calculate throughput.
+ *
* @return events/sec
*/
private static double calcThroughput(long count, long startTime, long endTime) {
diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/bolt/CountBolt.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/bolt/CountBolt.java
index ee1bf7ee1f7..0a4f0e82382 100644
--- a/examples/storm-perf/src/main/java/org/apache/storm/perf/bolt/CountBolt.java
+++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/bolt/CountBolt.java
@@ -28,7 +28,6 @@
import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values;
-
public class CountBolt extends BaseBasicBolt {
public static final String FIELDS_WORD = "word";
public static final String FIELDS_COUNT = "count";
diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/bolt/DevNullBolt.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/bolt/DevNullBolt.java
index cc181c04c47..6c0f6faf70c 100755
--- a/examples/storm-perf/src/main/java/org/apache/storm/perf/bolt/DevNullBolt.java
+++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/bolt/DevNullBolt.java
@@ -28,7 +28,6 @@
import org.apache.storm.utils.ObjectReader;
import org.slf4j.LoggerFactory;
-
public class DevNullBolt extends BaseRichBolt {
private static final org.slf4j.Logger LOG = LoggerFactory.getLogger(DevNullBolt.class);
private OutputCollector collector;
@@ -36,7 +35,8 @@ public class DevNullBolt extends BaseRichBolt {
private int count = 0;
@Override
- public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) {
+ public void prepare(Map topoConf, TopologyContext context,
+ OutputCollector collector) {
this.collector = collector;
this.sleepNanos = ObjectReader.getLong(topoConf.get("nullbolt.sleep.micros"), 0L) * 1_000;
}
diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/bolt/IdBolt.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/bolt/IdBolt.java
index 5387499bb8f..957ed7b1a75 100644
--- a/examples/storm-perf/src/main/java/org/apache/storm/perf/bolt/IdBolt.java
+++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/bolt/IdBolt.java
@@ -31,7 +31,8 @@ public class IdBolt extends BaseRichBolt {
private OutputCollector collector;
@Override
- public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) {
+ public void prepare(Map topoConf, TopologyContext context,
+ OutputCollector collector) {
this.collector = collector;
}
diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/bolt/SplitSentenceBolt.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/bolt/SplitSentenceBolt.java
index 85a3f5aa9d4..64e002b48c6 100644
--- a/examples/storm-perf/src/main/java/org/apache/storm/perf/bolt/SplitSentenceBolt.java
+++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/bolt/SplitSentenceBolt.java
@@ -27,7 +27,6 @@
import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values;
-
public class SplitSentenceBolt extends BaseBasicBolt {
public static final String FIELDS = "word";
diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/queuetest/Acker.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/queuetest/Acker.java
index adcc3a3d6bb..a5635bffe8b 100644
--- a/examples/storm-perf/src/main/java/org/apache/storm/perf/queuetest/Acker.java
+++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/queuetest/Acker.java
@@ -33,7 +33,6 @@ class Acker extends MyThread {
this.spoutInQ = spoutInQ;
}
-
@Override
public void run() {
long start = System.currentTimeMillis();
diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/queuetest/JCQueuePerfTest.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/queuetest/JCQueuePerfTest.java
index 2cbee19e340..883dc87cd7f 100644
--- a/examples/storm-perf/src/main/java/org/apache/storm/perf/queuetest/JCQueuePerfTest.java
+++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/queuetest/JCQueuePerfTest.java
@@ -47,8 +47,10 @@ public static void main(String[] args) throws Exception {
private static void ackingProducerSimulation() {
WaitStrategyPark ws = new WaitStrategyPark(100);
StormMetricRegistry registry = new StormMetricRegistry();
- JCQueue spoutQ = new JCQueue("spoutQ", "spoutQ", 1024, 0, 100, ws, "test", "test", Collections.singletonList(1000), 1000, registry);
- JCQueue ackQ = new JCQueue("ackQ", "ackQ", 1024, 0, 100, ws, "test", "test", Collections.singletonList(1000), 1000, registry);
+ JCQueue spoutQ = new JCQueue("spoutQ", "spoutQ", 1024, 0, 100, ws, "test", "test",
+ Collections.singletonList(1000), 1000, registry);
+ JCQueue ackQ = new JCQueue("ackQ", "ackQ", 1024, 0, 100, ws, "test", "test", Collections
+ .singletonList(1000), 1000, registry);
final AckingProducer ackingProducer = new AckingProducer(spoutQ, ackQ);
final Acker acker = new Acker(ackQ, spoutQ);
@@ -61,7 +63,8 @@ private static void producerFwdConsumer(int prodBatchSz) {
StormMetricRegistry registry = new StormMetricRegistry();
JCQueue q1 = new JCQueue("q1", "q1", 1024, 0, prodBatchSz, ws, "test", "test",
Collections.singletonList(1000), 1000, registry);
- JCQueue q2 = new JCQueue("q2", "q2", 1024, 0, prodBatchSz, ws, "test", "test", Collections.singletonList(1000), 1000, registry);
+ JCQueue q2 = new JCQueue("q2", "q2", 1024, 0, prodBatchSz, ws, "test", "test", Collections
+ .singletonList(1000), 1000, registry);
final Producer prod = new Producer(q1);
final Forwarder fwd = new Forwarder(q1, q2);
@@ -70,9 +73,9 @@ private static void producerFwdConsumer(int prodBatchSz) {
runAllThds(prod, fwd, cons);
}
-
private static void oneProducer1Consumer(int prodBatchSz) {
- JCQueue q1 = new JCQueue("q1", "q1", 50_000, 0, prodBatchSz, new WaitStrategyPark(100), "test", "test",
+ JCQueue q1 = new JCQueue("q1", "q1", 50_000, 0, prodBatchSz, new WaitStrategyPark(100),
+ "test", "test",
Collections.singletonList(1000), 1000, new StormMetricRegistry());
final Producer prod1 = new Producer(q1);
@@ -82,7 +85,8 @@ private static void oneProducer1Consumer(int prodBatchSz) {
}
private static void twoProducer1Consumer(int prodBatchSz) {
- JCQueue q1 = new JCQueue("q1", "q1", 50_000, 0, prodBatchSz, new WaitStrategyPark(100), "test", "test",
+ JCQueue q1 = new JCQueue("q1", "q1", 50_000, 0, prodBatchSz, new WaitStrategyPark(100),
+ "test", "test",
Collections.singletonList(1000), 1000, new StormMetricRegistry());
final Producer prod1 = new Producer(q1);
@@ -93,7 +97,8 @@ private static void twoProducer1Consumer(int prodBatchSz) {
}
private static void threeProducer1Consumer(int prodBatchSz) {
- JCQueue q1 = new JCQueue("q1", "q1", 50_000, 0, prodBatchSz, new WaitStrategyPark(100), "test", "test",
+ JCQueue q1 = new JCQueue("q1", "q1", 50_000, 0, prodBatchSz, new WaitStrategyPark(100),
+ "test", "test",
Collections.singletonList(1000), 1000, new StormMetricRegistry());
final Producer prod1 = new Producer(q1);
@@ -104,12 +109,13 @@ private static void threeProducer1Consumer(int prodBatchSz) {
runAllThds(prod1, prod2, prod3, cons1);
}
-
private static void oneProducer2Consumers(int prodBatchSz) {
WaitStrategyPark ws = new WaitStrategyPark(100);
StormMetricRegistry registry = new StormMetricRegistry();
- JCQueue q1 = new JCQueue("q1", "q1", 1024, 0, prodBatchSz, ws, "test", "test", Collections.singletonList(1000), 1000, registry);
- JCQueue q2 = new JCQueue("q2", "q2", 1024, 0, prodBatchSz, ws, "test", "test", Collections.singletonList(1000), 1000, registry);
+ JCQueue q1 = new JCQueue("q1", "q1", 1024, 0, prodBatchSz, ws, "test", "test", Collections
+ .singletonList(1000), 1000, registry);
+ JCQueue q2 = new JCQueue("q2", "q2", 1024, 0, prodBatchSz, ws, "test", "test", Collections
+ .singletonList(1000), 1000, registry);
final Producer2 prod1 = new Producer2(q1, q2);
final Consumer cons1 = new Consumer(q1);
@@ -140,7 +146,8 @@ public static void addShutdownHooks(MyThread... threads) {
}
for (MyThread thread : threads) {
- System.err.printf("%s : %d, Throughput: %,d \n", thread.getName(), thread.count, thread.throughput());
+ System.err.printf("%s : %d, Throughput: %,d \n", thread.getName(),
+ thread.count, thread.throughput());
}
} catch (InterruptedException e) {
return;
diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/spout/ConstSpout.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/spout/ConstSpout.java
index 656f992eff0..ed7ec079f2f 100755
--- a/examples/storm-perf/src/main/java/org/apache/storm/perf/spout/ConstSpout.java
+++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/spout/ConstSpout.java
@@ -52,7 +52,8 @@ public void declareOutputFields(OutputFieldsDeclarer declarer) {
}
@Override
- public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
+ public void open(Map conf, TopologyContext context,
+ SpoutOutputCollector collector) {
this.collector = collector;
this.sleep = ObjectReader.getLong(conf.get("spout.sleep"), 0L);
}
diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/spout/FileReadSpout.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/spout/FileReadSpout.java
index a819fa25e18..bdfb27fa316 100644
--- a/examples/storm-perf/src/main/java/org/apache/storm/perf/spout/FileReadSpout.java
+++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/spout/FileReadSpout.java
@@ -44,7 +44,6 @@ public class FileReadSpout extends BaseRichSpout {
private long count = 0;
-
public FileReadSpout(String file) {
this.file = file;
}
diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/spout/StringGenSpout.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/spout/StringGenSpout.java
index 1d0470c3eb6..d73e8613aec 100755
--- a/examples/storm-perf/src/main/java/org/apache/storm/perf/spout/StringGenSpout.java
+++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/spout/StringGenSpout.java
@@ -30,7 +30,8 @@
import org.apache.storm.tuple.Fields;
/**
- * Spout pre-computes a list with 30k fixed length random strings. Emits sequentially from this list, over and over again.
+ * Spout pre-computes a list with 30k fixed length random strings. Emits sequentially from this
+ * list, over and over again.
*/
public class StringGenSpout extends BaseRichSpout {
@@ -67,7 +68,8 @@ public void declareOutputFields(OutputFieldsDeclarer declarer) {
}
@Override
- public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
+ public void open(Map conf, TopologyContext context,
+ SpoutOutputCollector collector) {
this.records = genStringList(strLen, strCount);
this.collector = collector;
@@ -83,7 +85,6 @@ public void nextTuple() {
}
}
-
@Override
public void ack(Object msgId) {
super.ack(msgId);
diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/spout/WordGenSpout.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/spout/WordGenSpout.java
index a6a1fc9b391..c7ea29ac989 100644
--- a/examples/storm-perf/src/main/java/org/apache/storm/perf/spout/WordGenSpout.java
+++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/spout/WordGenSpout.java
@@ -45,7 +45,6 @@ public class WordGenSpout extends BaseRichSpout {
private ThroughputMeter emitMeter;
private ArrayList words;
-
public WordGenSpout(String file) {
this.file = file;
}
diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/toolstest/JCToolsPerfTest.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/toolstest/JCToolsPerfTest.java
index b5afe28b6a6..492a94549b4 100644
--- a/examples/storm-perf/src/main/java/org/apache/storm/perf/toolstest/JCToolsPerfTest.java
+++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/toolstest/JCToolsPerfTest.java
@@ -73,7 +73,6 @@ private static void threeProducer1Consumer() {
runAllThds(prod1, prod2, prod3, cons1);
}
-
private static void oneProducer2Consumers() {
MpscArrayQueue q1 = new MpscArrayQueue(50_000);
MpscArrayQueue q2 = new MpscArrayQueue(50_000);
@@ -107,7 +106,8 @@ public static void addShutdownHooks(MyThd... threads) {
}
for (MyThd thread : threads) {
- System.err.printf("%s : %d, Throughput: %,d \n", thread.getName(), thread.count, thread.throughput());
+ System.err.printf("%s : %d, Throughput: %,d \n", thread.getName(),
+ thread.count, thread.throughput());
}
} catch (InterruptedException e) {
return;
diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/utils/BasicMetricsCollector.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/utils/BasicMetricsCollector.java
index 735abe01a86..e94ff7b94f8 100755
--- a/examples/storm-perf/src/main/java/org/apache/storm/perf/utils/BasicMetricsCollector.java
+++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/utils/BasicMetricsCollector.java
@@ -132,10 +132,12 @@ void updateTopologyStats() {
void updateExecutorStats() {
long timeDiff = this.curSample.getSampleTime() - this.lastSample.getSampleTime();
- long transferredDiff = this.curSample.getTotalTransferred() - this.lastSample.getTotalTransferred();
+ long transferredDiff = this.curSample.getTotalTransferred() - this.lastSample
+ .getTotalTransferred();
long throughput = transferredDiff / (timeDiff / 1000);
- long spoutDiff = this.curSample.getSpoutTransferred() - this.lastSample.getSpoutTransferred();
+ long spoutDiff = this.curSample.getSpoutTransferred() - this.lastSample
+ .getSpoutTransferred();
long spoutAckedDiff = this.curSample.getTotalAcked() - this.lastSample.getTotalAcked();
long spoutThroughput = spoutDiff / (timeDiff / 1000);
@@ -192,11 +194,13 @@ void writeHeader(PrintWriter writer) {
}
writer.println(
- "\n------------------------------------------------------------------------------------------------------------------");
+ "\n-----------------------------------------------------------------------------------"
+ + "-------------------------------");
String str = Utils.join(header, ",");
writer.println(str);
writer
- .println("------------------------------------------------------------------------------------------------------------------");
+ .println("----------------------------------------------------------------------------"
+ + "--------------------------------------");
writer.flush();
}
diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/utils/Helper.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/utils/Helper.java
index a573308b607..8df90cf6e30 100755
--- a/examples/storm-perf/src/main/java/org/apache/storm/perf/utils/Helper.java
+++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/utils/Helper.java
@@ -27,7 +27,6 @@
import org.apache.storm.utils.ObjectReader;
import org.apache.storm.utils.Utils;
-
public class Helper {
public static void kill(Nimbus.Iface client, String topoName) throws Exception {
@@ -44,10 +43,12 @@ public static String getStr(Map map, Object key) {
return (String) map.get(key);
}
- public static void collectMetricsAndKill(String topologyName, Integer pollInterval, int duration) throws Exception {
+ public static void collectMetricsAndKill(String topologyName, Integer pollInterval,
+ int duration) throws Exception {
Map clusterConf = Utils.readStormConfig();
Nimbus.Iface client = NimbusClient.Builder.withConf(clusterConf).build().getClient();
- try (BasicMetricsCollector metricsCollector = new BasicMetricsCollector(topologyName, clusterConf)) {
+ try (BasicMetricsCollector metricsCollector = new BasicMetricsCollector(topologyName,
+ clusterConf)) {
if (duration > 0) {
int times = duration / pollInterval;
@@ -57,7 +58,7 @@ public static void collectMetricsAndKill(String topologyName, Integer pollInterv
metricsCollector.collect(client);
}
} else {
- while (true) { //until Ctrl-C
+ while (true) { // until Ctrl-C
metricsCollector.collect(client);
Thread.sleep(pollInterval * 1000);
}
@@ -87,7 +88,8 @@ public void run() {
});
}
- public static void runOnClusterAndPrintMetrics(int durationSec, String topoName, Map topoConf, StormTopology topology)
+ public static void runOnClusterAndPrintMetrics(int durationSec, String topoName, Map topoConf, StormTopology topology)
throws Exception {
// submit topology
StormSubmitter.submitTopologyWithProgressBar(topoName, topoConf, topology);
diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/utils/IdentityBolt.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/utils/IdentityBolt.java
index df27263b150..5c6d9d010bf 100755
--- a/examples/storm-perf/src/main/java/org/apache/storm/perf/utils/IdentityBolt.java
+++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/utils/IdentityBolt.java
@@ -29,7 +29,8 @@ public class IdentityBolt extends BaseRichBolt {
private OutputCollector collector;
@Override
- public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) {
+ public void prepare(Map topoConf, TopologyContext context,
+ OutputCollector collector) {
this.collector = collector;
}
diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/utils/MetricsSample.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/utils/MetricsSample.java
index c411844a34b..3aac2c5222e 100755
--- a/examples/storm-perf/src/main/java/org/apache/storm/perf/utils/MetricsSample.java
+++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/utils/MetricsSample.java
@@ -131,7 +131,7 @@ private static MetricsSample getMetricsSample(TopologyInfo topInfo) {
}
}
- Double total = 0d;
+ Double total = 0D;
Map vals = spoutStats.get_complete_ms_avg().get(":all-time");
if (vals != null) {
for (String key : vals.keySet()) {
diff --git a/examples/storm-redis-examples/pom.xml b/examples/storm-redis-examples/pom.xml
index 68db938f648..662f5e2a36c 100644
--- a/examples/storm-redis-examples/pom.xml
+++ b/examples/storm-redis-examples/pom.xml
@@ -89,6 +89,16 @@
+
+ org.openrewrite.maven
+ rewrite-maven-plugin
+
+
+
+ org.codehaus.gmavenplus
+ gmavenplus-plugin
+
+ org.apache.maven.pluginsmaven-checkstyle-plugin
diff --git a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/tools/Base64ToBinaryStateMigrationUtil.java b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/tools/Base64ToBinaryStateMigrationUtil.java
index 9d0d4c113d4..e3cafc3a2cc 100644
--- a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/tools/Base64ToBinaryStateMigrationUtil.java
+++ b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/tools/Base64ToBinaryStateMigrationUtil.java
@@ -21,7 +21,6 @@
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
-
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.DefaultParser;
@@ -38,7 +37,8 @@
import redis.clients.jedis.util.SafeEncoder;
public class Base64ToBinaryStateMigrationUtil {
- private static final Logger LOG = LoggerFactory.getLogger(Base64ToBinaryStateMigrationUtil.class);
+ private static final Logger LOG = LoggerFactory
+ .getLogger(Base64ToBinaryStateMigrationUtil.class);
private static final String OPTION_REDIS_HOST_SHORT = "h";
private static final String OPTION_REDIS_HOST_LONG = "host";
private static final String OPTION_REDIS_PORT_SHORT = "p";
@@ -139,7 +139,8 @@ public static void main(String[] args) throws IOException, ParseException {
.setTimeout(2000)
.build();
- Base64ToBinaryStateMigrationUtil migrationUtil = new Base64ToBinaryStateMigrationUtil(jedisPoolConfig);
+ Base64ToBinaryStateMigrationUtil migrationUtil =
+ new Base64ToBinaryStateMigrationUtil(jedisPoolConfig);
for (String namespace : namespaces) {
migrationUtil.migrate(namespace);
@@ -150,11 +151,16 @@ public static void main(String[] args) throws IOException, ParseException {
private static Options buildOptions() {
Options options = new Options();
- options.addOption(OPTION_NAMESPACE_SHORT, OPTION_NAMESPACE_LONG, true, "REQUIRED the list of namespace to migrate.");
- options.addOption(OPTION_REDIS_HOST_SHORT, OPTION_REDIS_HOST_LONG, true, "Redis hostname (default: localhost)");
- options.addOption(OPTION_REDIS_PORT_SHORT, OPTION_REDIS_PORT_LONG, true, "Redis port (default: 6379)");
- options.addOption(null, OPTION_REDIS_PASSWORD_LONG, true, "Redis password (default: no password)");
- options.addOption(OPTION_REDIS_DB_NUM_SHORT, OPTION_REDIS_DB_NUM_LONG, true, "Redis DB number (default: 0)");
+ options.addOption(OPTION_NAMESPACE_SHORT, OPTION_NAMESPACE_LONG, true,
+ "REQUIRED the list of namespace to migrate.");
+ options.addOption(OPTION_REDIS_HOST_SHORT, OPTION_REDIS_HOST_LONG, true,
+ "Redis hostname (default: localhost)");
+ options.addOption(OPTION_REDIS_PORT_SHORT, OPTION_REDIS_PORT_LONG, true,
+ "Redis port (default: 6379)");
+ options.addOption(null, OPTION_REDIS_PASSWORD_LONG, true,
+ "Redis password (default: no password)");
+ options.addOption(OPTION_REDIS_DB_NUM_SHORT, OPTION_REDIS_DB_NUM_LONG, true,
+ "Redis DB number (default: 0)");
return options;
}
diff --git a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/topology/LookupWordCount.java b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/topology/LookupWordCount.java
index 243f1bffc46..e7630dc06e3 100644
--- a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/topology/LookupWordCount.java
+++ b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/topology/LookupWordCount.java
@@ -7,9 +7,9 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
+ *
Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
@@ -19,11 +19,9 @@
package org.apache.storm.redis.topology;
import com.google.common.collect.Lists;
-
import java.util.List;
import java.util.Map;
import java.util.Random;
-
import org.apache.storm.Config;
import org.apache.storm.StormSubmitter;
import org.apache.storm.redis.bolt.RedisLookupBolt;
@@ -39,7 +37,6 @@
import org.apache.storm.tuple.ITuple;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -57,7 +54,8 @@ public static class PrintWordTotalCountBolt extends BaseRichBolt {
private OutputCollector collector;
@Override
- public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) {
+ public void prepare(Map topoConf, TopologyContext context,
+ OutputCollector collector) {
this.collector = collector;
}
@@ -101,7 +99,7 @@ public static void main(String[] args) throws Exception {
PrintWordTotalCountBolt printBolt = new PrintWordTotalCountBolt();
- //wordspout -> lookupbolt
+ // wordspout -> lookupbolt
TopologyBuilder builder = new TopologyBuilder();
builder.setSpout(WORD_SPOUT, spout, 1);
builder.setBolt(LOOKUP_BOLT, lookupBolt, 1).shuffleGrouping(WORD_SPOUT);
diff --git a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/topology/PersistentWordCount.java b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/topology/PersistentWordCount.java
index 93f790b9921..3b4e7f94ee2 100644
--- a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/topology/PersistentWordCount.java
+++ b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/topology/PersistentWordCount.java
@@ -7,9 +7,9 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
+ *
Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
@@ -64,7 +64,8 @@ public static void main(String[] args) throws Exception {
if (args.length == 3) {
topoName = args[2];
} else if (args.length > 3) {
- System.out.println("Usage: PersistentWordCount (topology name)");
+ System.out
+ .println("Usage: PersistentWordCount (topology name)");
return;
}
Config config = new Config();
diff --git a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/topology/WhitelistWordCount.java b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/topology/WhitelistWordCount.java
index 5b47f0cd51e..e600f1192c6 100644
--- a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/topology/WhitelistWordCount.java
+++ b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/topology/WhitelistWordCount.java
@@ -7,9 +7,9 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
+ *
Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
@@ -20,7 +20,6 @@
import java.util.Map;
import java.util.Random;
-
import org.apache.storm.Config;
import org.apache.storm.StormSubmitter;
import org.apache.storm.redis.bolt.RedisFilterBolt;
@@ -53,7 +52,8 @@ public static class PrintWordTotalCountBolt extends BaseRichBolt {
private OutputCollector collector;
@Override
- public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) {
+ public void prepare(Map topoConf, TopologyContext context,
+ OutputCollector collector) {
this.collector = collector;
}
@@ -99,7 +99,8 @@ public static void main(String[] args) throws Exception {
TopologyBuilder builder = new TopologyBuilder();
builder.setSpout(WORD_SPOUT, spout, 1);
builder.setBolt(WHITELIST_BOLT, whitelistBolt, 1).shuffleGrouping(WORD_SPOUT);
- builder.setBolt(COUNT_BOLT, wordCounterBolt, 1).fieldsGrouping(WHITELIST_BOLT, new Fields("word"));
+ builder.setBolt(COUNT_BOLT, wordCounterBolt, 1).fieldsGrouping(WHITELIST_BOLT,
+ new Fields("word"));
PrintWordTotalCountBolt printBolt = new PrintWordTotalCountBolt();
builder.setBolt(PRINT_BOLT, printBolt, 1).shuffleGrouping(COUNT_BOLT);
@@ -107,7 +108,8 @@ public static void main(String[] args) throws Exception {
if (args.length == 3) {
topoName = args[2];
} else if (args.length > 3) {
- System.out.println("Usage: WhitelistWordCount [topology name]");
+ System.out
+ .println("Usage: WhitelistWordCount [topology name]");
return;
}
Config config = new Config();
diff --git a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/topology/WordCounter.java b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/topology/WordCounter.java
index a0b9714c804..2d102234215 100644
--- a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/topology/WordCounter.java
+++ b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/topology/WordCounter.java
@@ -7,9 +7,9 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
+ *
Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
@@ -19,9 +19,7 @@
package org.apache.storm.redis.topology;
import com.google.common.collect.Maps;
-
import java.util.Map;
-
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.BasicOutputCollector;
import org.apache.storm.topology.IBasicBolt;
diff --git a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/topology/WordSpout.java b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/topology/WordSpout.java
index ff3ec39101b..78f8f84bbcc 100644
--- a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/topology/WordSpout.java
+++ b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/topology/WordSpout.java
@@ -7,9 +7,9 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
+ *
Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
@@ -21,7 +21,6 @@
import java.util.Map;
import java.util.Random;
import java.util.UUID;
-
import org.apache.storm.spout.SpoutOutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.IRichSpout;
@@ -32,7 +31,8 @@
public class WordSpout implements IRichSpout {
boolean isDistributed;
SpoutOutputCollector collector;
- public static final String[] words = new String[] { "apple", "orange", "pineapple", "banana", "watermelon" };
+ public static final String[] words =
+ new String[] { "apple", "orange", "pineapple", "banana", "watermelon" };
public WordSpout() {
this(true);
@@ -47,7 +47,8 @@ public boolean isDistributed() {
}
@Override
- public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
+ public void open(Map conf, TopologyContext context,
+ SpoutOutputCollector collector) {
this.collector = collector;
}
diff --git a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/PrintFunction.java b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/PrintFunction.java
index f32cfe8e037..b7229d09e97 100644
--- a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/PrintFunction.java
+++ b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/PrintFunction.java
@@ -7,9 +7,9 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
+ *
Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
@@ -19,11 +19,9 @@
package org.apache.storm.redis.trident;
import java.util.Random;
-
import org.apache.storm.trident.operation.BaseFunction;
import org.apache.storm.trident.operation.TridentCollector;
import org.apache.storm.trident.tuple.TridentTuple;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountLookupMapper.java b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountLookupMapper.java
index 04874d5f13c..909ffb6330b 100644
--- a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountLookupMapper.java
+++ b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountLookupMapper.java
@@ -7,9 +7,9 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
+ *
Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
@@ -20,7 +20,6 @@
import java.util.ArrayList;
import java.util.List;
-
import org.apache.storm.redis.common.mapper.RedisDataTypeDescription;
import org.apache.storm.redis.common.mapper.RedisLookupMapper;
import org.apache.storm.topology.OutputFieldsDeclarer;
diff --git a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountStoreMapper.java b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountStoreMapper.java
index 84538e232e4..d99454ad3e4 100644
--- a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountStoreMapper.java
+++ b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountStoreMapper.java
@@ -7,9 +7,9 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
+ *
Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
diff --git a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountTridentRedis.java b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountTridentRedis.java
index 1e9991fd3aa..4302486bb2f 100644
--- a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountTridentRedis.java
+++ b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountTridentRedis.java
@@ -7,9 +7,9 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
+ *
Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
@@ -82,6 +82,7 @@ public static void main(String[] args) throws Exception {
Config conf = new Config();
conf.setMaxSpoutPending(5);
conf.setNumWorkers(3);
- StormSubmitter.submitTopology("test_wordCounter_for_redis", conf, buildTopology(redisHost, redisPort));
+ StormSubmitter.submitTopology("test_wordCounter_for_redis", conf, buildTopology(redisHost,
+ redisPort));
}
}
diff --git a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountTridentRedisCluster.java b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountTridentRedisCluster.java
index 7ced2afdadf..fe8fdbbeeb7 100644
--- a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountTridentRedisCluster.java
+++ b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountTridentRedisCluster.java
@@ -7,9 +7,9 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
+ *
Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
@@ -21,7 +21,6 @@
import java.net.InetSocketAddress;
import java.util.HashSet;
import java.util.Set;
-
import org.apache.storm.Config;
import org.apache.storm.StormSubmitter;
import org.apache.storm.generated.StormTopology;
@@ -89,7 +88,8 @@ public static void main(String[] args) throws Exception {
Config conf = new Config();
conf.setMaxSpoutPending(5);
conf.setNumWorkers(3);
- StormSubmitter.submitTopology("test_wordCounter_for_redis", conf, buildTopology(redisHostPort));
+ StormSubmitter.submitTopology("test_wordCounter_for_redis", conf,
+ buildTopology(redisHostPort));
}
}
diff --git a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountTridentRedisClusterMap.java b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountTridentRedisClusterMap.java
index b1425c9bf34..9b5cadb3261 100644
--- a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountTridentRedisClusterMap.java
+++ b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountTridentRedisClusterMap.java
@@ -7,9 +7,9 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
+ *
Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
@@ -21,7 +21,6 @@
import java.net.InetSocketAddress;
import java.util.HashSet;
import java.util.Set;
-
import org.apache.storm.Config;
import org.apache.storm.StormSubmitter;
import org.apache.storm.generated.StormTopology;
@@ -59,7 +58,8 @@ public static StormTopology buildTopology(String redisHostPort) {
.build();
RedisDataTypeDescription dataTypeDescription = new RedisDataTypeDescription(
RedisDataTypeDescription.RedisDataType.HASH, "test");
- StateFactory factory = RedisClusterMapState.transactional(clusterConfig, dataTypeDescription);
+ StateFactory factory = RedisClusterMapState.transactional(clusterConfig,
+ dataTypeDescription);
TridentTopology topology = new TridentTopology();
Stream stream = topology.newStream("spout1", spout);
@@ -83,7 +83,8 @@ public static void main(String[] args) throws Exception {
Config conf = new Config();
conf.setMaxSpoutPending(5);
conf.setNumWorkers(3);
- StormSubmitter.submitTopology("test_wordCounter_for_redis", conf, buildTopology(redisHostPort));
+ StormSubmitter.submitTopology("test_wordCounter_for_redis", conf,
+ buildTopology(redisHostPort));
}
}
diff --git a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountTridentRedisMap.java b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountTridentRedisMap.java
index 2a7af97f3b8..02d34d9fcb4 100644
--- a/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountTridentRedisMap.java
+++ b/examples/storm-redis-examples/src/main/java/org/apache/storm/redis/trident/WordCountTridentRedisMap.java
@@ -7,9 +7,9 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
+ *
Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
@@ -77,7 +77,8 @@ public static void main(String[] args) throws Exception {
Config conf = new Config();
conf.setMaxSpoutPending(5);
conf.setNumWorkers(3);
- StormSubmitter.submitTopology("test_wordCounter_for_redis", conf, buildTopology(redisHost, redisPort));
+ StormSubmitter.submitTopology("test_wordCounter_for_redis", conf, buildTopology(redisHost,
+ redisPort));
}
}
diff --git a/examples/storm-starter/pom.xml b/examples/storm-starter/pom.xml
index f7632397f7a..b6dc1e66c3d 100644
--- a/examples/storm-starter/pom.xml
+++ b/examples/storm-starter/pom.xml
@@ -174,6 +174,16 @@
none
+
+ org.openrewrite.maven
+ rewrite-maven-plugin
+
+
+
+ org.codehaus.gmavenplus
+ gmavenplus-plugin
+
+ org.apache.maven.pluginsmaven-checkstyle-plugin
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/AnchoredWordCount.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/AnchoredWordCount.java
index 1809cf19fa1..d6f7d63e905 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/AnchoredWordCount.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/AnchoredWordCount.java
@@ -1,12 +1,17 @@
/*
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -58,7 +63,6 @@ public static class RandomSentenceSpout extends BaseRichSpout {
SpoutOutputCollector collector;
Random random;
-
@Override
public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
this.collector = collector;
@@ -69,7 +73,8 @@ public void open(Map conf, TopologyContext context, SpoutOutputCollector collect
public void nextTuple() {
Utils.sleep(10);
String[] sentences = new String[]{
- sentence("the cow jumped over the moon"), sentence("an apple a day keeps the doctor away"),
+ sentence("the cow jumped over the moon"), sentence("an apple a day keeps the "
+ + "doctor away"),
sentence("four score and seven years ago"),
sentence("snow white and the seven dwarfs"), sentence("i am at two with nature")
};
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/BasicDRPCTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/BasicDRPCTopology.java
index 70e03a4761e..f647fa7d34c 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/BasicDRPCTopology.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/BasicDRPCTopology.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -24,7 +30,8 @@
import org.apache.storm.utils.DRPCClient;
/**
- * This topology is a basic example of doing distributed RPC on top of Storm. It implements a function that appends a
+ * This topology is a basic example of doing distributed RPC on top of Storm. It implements a
+ * function that appends a
* "!" to any string you send the DRPC function.
*
* @see Distributed RPC
@@ -48,13 +55,15 @@ public static void main(String[] args) throws Exception {
builder.addBolt(new ExclaimBolt(), 3);
conf.setNumWorkers(3);
- StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder.createRemoteTopology());
+ StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder
+ .createRemoteTopology());
if (args != null && args.length > 2) {
try (DRPCClient drpc = DRPCClient.getConfiguredClient(conf)) {
for (int i = 2; i < args.length; i++) {
String word = args[i];
- System.out.println("Result for \"" + word + "\": " + drpc.execute(function, word));
+ System.out.println("Result for \"" + word + "\": " + drpc.execute(function,
+ word));
}
}
}
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/BlobStoreAPIWordCountTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/BlobStoreAPIWordCountTopology.java
index f8059f3c1fc..be2b89292be 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/BlobStoreAPIWordCountTopology.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/BlobStoreAPIWordCountTopology.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -69,7 +75,8 @@ public static void prepare() {
// Equivalent create command on command line
// storm blobstore create --file blacklist.txt --acl o::rwa key
- private static void createBlobWithContent(String blobKey, ClientBlobStore clientBlobStore, File file)
+ private static void createBlobWithContent(String blobKey, ClientBlobStore clientBlobStore,
+ File file)
throws AuthorizationException, KeyAlreadyExistsException, IOException, KeyNotFoundException {
String stringBlobAcl = "o::rwa";
AccessControl blobAcl = BlobStoreAclHandler.parseAccessControl(stringBlobAcl);
@@ -83,7 +90,8 @@ private static void createBlobWithContent(String blobKey, ClientBlobStore client
// Equivalent update command on command line
// storm blobstore update --file blacklist.txt key
- private static void updateBlobWithContent(String blobKey, ClientBlobStore clientBlobStore, File file)
+ private static void updateBlobWithContent(String blobKey, ClientBlobStore clientBlobStore,
+ File file)
throws KeyNotFoundException, AuthorizationException, IOException {
AtomicOutputStream blobOutputStream = clientBlobStore.updateBlob(blobKey);
blobOutputStream.write(readFile(file).toString().getBytes());
@@ -93,7 +101,8 @@ private static void updateBlobWithContent(String blobKey, ClientBlobStore client
private static String getRandomSentence() {
String[] sentences = new String[]{
"the cow jumped over the moon", "an apple a day keeps the doctor away",
- "four score and seven years ago", "snow white and the seven dwarfs", "i am at two with nature"
+ "four score and seven years ago", "snow white and the seven dwarfs", "i am at two "
+ + "with nature"
};
String sentence = sentences[new Random().nextInt(sentences.length)];
return sentence;
@@ -218,7 +227,8 @@ public static class RandomSentenceSpout extends BaseRichSpout {
SpoutOutputCollector collector;
@Override
- public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
+ public void open(Map conf, TopologyContext context,
+ SpoutOutputCollector collector) {
this.collector = collector;
}
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/ExclamationTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/ExclamationTopology.java
index 31854f613f3..1157b12641c 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/ExclamationTopology.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/ExclamationTopology.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -58,7 +64,8 @@ public static class ExclamationBolt extends BaseRichBolt {
OutputCollector collector;
@Override
- public void prepare(Map conf, TopologyContext context, OutputCollector collector) {
+ public void prepare(Map conf, TopologyContext context,
+ OutputCollector collector) {
this.collector = collector;
}
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/FastWordCountTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/FastWordCountTopology.java
index e5da703380b..a62656515b7 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/FastWordCountTopology.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/FastWordCountTopology.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -107,7 +113,7 @@ public static void main(String[] args) throws Exception {
clusterConf.putAll(Utils.readCommandLineOpts());
Nimbus.Iface client = NimbusClient.Builder.withConf(clusterConf).build().getClient();
- //Sleep for 5 mins
+ // Sleep for 5 mins
for (int i = 0; i < 10; i++) {
Thread.sleep(30 * 1000);
printMetrics(client, name);
@@ -127,7 +133,8 @@ public static class FastRandomSentenceSpout extends BaseRichSpout {
Random rand;
@Override
- public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
+ public void open(Map conf, TopologyContext context,
+ SpoutOutputCollector collector) {
this.collector = collector;
rand = ThreadLocalRandom.current();
}
@@ -140,7 +147,7 @@ public void nextTuple() {
@Override
public void ack(Object id) {
- //Ignored
+ // Ignored
}
@Override
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/InOrderDeliveryTest.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/InOrderDeliveryTest.java
index 4820aef2026..6e5c141fb16 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/InOrderDeliveryTest.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/InOrderDeliveryTest.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -100,7 +106,7 @@ public static void main(String[] args) throws Exception {
clusterConf.putAll(Utils.readCommandLineOpts());
Nimbus.Iface client = NimbusClient.Builder.withConf(clusterConf).build().getClient();
- //Sleep for 50 mins
+ // Sleep for 50 mins
for (int i = 0; i < 50; i++) {
Thread.sleep(30 * 1000);
printMetrics(client, name);
@@ -114,7 +120,8 @@ public static class InOrderSpout extends BaseRichSpout {
int count = 0;
@Override
- public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
+ public void open(Map conf, TopologyContext context,
+ SpoutOutputCollector collector) {
this.collector = collector;
base = context.getThisTaskIndex();
}
@@ -128,12 +135,12 @@ public void nextTuple() {
@Override
public void ack(Object id) {
- //Ignored
+ // Ignored
}
@Override
public void fail(Object id) {
- //Ignored
+ // Ignored
}
@Override
@@ -163,7 +170,7 @@ public void execute(Tuple tuple, BasicOutputCollector collector) {
@Override
public void declareOutputFields(OutputFieldsDeclarer declarer) {
- //Empty
+ // Empty
}
}
}
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/JoinBoltExample.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/JoinBoltExample.java
index 6fc373970bf..ccaee7a8584 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/JoinBoltExample.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/JoinBoltExample.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -50,7 +56,8 @@ public static void main(String[] args) throws Exception {
builder.setBolt("printer", new PrinterBolt()).shuffleGrouping("joiner");
Config conf = new Config();
- StormSubmitter.submitTopologyWithProgressBar("join-example", conf, builder.createTopology());
+ StormSubmitter.submitTopologyWithProgressBar("join-example", conf, builder
+ .createTopology());
generateGenderData(genderSpout);
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/LambdaTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/LambdaTopology.java
index d5dec296896..78a7b9c74c5 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/LambdaTopology.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/LambdaTopology.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/ManualDRPC.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/ManualDRPC.java
index e685ca16c2e..640404667a3 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/ManualDRPC.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/ManualDRPC.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/MultiThreadWordCountTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/MultiThreadWordCountTopology.java
index 528b98d2b9f..b89c4591d7d 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/MultiThreadWordCountTopology.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/MultiThreadWordCountTopology.java
@@ -30,8 +30,10 @@ Licensed to the Apache Software Foundation (ASF) under one or more contributor l
import org.apache.storm.tuple.Values;
/**
- * Some topologies might spawn some threads within bolts to do some work and emit tuples from those threads.
- * This is a simple wordcount topology example that mimics those use cases and might help us catch possible race conditions.
+ * Some topologies might spawn some threads within bolts to do some work and emit tuples from those
+ * threads.
+ * This is a simple wordcount topology example that mimics those use cases and might help us catch
+ * possible race conditions.
*/
public class MultiThreadWordCountTopology extends ConfigurableTopology {
public static void main(String[] args) {
@@ -45,12 +47,14 @@ protected int run(String[] args) {
builder.setSpout("spout", new RandomSentenceSpout(), 1);
builder.setBolt("split", new MultiThreadedSplitSentence(), 1).shuffleGrouping("spout");
- builder.setBolt("count", new WordCountBolt(), 1).fieldsGrouping("split", new Fields("word"));
+ builder.setBolt("count", new WordCountBolt(), 1).fieldsGrouping("split",
+ new Fields("word"));
- //this makes sure there is only one executor per worker, easier to debug
- //problems involving serialization/deserialization will only happen in inter-worker data transfer
+ // this makes sure there is only one executor per worker, easier to debug
+ // problems involving serialization/deserialization will only happen in inter-worker data
+ // transfer
conf.put(Config.TOPOLOGY_RAS_ONE_EXECUTOR_PER_WORKER, true);
- //this involves metricsTick
+ // this involves metricsTick
conf.registerMetricsConsumer(LoggingMetricsConsumer.class);
conf.setTopologyWorkerMaxHeapSize(128);
@@ -69,11 +73,13 @@ public static class MultiThreadedSplitSentence implements IRichBolt {
private ExecutorService executor;
@Override
- public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) {
+ public void prepare(Map topoConf, TopologyContext context,
+ OutputCollector collector) {
this.collector = collector;
executor = Executors.newFixedThreadPool(6);
- //This makes sure metricsTick to be called every 1 second
- //it makes the race condition between metricsTick and outputCollector easier to happen if any
+ // This makes sure metricsTick to be called every 1 second
+ // it makes the race condition between metricsTick and outputCollector easier to happen
+ // if any
context.registerMetric("dummy-counter", () -> 0, 1);
}
@@ -82,7 +88,7 @@ public void execute(Tuple input) {
String str = input.getString(0);
String[] splits = str.split("\\s+");
for (String s : splits) {
- //spawn other threads to do the work and emit
+ // spawn other threads to do the work and emit
Runnable runnableTask = () -> {
collector.emit(new Values(s));
};
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/MultipleLoggerTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/MultipleLoggerTopology.java
index 9410931553f..f7491b8994d 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/MultipleLoggerTopology.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/MultipleLoggerTopology.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -57,7 +63,8 @@ public static class ExclamationLoggingBolt extends BaseRichBolt {
Logger subLogger = LoggerFactory.getLogger("com.myapp.sub");
@Override
- public void prepare(Map conf, TopologyContext context, OutputCollector collector) {
+ public void prepare(Map conf, TopologyContext context,
+ OutputCollector collector) {
this.collector = collector;
}
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/PersistentWindowingTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/PersistentWindowingTopology.java
index 62b3ed577ec..37f0eabacd8 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/PersistentWindowingTopology.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/PersistentWindowingTopology.java
@@ -41,14 +41,18 @@
import org.slf4j.LoggerFactory;
/**
- * An example that demonstrates the usage of {@link org.apache.storm.topology.IStatefulWindowedBolt} with window persistence.
- *
- * The framework automatically checkpoints the tuples in the window along with the bolt's state and restores the same during restarts.
+ * An example that demonstrates the usage of {@link org.apache.storm.topology.IStatefulWindowedBolt}
+ * with window persistence.
+ *
+ *
The framework automatically checkpoints the tuples in the window along with the bolt's state
+ * and
+ * restores the same during restarts.
*
*
- *
- * This topology uses 'redis' for state persistence, so you should also start a redis instance before deploying. If you are running in local
- * mode you can just start a redis server locally which will be used for storing the state. The default RedisKeyValueStateProvider
+ *
This topology uses 'redis' for state persistence, so you should also start a redis instance
+ * before deploying. If you are running in local
+ * mode you can just start a redis server locally which will be used for storing the state. The
+ * default RedisKeyValueStateProvider
* parameters can be overridden by setting {@link Config#TOPOLOGY_STATE_PROVIDER_CONFIG}, for e.g.
*
* {
@@ -88,7 +92,8 @@ public static void main(String[] args) throws Exception {
.shuffleGrouping("spout");
// print the values to stdout
- builder.setBolt("printer", (x, y) -> System.out.println(x.getValue(0)), 1).shuffleGrouping("avgbolt");
+ builder.setBolt("printer", (x, y) -> System.out.println(x.getValue(0)), 1)
+ .shuffleGrouping("avgbolt");
Config conf = new Config();
conf.setDebug(false);
@@ -97,7 +102,8 @@ public static void main(String[] args) throws Exception {
conf.put(Config.TOPOLOGY_STATE_CHECKPOINT_INTERVAL, 5000);
// use redis for state persistence
- conf.put(Config.TOPOLOGY_STATE_PROVIDER, "org.apache.storm.redis.state.RedisKeyValueStateProvider");
+ conf.put(Config.TOPOLOGY_STATE_PROVIDER,
+ "org.apache.storm.redis.state.RedisKeyValueStateProvider");
String topoName = "test";
if (args != null && args.length > 0) {
@@ -119,7 +125,8 @@ private static class Averages {
@Override
public String toString() {
- return "Averages{" + "global=" + String.format("%.2f", global) + ", window=" + String.format("%.2f", window) + '}';
+ return "Averages{" + "global=" + String.format("%.2f", global) + ", window=" + String
+ .format("%.2f", window) + '}';
}
}
@@ -134,7 +141,8 @@ private static class AvgBolt extends BaseStatefulWindowedBolt globalAvg;
@Override
- public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) {
+ public void prepare(Map topoConf, TopologyContext context,
+ OutputCollector collector) {
this.collector = collector;
}
@@ -142,7 +150,8 @@ public void prepare(Map topoConf, TopologyContext context, Outpu
public void initState(KeyValueState> state) {
this.state = state;
globalAvg = state.get(STATE_KEY, Pair.of(0L, 0L));
- LOG.info("initState with global avg [" + (double) globalAvg.getFirst() / globalAvg.getSecond() + "]");
+ LOG.info("initState with global avg [" + (double) globalAvg.getFirst() / globalAvg
+ .getSecond() + "]");
}
@Override
@@ -161,7 +170,8 @@ public void execute(TupleWindow window) {
// update the value in state
state.put(STATE_KEY, globalAvg);
// emit the averages downstream
- collector.emit(new Values(new Averages((double) globalAvg.getFirst() / globalAvg.getSecond(), (double) sum / count)));
+ collector.emit(new Values(new Averages((double) globalAvg.getFirst() / globalAvg
+ .getSecond(), (double) sum / count)));
}
@Override
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/Prefix.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/Prefix.java
index 31f200a83dc..1ce4d9e9de7 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/Prefix.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/Prefix.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/ReachTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/ReachTopology.java
index 59051326942..f2a1bc2ac32 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/ReachTopology.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/ReachTopology.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -33,18 +39,24 @@
import org.apache.storm.utils.DRPCClient;
/**
- * This is a good example of doing complex Distributed RPC on top of Storm. This program creates a topology that can
+ * This is a good example of doing complex Distributed RPC on top of Storm. This program creates a
+ * topology that can
* compute the reach for any URL on Twitter in realtime by parallelizing the whole computation.
*
- *
Reach is the number of unique people exposed to a URL on Twitter. To compute reach, you have to get all the people
- * who tweeted the URL, get all the followers of all those people, unique that set of followers, and then count the
- * unique set. It's an intense computation that can involve thousands of database calls and tens of millions of follower
+ *
Reach is the number of unique people exposed to a URL on Twitter. To compute reach, you have
+ * to get all the people
+ * who tweeted the URL, get all the followers of all those people, unique that set of followers, and
+ * then count the
+ * unique set. It's an intense computation that can involve thousands of database calls and tens of
+ * millions of follower
* records.
*
- *
This Storm topology does every piece of that computation in parallel, turning what would be a computation that takes
+ *
This Storm topology does every piece of that computation in parallel, turning what would be a
+ * computation that takes
* minutes on a single machine into one that takes just a couple seconds.
*
- *
For the purposes of demonstration, this topology replaces the use of actual DBs with in-memory hashmaps.
+ *
For the purposes of demonstration, this topology replaces the use of actual DBs with in-memory
+ * hashmaps.
*
* @see Distributed RPC
*/
@@ -52,7 +64,8 @@ public class ReachTopology {
public static Map> TWEETERS_DB = new HashMap>() {
{
put("foo.com/blog/1", Arrays.asList("sally", "bob", "tim", "george", "nathan"));
- put("engineering.twitter.com/blog/5", Arrays.asList("adam", "david", "sally", "nathan"));
+ put("engineering.twitter.com/blog/5", Arrays.asList("adam", "david", "sally",
+ "nathan"));
put("tech.backtype.com/blog/123", Arrays.asList("tim", "mike", "john"));
}
};
@@ -62,7 +75,8 @@ public class ReachTopology {
put("sally", Arrays.asList("bob", "tim", "alice", "adam", "jim", "chris", "jai"));
put("bob", Arrays.asList("sally", "nathan", "jim", "mary", "david", "vivian"));
put("tim", Arrays.asList("alex"));
- put("nathan", Arrays.asList("sally", "bob", "adam", "harry", "chris", "vivian", "emily", "jordan"));
+ put("nathan", Arrays.asList("sally", "bob", "adam", "harry", "chris", "vivian", "emily",
+ "jordan"));
put("adam", Arrays.asList("david", "carissa"));
put("mike", Arrays.asList("john", "bob"));
put("john", Arrays.asList("alice", "nathan", "jim", "mike", "bob"));
@@ -87,10 +101,12 @@ public static void main(String[] args) throws Exception {
if (args.length > 0) {
topoName = args[0];
}
- StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder.createRemoteTopology());
+ StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder
+ .createRemoteTopology());
try (DRPCClient drpc = DRPCClient.getConfiguredClient(conf)) {
- String[] urlsToTry = new String[]{ "foo.com/blog/1", "engineering.twitter.com/blog/5", "notaurl.com" };
+ String[] urlsToTry =
+ new String[]{ "foo.com/blog/1", "engineering.twitter.com/blog/5", "notaurl.com" };
for (String url : urlsToTry) {
System.out.println("Reach of " + url + ": " + drpc.execute("reach", url));
}
@@ -141,7 +157,8 @@ public static class PartialUniquer extends BaseBatchBolt {
Set followers = new HashSet();
@Override
- public void prepare(Map conf, TopologyContext context, BatchOutputCollector collector, Object id) {
+ public void prepare(Map conf, TopologyContext context,
+ BatchOutputCollector collector, Object id) {
this.collector = collector;
this.id = id;
}
@@ -168,7 +185,8 @@ public static class CountAggregator extends BaseBatchBolt {
int count = 0;
@Override
- public void prepare(Map conf, TopologyContext context, BatchOutputCollector collector, Object id) {
+ public void prepare(Map conf, TopologyContext context,
+ BatchOutputCollector collector, Object id) {
this.collector = collector;
this.id = id;
}
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/ResourceAwareExampleTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/ResourceAwareExampleTopology.java
index 20f951e7602..92659f5b667 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/ResourceAwareExampleTopology.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/ResourceAwareExampleTopology.java
@@ -1,20 +1,26 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
package org.apache.storm.starter;
import java.util.Iterator;
-import java.util.Map;
import java.util.Map.Entry;
+import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.storm.Config;
import org.apache.storm.StormSubmitter;
@@ -35,13 +41,13 @@ public class ResourceAwareExampleTopology {
public static void main(String[] args) throws Exception {
TopologyBuilder builder = new TopologyBuilder();
- //A topology can set resources in terms of CPU and Memory for each component
+ // A topology can set resources in terms of CPU and Memory for each component
// These can be chained (like with setting the CPU requirement)
SpoutDeclarer spout = builder.setSpout("word", new TestWordSpout(), 10).setCPULoad(20);
// Or done separately like with setting the
// onheap and offheap memory requirement
spout.setMemoryLoad(64, 16);
- //On heap memory is used to help calculate the heap of the java process for the worker
+ // On heap memory is used to help calculate the heap of the java process for the worker
// off heap memory is for things like JNI memory allocated off heap, or when using the
// ShellBolt or ShellSpout. In this case the 16 MB of off heap is just as an example
// as we are not using it.
@@ -54,7 +60,8 @@ public static void main(String[] args) throws Exception {
SharedOffHeapWithinNode notImplementedButJustAnExample =
new SharedOffHeapWithinNode(500, "not-implemented-node-level-cache");
- //If CPU or memory is not set the values stored in topology.component.resources.onheap.memory.mb,
+ // If CPU or memory is not set the values stored in
+ // topology.component.resources.onheap.memory.mb,
// topology.component.resources.offheap.memory.mb and topology.component.cpu.pcore.percent
// will be used instead
builder
@@ -72,26 +79,34 @@ public static void main(String[] args) throws Exception {
Config conf = new Config();
conf.setDebug(true);
- //Under RAS the number of workers is determined by the scheduler and the settings in the conf are ignored
- //conf.setNumWorkers(3);
+ // Under RAS the number of workers is determined by the scheduler and the settings in the
+ // conf are ignored
+ // conf.setNumWorkers(3);
- //Instead the scheduler lets you set the maximum heap size for any worker.
+ // Instead the scheduler lets you set the maximum heap size for any worker.
conf.setTopologyWorkerMaxHeapSize(1024.0);
- //The scheduler generally will try to pack executors into workers until the max heap size is met, but
+ // The scheduler generally will try to pack executors into workers until the max heap size
+ // is met, but
// this can vary depending on the specific scheduling strategy selected.
- // The reason for this is to try and balance the maximum pause time GC might take (which is larger for larger heaps)
+ // The reason for this is to try and balance the maximum pause time GC might take (which is
+ // larger for larger heaps)
// against better performance because of not needing to serialize/deserialize tuples.
- //The priority of a topology describes the importance of the topology in decreasing importance
- // starting from 0 (i.e. 0 is the highest priority and the priority importance decreases as the priority number increases).
- //Recommended range of 0-29 but no hard limit set.
- // If there are not enough resources in a cluster the priority in combination with how far over a guarantees
+ // The priority of a topology describes the importance of the topology in decreasing
+ // importance
+ // starting from 0 (i.e. 0 is the highest priority and the priority importance decreases as
+ // the priority number increases).
+ // Recommended range of 0-29 but no hard limit set.
+ // If there are not enough resources in a cluster the priority in combination with how far
+ // over a guarantees
// a user is will decide which topologies are run and which ones are not.
conf.setTopologyPriority(29);
- //set to use the default resource aware strategy when using the MultitenantResourceAwareBridgeScheduler
+ // set to use the default resource aware strategy when using the
+ // MultitenantResourceAwareBridgeScheduler
conf.setTopologyStrategy(
- "org.apache.storm.scheduler.resource.strategies.scheduling.DefaultResourceAwareStrategy");
+ "org.apache.storm.scheduler.resource.strategies.scheduling.DefaultResourceAwareStrateg"
+ + "y");
String topoName = "test";
if (args != null && args.length > 0) {
@@ -102,7 +117,7 @@ public static void main(String[] args) throws Exception {
}
public static class ExclamationBolt extends BaseRichBolt {
- //Have a crummy cache to show off shared memory accounting
+ // Have a crummy cache to show off shared memory accounting
private static final ConcurrentHashMap myCrummyCache =
new ConcurrentHashMap<>();
private static final int CACHE_SIZE = 100_000;
@@ -116,7 +131,7 @@ protected static void addToCache(String key, String value) {
myCrummyCache.putIfAbsent(key, value);
int numToRemove = myCrummyCache.size() - CACHE_SIZE;
if (numToRemove > 0) {
- //Remove something randomly...
+ // Remove something randomly...
Iterator> it = myCrummyCache.entrySet().iterator();
for (; numToRemove > 0 && it.hasNext(); numToRemove--) {
it.next();
@@ -126,7 +141,8 @@ protected static void addToCache(String key, String value) {
}
@Override
- public void prepare(Map conf, TopologyContext context, OutputCollector collector) {
+ public void prepare(Map conf, TopologyContext context,
+ OutputCollector collector) {
this.collector = collector;
}
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/RollingTopWords.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/RollingTopWords.java
index ea40e8bfaad..79ffdaf26cf 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/RollingTopWords.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/RollingTopWords.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -51,7 +57,8 @@ public static void main(String[] args) throws Exception {
*
Examples:
* ```
* # Runs in remote/cluster mode, with topology name "production-topology"
- * $ storm jar storm-starter-jar-with-dependencies.jar org.apache.storm.starter.RollingTopWords production-topology ```
+ * $ storm jar storm-starter-jar-with-dependencies.jar org.apache.storm.starter.RollingTopWords
+ * production-topology ```
*
* @param args
* First positional argument (optional) is topology name, second
@@ -69,11 +76,14 @@ protected int run(String[] args) {
String counterId = "counter";
String intermediateRankerId = "intermediateRanker";
builder.setSpout(spoutId, new TestWordSpout(), 5);
- builder.setBolt(counterId, new RollingCountBolt(9, 3), 4).fieldsGrouping(spoutId, new Fields("word"));
- builder.setBolt(intermediateRankerId, new IntermediateRankingsBolt(TOP_N), 4).fieldsGrouping(counterId,
+ builder.setBolt(counterId, new RollingCountBolt(9, 3), 4).fieldsGrouping(spoutId,
+ new Fields("word"));
+ builder.setBolt(intermediateRankerId, new IntermediateRankingsBolt(TOP_N), 4)
+ .fieldsGrouping(counterId,
new Fields("obj"));
String totalRankerId = "finalRanker";
- builder.setBolt(totalRankerId, new TotalRankingsBolt(TOP_N)).globalGrouping(intermediateRankerId);
+ builder.setBolt(totalRankerId, new TotalRankingsBolt(TOP_N))
+ .globalGrouping(intermediateRankerId);
LOG.info("Topology name: " + topologyName);
return submit(topologyName, conf, builder);
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/SingleJoinExample.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/SingleJoinExample.java
index ad3ab3d48e2..37433a59504 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/SingleJoinExample.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/SingleJoinExample.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -37,7 +43,8 @@ public static void main(String[] args) throws Exception {
TopologyBuilder builder = new TopologyBuilder();
builder.setSpout("gender", genderSpout);
builder.setSpout("age", ageSpout);
- builder.setBolt("join", new SingleJoinBolt(new Fields("gender", "age"))).fieldsGrouping("gender", new Fields("id"))
+ builder.setBolt("join", new SingleJoinBolt(new Fields("gender", "age")))
+ .fieldsGrouping("gender", new Fields("id"))
.fieldsGrouping("age", new Fields("id"));
Config conf = new Config();
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/SkewedRollingTopWords.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/SkewedRollingTopWords.java
index a983b7c3d24..ce71516abfc 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/SkewedRollingTopWords.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/SkewedRollingTopWords.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -55,7 +61,8 @@ public static void main(String[] args) throws Exception {
*
*
```
* # Runs in remote/cluster mode, with topology name "production-topology"
- * $ storm jar storm-starter-jar-with-dependencies.jar org.apache.storm.starter.SkewedRollingTopWords production-topology ```
+ * $ storm jar storm-starter-jar-with-dependencies.jar
+ * org.apache.storm.starter.SkewedRollingTopWords production-topology ```
*
* @param args
* First positional argument (optional) is topology name, second
@@ -73,12 +80,16 @@ protected int run(String[] args) {
String counterId = "counter";
String aggId = "aggregator";
builder.setSpout(spoutId, new TestWordSpout(), 5);
- builder.setBolt(counterId, new RollingCountBolt(9, 3), 4).partialKeyGrouping(spoutId, new Fields("word"));
- builder.setBolt(aggId, new RollingCountAggBolt(), 4).fieldsGrouping(counterId, new Fields("obj"));
+ builder.setBolt(counterId, new RollingCountBolt(9, 3), 4).partialKeyGrouping(spoutId,
+ new Fields("word"));
+ builder.setBolt(aggId, new RollingCountAggBolt(), 4).fieldsGrouping(counterId,
+ new Fields("obj"));
String intermediateRankerId = "intermediateRanker";
- builder.setBolt(intermediateRankerId, new IntermediateRankingsBolt(TOP_N), 4).fieldsGrouping(aggId, new Fields("obj"));
+ builder.setBolt(intermediateRankerId, new IntermediateRankingsBolt(TOP_N), 4)
+ .fieldsGrouping(aggId, new Fields("obj"));
String totalRankerId = "finalRanker";
- builder.setBolt(totalRankerId, new TotalRankingsBolt(TOP_N)).globalGrouping(intermediateRankerId);
+ builder.setBolt(totalRankerId, new TotalRankingsBolt(TOP_N))
+ .globalGrouping(intermediateRankerId);
LOG.info("Topology name: " + topologyName);
return submit(topologyName, conf, builder);
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/SlidingTupleTsTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/SlidingTupleTsTopology.java
index d179899d69b..742935cb26f 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/SlidingTupleTsTopology.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/SlidingTupleTsTopology.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -19,8 +25,8 @@
import org.apache.storm.starter.bolt.SlidingWindowSumBolt;
import org.apache.storm.starter.spout.RandomIntegerSpout;
import org.apache.storm.topology.TopologyBuilder;
-import org.apache.storm.topology.base.BaseWindowedBolt;
import org.apache.storm.topology.base.BaseWindowedBolt.Duration;
+import org.apache.storm.topology.base.BaseWindowedBolt;
/**
* Windowing based on tuple timestamp (e.g. the time when tuple is generated
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/SlidingWindowTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/SlidingWindowTopology.java
index fda309df70f..e646d17e84c 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/SlidingWindowTopology.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/SlidingWindowTopology.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -23,8 +29,8 @@
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.topology.TopologyBuilder;
-import org.apache.storm.topology.base.BaseWindowedBolt;
import org.apache.storm.topology.base.BaseWindowedBolt.Count;
+import org.apache.storm.topology.base.BaseWindowedBolt;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values;
@@ -43,9 +49,11 @@ public class SlidingWindowTopology {
public static void main(String[] args) throws Exception {
TopologyBuilder builder = new TopologyBuilder();
builder.setSpout("integer", new RandomIntegerSpout(), 1);
- builder.setBolt("slidingsum", new SlidingWindowSumBolt().withWindow(Count.of(30), Count.of(10)), 1)
+ builder.setBolt("slidingsum", new SlidingWindowSumBolt().withWindow(Count.of(30), Count
+ .of(10)), 1)
.shuffleGrouping("integer");
- builder.setBolt("tumblingavg", new TumblingWindowAvgBolt().withTumblingWindow(Count.of(3)), 1)
+ builder.setBolt("tumblingavg", new TumblingWindowAvgBolt().withTumblingWindow(Count.of(3)),
+ 1)
.shuffleGrouping("slidingsum");
builder.setBolt("printer", new PrinterBolt(), 1).shuffleGrouping("tumblingavg");
Config conf = new Config();
@@ -65,7 +73,8 @@ private static class TumblingWindowAvgBolt extends BaseWindowedBolt {
private OutputCollector collector;
@Override
- public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) {
+ public void prepare(Map topoConf, TopologyContext context,
+ OutputCollector collector) {
this.collector = collector;
}
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/StatefulTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/StatefulTopology.java
index be8d28ed848..ba5c82fca58 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/StatefulTopology.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/StatefulTopology.java
@@ -37,7 +37,8 @@
import org.slf4j.LoggerFactory;
/**
- * An example topology that demonstrates the use of {@link org.apache.storm.topology.IStatefulBolt} to manage state. To run the example,
+ * An example topology that demonstrates the use of {@link org.apache.storm.topology.IStatefulBolt}
+ * to manage state. To run the example,
*
* $ storm jar examples/storm-starter/storm-starter-topologies-*.jar storm.starter.StatefulTopology statetopology
*
@@ -92,7 +93,8 @@ private static class StatefulSumBolt extends BaseStatefulBolt topoConf, TopologyContext context, OutputCollector collector) {
+ public void prepare(Map topoConf, TopologyContext context,
+ OutputCollector collector) {
this.collector = collector;
}
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/StatefulWindowingTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/StatefulWindowingTopology.java
index eb0132e4f53..6910248961c 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/StatefulWindowingTopology.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/StatefulWindowingTopology.java
@@ -39,14 +39,16 @@
import org.slf4j.LoggerFactory;
/**
- * A simple example that demonstrates the usage of {@link org.apache.storm.topology.IStatefulWindowedBolt} to save the state of the
+ * A simple example that demonstrates the usage of {@link
+ * org.apache.storm.topology.IStatefulWindowedBolt} to save the state of the
* windowing operation to avoid re-computation in case of failures.
- *
- * The framework internally manages the window boundaries and does not invoke
+ *
+ *
The framework internally manages the window boundaries and does not invoke
* {@link org.apache.storm.topology.IWindowedBolt#execute(TupleWindow)}
* for the already evaluated windows in case of restarts during failures. The
* {@link org.apache.storm.topology.IStatefulBolt#initState(State)}
- * is invoked with the previously saved state of the bolt after prepare, before the execute() method is invoked.
+ * is invoked with the previously saved state of the bolt after prepare, before the execute() method
+ * is invoked.
*
*/
public class StatefulWindowingTopology {
@@ -56,11 +58,13 @@ public static void main(String[] args) throws Exception {
TopologyBuilder builder = new TopologyBuilder();
builder.setSpout("spout", new RandomIntegerSpout());
builder.setBolt("sumbolt", new WindowSumBolt().withWindow(new Count(5), new Count(3))
- .withMessageIdField("msgid"), 1).shuffleGrouping("spout");
+ .withMessageIdField("msgid"), 1)
+ .shuffleGrouping("spout");
builder.setBolt("printer", new PrinterBolt(), 1).shuffleGrouping("sumbolt");
Config conf = new Config();
conf.setDebug(false);
- //conf.put(Config.TOPOLOGY_STATE_PROVIDER, "org.apache.storm.redis.state.RedisKeyValueStateProvider");
+ // conf.put(Config.TOPOLOGY_STATE_PROVIDER,
+ // "org.apache.storm.redis.state.RedisKeyValueStateProvider");
String topoName = "test";
if (args != null && args.length > 0) {
@@ -77,7 +81,8 @@ private static class WindowSumBolt extends BaseStatefulWindowedBolt topoConf, TopologyContext context, OutputCollector collector) {
+ public void prepare(Map topoConf, TopologyContext context,
+ OutputCollector collector) {
this.collector = collector;
}
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/WordCountTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/WordCountTopology.java
index 71a3b42ed28..21d56dc7598 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/WordCountTopology.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/WordCountTopology.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -44,7 +50,8 @@ protected int run(String[] args) throws Exception {
builder.setSpout("spout", new RandomSentenceSpout(), 5);
builder.setBolt("split", new SplitSentence(), 8).shuffleGrouping("spout");
- builder.setBolt("count", new WordCountBolt(), 12).fieldsGrouping("split", new Fields("word"));
+ builder.setBolt("count", new WordCountBolt(), 12).fieldsGrouping("split",
+ new Fields("word"));
conf.setDebug(true);
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/WordCountTopologyNode.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/WordCountTopologyNode.java
index 6df76789311..f738e80b15f 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/WordCountTopologyNode.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/WordCountTopologyNode.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/AbstractRankerBolt.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/AbstractRankerBolt.java
index b04196ca335..42c7cd5db5e 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/AbstractRankerBolt.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/AbstractRankerBolt.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -26,10 +32,13 @@
import org.apache.storm.utils.TupleUtils;
/**
- * This abstract bolt provides the basic behavior of bolts that rank objects according to their count.
+ * This abstract bolt provides the basic behavior of bolts that rank objects according to their
+ * count.
*
- * It uses a template method design pattern for {@link AbstractRankerBolt#execute(Tuple, BasicOutputCollector)} to allow
- * actual bolt implementations to specify how incoming tuples are processed, i.e. how the objects embedded within those
+ * It uses a template method design pattern for {@link AbstractRankerBolt#execute(Tuple,
+ * BasicOutputCollector)} to allow
+ * actual bolt implementations to specify how incoming tuples are processed, i.e. how the objects
+ * embedded within those
* tuples are retrieved and counted.
*/
public abstract class AbstractRankerBolt extends BaseBasicBolt {
@@ -56,7 +65,8 @@ public AbstractRankerBolt(int topN, int emitFrequencyInSeconds) {
}
if (emitFrequencyInSeconds < 1) {
throw new IllegalArgumentException(
- "The emit frequency must be >= 1 seconds (you requested " + emitFrequencyInSeconds + " seconds)");
+ "The emit frequency must be >= 1 seconds (you requested " + emitFrequencyInSeconds
+ + " seconds)");
}
count = topN;
this.emitFrequencyInSeconds = emitFrequencyInSeconds;
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/IntermediateRankingsBolt.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/IntermediateRankingsBolt.java
index a6a8b49f9b0..076a1b40300 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/IntermediateRankingsBolt.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/IntermediateRankingsBolt.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -20,7 +26,8 @@
/**
* This bolt ranks incoming objects by their count.
*
- * It assumes the input tuples to adhere to the following format: (object, object_count, additionalField1,
+ * It assumes the input tuples to adhere to the following format: (object, object_count,
+ * additionalField1,
* additionalField2, ..., additionalFieldN).
*/
public final class IntermediateRankingsBolt extends AbstractRankerBolt {
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/PrinterBolt.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/PrinterBolt.java
index 8364222a8b7..06167986b23 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/PrinterBolt.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/PrinterBolt.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -17,7 +23,6 @@
import org.apache.storm.topology.base.BaseBasicBolt;
import org.apache.storm.tuple.Tuple;
-
public class PrinterBolt extends BaseBasicBolt {
@Override
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/RollingCountAggBolt.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/RollingCountAggBolt.java
index 9db1dd62963..c62c98d49a7 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/RollingCountAggBolt.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/RollingCountAggBolt.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -29,13 +35,13 @@
public class RollingCountAggBolt extends BaseRichBolt {
private static final long serialVersionUID = 5537727428628598519L;
private static final Logger LOG = Logger.getLogger(RollingCountAggBolt.class);
- //Mapping of key->upstreamBolt->count
+ // Mapping of key->upstreamBolt->count
private Map> counts = new HashMap>();
private OutputCollector collector;
-
@Override
- public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) {
+ public void prepare(Map topoConf, TopologyContext context,
+ OutputCollector collector) {
this.collector = collector;
}
@@ -49,9 +55,9 @@ public void execute(Tuple tuple) {
subCounts = new HashMap();
counts.put(obj, subCounts);
}
- //Update the current count for this object
+ // Update the current count for this object
subCounts.put(source, count);
- //Output the sum of all the known counts so for this key
+ // Output the sum of all the known counts so for this key
long sum = 0;
for (Long val : subCounts.values()) {
sum += val;
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/RollingCountBolt.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/RollingCountBolt.java
index 0e5cef57d9f..cb1646e0f75 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/RollingCountBolt.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/RollingCountBolt.java
@@ -1,20 +1,26 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
package org.apache.storm.starter.bolt;
import java.util.HashMap;
-import java.util.Map;
import java.util.Map.Entry;
+import java.util.Map;
import org.apache.log4j.Logger;
import org.apache.storm.Config;
import org.apache.storm.starter.tools.NthLastModifiedTimeTracker;
@@ -31,20 +37,31 @@
/**
* This bolt performs rolling counts of incoming objects, i.e. sliding window based counting.
*
- * The bolt is configured by two parameters, the length of the sliding window in seconds (which influences the output
- * data of the bolt, i.e. how it will count objects) and the emit frequency in seconds (which influences how often the
- * bolt will output the latest window counts). For instance, if the window length is set to an equivalent of five
- * minutes and the emit frequency to one minute, then the bolt will output the latest five-minute sliding window every
+ * The bolt is configured by two parameters, the length of the sliding window in seconds (which
+ * influences the output
+ * data of the bolt, i.e. how it will count objects) and the emit frequency in seconds (which
+ * influences how often the
+ * bolt will output the latest window counts). For instance, if the window length is set to an
+ * equivalent of five
+ * minutes and the emit frequency to one minute, then the bolt will output the latest five-minute
+ * sliding window every
* minute.
*
- * The bolt emits a rolling count tuple per object, consisting of the object itself, its latest rolling count, and the
- * actual duration of the sliding window. The latter is included in case the expected sliding window length (as
- * configured by the user) is different from the actual length, e.g. due to high system load. Note that the actual
- * window length is tracked and calculated for the window, and not individually for each object within a window.
+ * The bolt emits a rolling count tuple per object, consisting of the object itself, its latest
+ * rolling count, and the
+ * actual duration of the sliding window. The latter is included in case the expected sliding window
+ * length (as
+ * configured by the user) is different from the actual length, e.g. due to high system load. Note
+ * that the actual
+ * window length is tracked and calculated for the window, and not individually for each object
+ * within a window.
*
- * Note: During the startup phase you will usually observe that the bolt warns you about the actual sliding window
- * length being smaller than the expected length. This behavior is expected and is caused by the way the sliding window
- * counts are initially "loaded up". You can safely ignore this warning during startup (e.g. you will see this warning
+ * Note: During the startup phase you will usually observe that the bolt warns you about the actual
+ * sliding window
+ * length being smaller than the expected length. This behavior is expected and is caused by the way
+ * the sliding window
+ * counts are initially "loaded up". You can safely ignore this warning during startup (e.g. you
+ * will see this warning
* during the first ~ five minutes of startup time if the window length is set to five minutes).
*/
public class RollingCountBolt extends BaseRichBolt {
@@ -53,7 +70,8 @@ public class RollingCountBolt extends BaseRichBolt {
private static final Logger LOG = Logger.getLogger(RollingCountBolt.class);
private static final int NUM_WINDOW_CHUNKS = 5;
private static final int DEFAULT_SLIDING_WINDOW_IN_SECONDS = NUM_WINDOW_CHUNKS * 60;
- private static final int DEFAULT_EMIT_FREQUENCY_IN_SECONDS = DEFAULT_SLIDING_WINDOW_IN_SECONDS / NUM_WINDOW_CHUNKS;
+ private static final int DEFAULT_EMIT_FREQUENCY_IN_SECONDS =
+ DEFAULT_SLIDING_WINDOW_IN_SECONDS / NUM_WINDOW_CHUNKS;
private static final String WINDOW_LENGTH_WARNING_TEMPLATE =
"Actual window length is %d seconds when it should be %d seconds"
+ " (you can safely ignore this warning during the startup phase)";
@@ -71,18 +89,22 @@ public RollingCountBolt() {
public RollingCountBolt(int windowLengthInSeconds, int emitFrequencyInSeconds) {
this.windowLengthInSeconds = windowLengthInSeconds;
this.emitFrequencyInSeconds = emitFrequencyInSeconds;
- counter = new SlidingWindowCounter(deriveNumWindowChunksFrom(this.windowLengthInSeconds,
+ counter =
+ new SlidingWindowCounter(deriveNumWindowChunksFrom(this.windowLengthInSeconds,
this.emitFrequencyInSeconds));
}
- private int deriveNumWindowChunksFrom(int windowLengthInSeconds, int windowUpdateFrequencyInSeconds) {
+ private int deriveNumWindowChunksFrom(int windowLengthInSeconds,
+ int windowUpdateFrequencyInSeconds) {
return windowLengthInSeconds / windowUpdateFrequencyInSeconds;
}
@Override
- public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) {
+ public void prepare(Map topoConf, TopologyContext context,
+ OutputCollector collector) {
this.collector = collector;
- lastModifiedTracker = new NthLastModifiedTimeTracker(deriveNumWindowChunksFrom(this.windowLengthInSeconds,
+ lastModifiedTracker =
+ new NthLastModifiedTimeTracker(deriveNumWindowChunksFrom(this.windowLengthInSeconds,
this.emitFrequencyInSeconds));
}
@@ -101,7 +123,8 @@ private void emitCurrentWindowCounts() {
int actualWindowLengthInSeconds = lastModifiedTracker.secondsSinceOldestModification();
lastModifiedTracker.markAsModified();
if (actualWindowLengthInSeconds != windowLengthInSeconds) {
- LOG.warn(String.format(WINDOW_LENGTH_WARNING_TEMPLATE, actualWindowLengthInSeconds, windowLengthInSeconds));
+ LOG.warn(String.format(WINDOW_LENGTH_WARNING_TEMPLATE, actualWindowLengthInSeconds,
+ windowLengthInSeconds));
}
emit(counts, actualWindowLengthInSeconds);
}
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/SingleJoinBolt.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/SingleJoinBolt.java
index 5f9b22525ad..0449a13d326 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/SingleJoinBolt.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/SingleJoinBolt.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -45,15 +51,18 @@ public SingleJoinBolt(Fields outFields) {
}
@Override
- public void prepare(Map conf, TopologyContext context, OutputCollector collector) {
+ public void prepare(Map conf, TopologyContext context,
+ OutputCollector collector) {
fieldLocations = new HashMap();
this.collector = collector;
int timeout = ((Number) conf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS)).intValue();
- pending = new TimeCacheMap, Map>(timeout, new ExpireCallback());
+ pending = new TimeCacheMap, Map>(timeout,
+ new ExpireCallback());
numSources = context.getThisSources().size();
Set idFields = null;
for (GlobalStreamId source : context.getThisSources().keySet()) {
- Fields fields = context.getComponentOutputFields(source.get_componentId(), source.get_streamId());
+ Fields fields = context.getComponentOutputFields(source.get_componentId(), source
+ .get_streamId());
Set setFields = new HashSet(fields.toList());
if (idFields == null) {
idFields = setFields;
@@ -79,7 +88,8 @@ public void prepare(Map conf, TopologyContext context, OutputCol
@Override
public void execute(Tuple tuple) {
List id = tuple.select(idFields);
- GlobalStreamId streamId = new GlobalStreamId(tuple.getSourceComponent(), tuple.getSourceStreamId());
+ GlobalStreamId streamId = new GlobalStreamId(tuple.getSourceComponent(), tuple
+ .getSourceStreamId());
if (!pending.containsKey(id)) {
pending.put(id, new HashMap());
}
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/SlidingWindowSumBolt.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/SlidingWindowSumBolt.java
index 27021817350..40fe8633a26 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/SlidingWindowSumBolt.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/SlidingWindowSumBolt.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -35,7 +41,8 @@ public class SlidingWindowSumBolt extends BaseWindowedBolt {
private OutputCollector collector;
@Override
- public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) {
+ public void prepare(Map topoConf, TopologyContext context,
+ OutputCollector collector) {
this.collector = collector;
}
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/TotalRankingsBolt.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/TotalRankingsBolt.java
index e185a9ec85d..2bd680d926d 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/TotalRankingsBolt.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/TotalRankingsBolt.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -19,8 +25,10 @@
/**
* This bolt merges incoming {@link Rankings}.
*
- * It can be used to merge intermediate rankings generated by {@link IntermediateRankingsBolt} into a final,
- * consolidated ranking. To do so, configure this bolt with a globalGrouping on {@link IntermediateRankingsBolt}.
+ * It can be used to merge intermediate rankings generated by {@link IntermediateRankingsBolt} into
+ * a final,
+ * consolidated ranking. To do so, configure this bolt with a globalGrouping on {@link
+ * IntermediateRankingsBolt}.
*/
public final class TotalRankingsBolt extends AbstractRankerBolt {
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/WordCountBolt.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/WordCountBolt.java
index 870d5bd1931..a8f506dbe70 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/WordCountBolt.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/WordCountBolt.java
@@ -1,12 +1,17 @@
/*
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/spout/RandomIntegerSpout.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/spout/RandomIntegerSpout.java
index e1a9ee51404..7ceb5031275 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/spout/RandomIntegerSpout.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/spout/RandomIntegerSpout.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -40,7 +46,8 @@ public void declareOutputFields(OutputFieldsDeclarer declarer) {
}
@Override
- public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
+ public void open(Map conf, TopologyContext context,
+ SpoutOutputCollector collector) {
this.collector = collector;
this.rand = new Random();
}
@@ -48,7 +55,8 @@ public void open(Map conf, TopologyContext context, SpoutOutputC
@Override
public void nextTuple() {
Utils.sleep(100);
- collector.emit(new Values(rand.nextInt(1000), System.currentTimeMillis() - (24 * 60 * 60 * 1000), ++msgId), msgId);
+ collector.emit(new Values(rand.nextInt(1000), System
+ .currentTimeMillis() - (24 * 60 * 60 * 1000), ++msgId), msgId);
}
@Override
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/spout/RandomNumberGeneratorSpout.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/spout/RandomNumberGeneratorSpout.java
index e6c48048c33..e32035cbb90 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/spout/RandomNumberGeneratorSpout.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/spout/RandomNumberGeneratorSpout.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
+ *
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/spout/RandomSentenceSpout.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/spout/RandomSentenceSpout.java
index 92af62655e7..2be8a8d55be 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/spout/RandomSentenceSpout.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/spout/RandomSentenceSpout.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -32,9 +38,9 @@ public class RandomSentenceSpout extends BaseRichSpout {
SpoutOutputCollector collector;
Random rand;
-
@Override
- public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
+ public void open(Map conf, TopologyContext context,
+ SpoutOutputCollector collector) {
this.collector = collector;
rand = new Random();
}
@@ -43,7 +49,8 @@ public void open(Map conf, TopologyContext context, SpoutOutputC
public void nextTuple() {
Utils.sleep(100);
String[] sentences = new String[]{
- sentence("the cow jumped over the moon"), sentence("an apple a day keeps the doctor away"),
+ sentence("the cow jumped over the moon"), sentence("an apple a day keeps the doctor "
+ + "away"),
sentence("four score and seven years ago"), sentence("snow white and the seven dwarfs"), sentence("i am at two with nature")
};
final String sentence = sentences[rand.nextInt(sentences.length)];
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/AggregateExample.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/AggregateExample.java
index 338a894031e..a8a89e9b0e1 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/AggregateExample.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/AggregateExample.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -30,8 +36,10 @@ public class AggregateExample {
public static void main(String[] args) throws Exception {
StreamBuilder builder = new StreamBuilder();
/*
- * Computes average of the stream of numbers emitted by the spout. Internally the per-partition
- * sum and counts are accumulated and emitted to a downstream task where the partially accumulated
+ * Computes average of the stream of numbers emitted by the spout. Internally the
+ * per-partition
+ * sum and counts are accumulated and emitted to a downstream task where the partially
+ * accumulated
* results are merged and the final result is emitted.
*/
builder.newStream(new RandomIntegerSpout(), new ValueMapper(0), 2)
@@ -61,7 +69,8 @@ public Pair apply(Pair sumAndCount, Integer
}
@Override
- public Pair merge(Pair sumAndCount1, Pair sumAndCount2) {
+ public Pair merge(Pair sumAndCount1, Pair sumAndCount2) {
System.out.println("Merge " + sumAndCount1 + " and " + sumAndCount2);
return Pair.of(
sumAndCount1.value1 + sumAndCount2.value1,
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/BranchExample.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/BranchExample.java
index fc7e74a5a7f..7187af65742 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/BranchExample.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/BranchExample.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/GroupByKeyAndWindowExample.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/GroupByKeyAndWindowExample.java
index 02617a2cf57..23ecd68aebb 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/GroupByKeyAndWindowExample.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/GroupByKeyAndWindowExample.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -47,7 +53,8 @@ public static void main(String[] args) throws Exception {
* together and the corresponding values will be merged.
*
* The result is a PairStream> with
- * 'stock symbol' as the key and 'stock prices' for that symbol within the window as the value.
+ * 'stock symbol' as the key and 'stock prices' for that symbol within the window as
+ * the value.
*/
.groupByKeyAndWindow(SlidingWindows.of(Count.of(6), Count.of(3)))
.print();
@@ -59,9 +66,11 @@ public static void main(String[] args) throws Exception {
* together and their values will be reduced using the given reduce function.
*
* Here the result is a PairStream with
- * 'stock symbol' as the key and the maximum price for that symbol within the window as the value.
+ * 'stock symbol' as the key and the maximum price for that symbol within the window
+ * as the value.
*/
- .reduceByKeyAndWindow((x, y) -> x > y ? x : y, SlidingWindows.of(Count.of(6), Count.of(3)))
+ .reduceByKeyAndWindow((x, y) -> x > y ? x : y, SlidingWindows.of(Count.of(6), Count
+ .of(3)))
.print();
Config config = new Config();
@@ -75,15 +84,19 @@ public static void main(String[] args) throws Exception {
private static class StockQuotes extends BaseRichSpout {
private final List> values = Arrays.asList(
- Arrays.asList(new Values("AAPL", 100.0), new Values("GOOG", 780.0), new Values("FB", 125.0)),
- Arrays.asList(new Values("AAPL", 105.0), new Values("GOOG", 790.0), new Values("FB", 130.0)),
- Arrays.asList(new Values("AAPL", 102.0), new Values("GOOG", 788.0), new Values("FB", 128.0))
+ Arrays.asList(new Values("AAPL", 100.0), new Values("GOOG", 780.0), new Values("FB",
+ 125.0)),
+ Arrays.asList(new Values("AAPL", 105.0), new Values("GOOG", 790.0), new Values("FB",
+ 130.0)),
+ Arrays.asList(new Values("AAPL", 102.0), new Values("GOOG", 788.0), new Values("FB",
+ 128.0))
);
private SpoutOutputCollector collector;
private int index = 0;
@Override
- public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
+ public void open(Map conf, TopologyContext context,
+ SpoutOutputCollector collector) {
this.collector = collector;
}
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/JoinExample.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/JoinExample.java
index 49085c5102e..23db7b778cd 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/JoinExample.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/JoinExample.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -77,7 +83,8 @@ private static class NumberSpout extends BaseRichSpout {
}
@Override
- public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
+ public void open(Map conf, TopologyContext context,
+ SpoutOutputCollector collector) {
this.collector = collector;
}
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/StateQueryExample.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/StateQueryExample.java
index b72b67614e0..7bf5bc8828d 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/StateQueryExample.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/StateQueryExample.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -30,7 +36,7 @@
import org.apache.storm.utils.Utils;
/**
- * An example that uses {@link Stream#stateQuery(StreamState)} to query the state
+ * An example that uses {@link Stream#stateQuery(StreamState)} to query the state.
*
*
You should start a local redis instance before running the 'storm jar' command. By default
* the connection will be attempted at localhost:6379. The default
@@ -46,15 +52,19 @@
public class StateQueryExample {
public static void main(String[] args) throws Exception {
StreamBuilder builder = new StreamBuilder();
- StreamState ss = builder.newStream(new TestWordSpout(), new ValueMapper(0), 2)
+ StreamState ss = builder.newStream(new TestWordSpout(),
+ new ValueMapper(0), 2)
/*
- * Transform the stream of words to a stream of (word, 1) pairs
+ * Transform the stream of words to a stream of (word,
+ * 1) pairs
*/
.mapToPair(w -> Pair.of(w, 1))
/*
- * Update the count in the state. Here the first argument 0L is the initial value for the
+ * Update the count in the state. Here the first
+ * argument 0L is the initial value for the
* count and
- * the second argument is a function that increments the count for each value received.
+ * the second argument is a function that increments
+ * the count for each value received.
*/
.updateStateByKey(0L, (count, val) -> count + 1);
@@ -72,7 +82,8 @@ public static void main(String[] args) throws Exception {
Config config = new Config();
// use redis based state store for persistence
- config.put(Config.TOPOLOGY_STATE_PROVIDER, "org.apache.storm.redis.state.RedisKeyValueStateProvider");
+ config.put(Config.TOPOLOGY_STATE_PROVIDER,
+ "org.apache.storm.redis.state.RedisKeyValueStateProvider");
String topoName = "test";
if (args.length > 0) {
topoName = args[0];
@@ -86,7 +97,8 @@ private static class QuerySpout extends BaseRichSpout {
private SpoutOutputCollector collector;
@Override
- public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
+ public void open(Map conf, TopologyContext context,
+ SpoutOutputCollector collector) {
this.collector = collector;
}
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/StatefulWordCount.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/StatefulWordCount.java
index 39534b38066..8e1df052304 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/StatefulWordCount.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/StatefulWordCount.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -55,7 +61,8 @@ public static void main(String[] args) throws Exception {
/*
* update the word counts in the state.
* Here the first argument 0L is the initial value for the state
- * and the second argument is a function that adds the count to the current value in the state.
+ * and the second argument is a function that adds the count to the current value in
+ * the state.
*/
.updateStateByKey(0L, (state, count) -> state + count)
/*
@@ -66,7 +73,8 @@ public static void main(String[] args) throws Exception {
Config config = new Config();
// use redis based state store for persistence
- config.put(Config.TOPOLOGY_STATE_PROVIDER, "org.apache.storm.redis.state.RedisKeyValueStateProvider");
+ config.put(Config.TOPOLOGY_STATE_PROVIDER,
+ "org.apache.storm.redis.state.RedisKeyValueStateProvider");
String topoName = "test";
if (args.length > 0) {
topoName = args[0];
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/TypedTupleExample.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/TypedTupleExample.java
index f3e8a5ac5fe..4c4c0674091 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/TypedTupleExample.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/TypedTupleExample.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -25,19 +31,24 @@
import org.apache.storm.topology.base.BaseWindowedBolt.Count;
/**
- * An example that illustrates the usage of typed tuples (TupleN<..>) and {@link TupleValueMappers}.
+ * An example that illustrates the usage of typed tuples (TupleN<..>) and {@link
+ * TupleValueMappers}.
*/
public class TypedTupleExample {
/**
- * The spout emits sequences of (Integer, Long, Long). TupleValueMapper can be used to extract fields
- * from the values and produce a stream of typed tuple (Tuple3<Integer, Long, Long> in this case.
+ * The spout emits sequences of (Integer, Long, Long). TupleValueMapper can be used to extract
+ * fields
+ * from the values and produce a stream of typed tuple (Tuple3<Integer, Long, Long> in
+ * this case.
*/
public static void main(String[] args) throws Exception {
StreamBuilder builder = new StreamBuilder();
- Stream> stream = builder.newStream(new RandomIntegerSpout(), TupleValueMappers.of(0, 1, 2));
+ Stream> stream = builder.newStream(new RandomIntegerSpout(),
+ TupleValueMappers.of(0, 1, 2));
- PairStream pairs = stream.mapToPair(t -> Pair.of(t.value2 / 10000, t.value1));
+ PairStream pairs = stream.mapToPair(t -> Pair.of(t.value2 / 10000,
+ t.value1));
pairs.window(TumblingWindows.of(Count.of(10))).groupByKey().print();
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/WindowedWordCount.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/WindowedWordCount.java
index ef8edd4a6fb..e70f326f669 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/WindowedWordCount.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/WindowedWordCount.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/WordCountToBolt.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/WordCountToBolt.java
index 997e642d4f8..cb77ccd0322 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/WordCountToBolt.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/WordCountToBolt.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -74,7 +80,8 @@ private static class WordCountStoreMapper implements RedisStoreMapper {
private final String hashKey = "wordCount";
WordCountStoreMapper() {
- description = new RedisDataTypeDescription(RedisDataTypeDescription.RedisDataType.HASH, hashKey);
+ description = new RedisDataTypeDescription(RedisDataTypeDescription.RedisDataType.HASH,
+ hashKey);
}
@Override
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/NthLastModifiedTimeTracker.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/NthLastModifiedTimeTracker.java
index 67260a3b580..d9ccaacf118 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/NthLastModifiedTimeTracker.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/NthLastModifiedTimeTracker.java
@@ -1,19 +1,24 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
+ *
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
package org.apache.storm.starter.tools;
import java.util.concurrent.ArrayBlockingQueue;
-
import org.apache.storm.utils.Time;
/**
@@ -21,7 +26,8 @@
*
* For example, create a 5-slot tracker to track the five most recent time-since-last-modify.
*
- * You must manually "mark" that the "something" that you want to track -- in terms of modification times -- has just
+ * You must manually "mark" that the "something" that you want to track -- in terms of modification
+ * times -- has just
* been modified.
*/
public class NthLastModifiedTimeTracker {
@@ -33,7 +39,8 @@ public class NthLastModifiedTimeTracker {
public NthLastModifiedTimeTracker(int numTimesToTrack) {
if (numTimesToTrack < 1) {
throw new IllegalArgumentException(
- "numTimesToTrack must be greater than zero (you requested " + numTimesToTrack + ")");
+ "numTimesToTrack must be greater than zero (you requested " + numTimesToTrack
+ + ")");
}
lastModifiedTimesMillis = new ArrayBlockingQueue<>(numTimesToTrack);
initLastModifiedTimesMillis(numTimesToTrack);
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/Rankable.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/Rankable.java
index ea9e6d66d1f..7c81cd66c0b 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/Rankable.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/Rankable.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/RankableObjectWithFields.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/RankableObjectWithFields.java
index fea589691c9..8fcb18ce427 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/RankableObjectWithFields.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/RankableObjectWithFields.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -21,7 +27,8 @@
/**
* This class wraps an objects and its associated count, including any additional data fields.
*
- * This class can be used, for instance, to track the number of occurrences of an object in a Storm topology.
+ * This class can be used, for instance, to track the number of occurrences of an object in a Storm
+ * topology.
*/
public class RankableObjectWithFields implements Rankable, Serializable {
@@ -48,9 +55,12 @@ public RankableObjectWithFields(Object obj, long count, Object... otherFields) {
/**
* Construct a new instance based on the provided {@link Tuple}.
*
- * This method expects the object to be ranked in the first field (index 0) of the provided tuple, and the number of
- * occurrences of the object (its count) in the second field (index 1). Any further fields in the tuple will be
- * extracted and tracked, too. These fields can be accessed via {@link RankableObjectWithFields#getFields()}.
+ * This method expects the object to be ranked in the first field (index 0) of the provided
+ * tuple, and the number of
+ * occurrences of the object (its count) in the second field (index 1). Any further fields in
+ * the tuple will be
+ * extracted and tracked, too. These fields can be accessed via {@link
+ * RankableObjectWithFields#getFields()}.
*
* @param tuple
*
@@ -75,7 +85,9 @@ public long getCount() {
/**
* Get fields.
- * @return an immutable list of any additional data fields of the object (may be empty but will never be null)
+ *
+ * @return an immutable list of any additional data fields of the object (may be empty but will
+ * never be null)
*/
public List getFields() {
return fields;
@@ -130,8 +142,10 @@ public String toString() {
}
/**
- * Note: We do not defensively copy the wrapped object and any accompanying fields. We do guarantee, however,
- * do return a defensive (shallow) copy of the List object that is wrapping any accompanying fields.
+ * Note: We do not defensively copy the wrapped object and any accompanying fields. We do
+ * guarantee, however,
+ * do return a defensive (shallow) copy of the List object that is wrapping any accompanying
+ * fields.
*/
@Override
public Rankable copy() {
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/Rankings.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/Rankings.java
index 89bba59da90..5503cd1a577 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/Rankings.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/Rankings.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -47,6 +53,7 @@ public Rankings(Rankings other) {
/**
* Get max size.
+ *
* @return the maximum possible number (size) of ranked objects this instance can hold
*/
public int maxSize() {
@@ -55,6 +62,7 @@ public int maxSize() {
/**
* Get size.
+ *
* @return the number (size) of ranked objects this instance is currently holding
*/
public int size() {
@@ -62,9 +70,12 @@ public int size() {
}
/**
- * The returned defensive copy is only "somewhat" defensive. We do, for instance, return a defensive copy of the
- * enclosing List instance, and we do try to defensively copy any contained Rankable objects, too. However, the
- * contract of {@link org.apache.storm.starter.tools.Rankable#copy()} does not guarantee that any Object's embedded within
+ * The returned defensive copy is only "somewhat" defensive. We do, for instance, return a
+ * defensive copy of the
+ * enclosing List instance, and we do try to defensively copy any contained Rankable objects,
+ * too. However, the
+ * contract of {@link org.apache.storm.starter.tools.Rankable#copy()} does not guarantee that
+ * any Object's embedded within
* a Rankable will be defensively copied, too.
*
* @return a somewhat defensive copy of ranked items
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/SlidingWindowCounter.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/SlidingWindowCounter.java
index 136d586abd4..210ed8ae392 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/SlidingWindowCounter.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/SlidingWindowCounter.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -15,22 +21,29 @@
import java.io.Serializable;
import java.util.Map;
-
/**
* This class counts objects in a sliding window fashion.
*
- * It is designed 1) to give multiple "producer" threads write access to the counter, i.e. being able to increment
- * counts of objects, and 2) to give a single "consumer" thread (e.g. {@link org.apache.storm.starter.bolt.RollingCountBolt})
- * read access to the counter. Whenever the consumer thread performs a read operation, this class will advance the head slot
- * of the sliding window counter. This means that the consumer thread indirectly controls where writes of the producer threads
+ * It is designed 1) to give multiple "producer" threads write access to the counter, i.e. being
+ * able to increment
+ * counts of objects, and 2) to give a single "consumer" thread (e.g. {@link
+ * org.apache.storm.starter.bolt.RollingCountBolt})
+ * read access to the counter. Whenever the consumer thread performs a read operation, this class
+ * will advance the head slot
+ * of the sliding window counter. This means that the consumer thread indirectly controls where
+ * writes of the producer threads
* will go to. Also, by itself this class will not advance the head slot.
*
- * A note for analyzing data based on a sliding window count: During the initial windowLengthInSlots
- * iterations, this sliding window counter will always return object counts that are equal or greater than in the
- * previous iteration. This is the effect of the counter "loading up" at the very start of its existence. Conceptually,
+ * A note for analyzing data based on a sliding window count: During the initial
+ * windowLengthInSlots
+ * iterations, this sliding window counter will always return object counts that are equal or
+ * greater than in the
+ * previous iteration. This is the effect of the counter "loading up" at the very start of its
+ * existence. Conceptually,
* this is the desired behavior.
*
- * To give an example, using a counter with 5 slots which for the sake of this example represent 1 minute of time each:
+ * To give an example, using a counter with 5 slots which for the sake of this example represent 1
+ * minute of time each:
*
*
* {@code
@@ -47,16 +60,24 @@
* }
*
*
- * As you can see in this example, for the first windowLengthInSlots (here: the first five minutes) the
- * counter will always return counts equal or greater than in the previous iteration (1, 2, 3, 4, 4). This initial load
- * effect needs to be accounted for whenever you want to perform analyses such as trending topics; otherwise your
- * analysis algorithm might falsely identify the object to be trending as the counter seems to observe continuously
- * increasing counts. Also, note that during the initial load phase every object will exhibit increasing
+ * As you can see in this example, for the first windowLengthInSlots (here: the first
+ * five minutes) the
+ * counter will always return counts equal or greater than in the previous iteration (1, 2, 3, 4,
+ * 4). This initial load
+ * effect needs to be accounted for whenever you want to perform analyses such as trending topics;
+ * otherwise your
+ * analysis algorithm might falsely identify the object to be trending as the counter seems to
+ * observe continuously
+ * increasing counts. Also, note that during the initial load phase every object will
+ * exhibit increasing
* counts.
*
- * On a high-level, the counter exhibits the following behavior: If you asked the example counter after two minutes,
- * "how often did you count the object during the past five minutes?", then it should reply "I have counted it 2 times
- * in the past five minutes", implying that it can only account for the last two of those five minutes because the
+ * On a high-level, the counter exhibits the following behavior: If you asked the example counter
+ * after two minutes,
+ * "how often did you count the object during the past five minutes?", then it should reply "I have
+ * counted it 2 times
+ * in the past five minutes", implying that it can only account for the last two of those five
+ * minutes because the
* counter was not running before that time.
*
* @param The type of those objects we want to count.
@@ -73,7 +94,8 @@ public final class SlidingWindowCounter implements Serializable {
public SlidingWindowCounter(int windowLengthInSlots) {
if (windowLengthInSlots < 2) {
throw new IllegalArgumentException(
- "Window length in slots must be at least two (you requested " + windowLengthInSlots + ")");
+ "Window length in slots must be at least two (you requested " + windowLengthInSlots
+ + ")");
}
this.windowLengthInSlots = windowLengthInSlots;
this.objCounter = new SlotBasedCounter(this.windowLengthInSlots);
@@ -89,8 +111,10 @@ public void incrementCount(T obj) {
/**
* Return the current (total) counts of all tracked objects, then advance the window.
*
- * Whenever this method is called, we consider the counts of the current sliding window to be available to and
- * successfully processed "upstream" (i.e. by the caller). Knowing this we will start counting any subsequent
+ * Whenever this method is called, we consider the counts of the current sliding window to be
+ * available to and
+ * successfully processed "upstream" (i.e. by the caller). Knowing this we will start counting
+ * any subsequent
* objects within the next "chunk" of the sliding window.
*
* @return The current (total) counts of all tracked objects.
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/SlotBasedCounter.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/SlotBasedCounter.java
index 6f48c4784ea..f60dd589b15 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/SlotBasedCounter.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/SlotBasedCounter.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -20,7 +26,8 @@
/**
* This class provides per-slot counts of the occurrences of objects.
*
- * It can be used, for instance, as a building block for implementing sliding window counting of objects.
+ * It can be used, for instance, as a building block for implementing sliding window counting of
+ * objects.
*
* @param The type of those objects we want to count.
*/
@@ -33,7 +40,8 @@ public final class SlotBasedCounter implements Serializable {
public SlotBasedCounter(int numSlots) {
if (numSlots <= 0) {
- throw new IllegalArgumentException("Number of slots must be greater than zero (you requested " + numSlots + ")");
+ throw new IllegalArgumentException("Number of slots must be greater than zero (you "
+ + "requested " + numSlots + ")");
}
this.numSlots = numSlots;
}
@@ -95,7 +103,8 @@ private boolean shouldBeRemovedFromCounter(T obj) {
* Remove any object from the counter whose total count is zero (to free up memory).
*/
public void wipeZeros() {
- for (Iterator> it = objToCounts.entrySet().iterator(); it.hasNext(); ) {
+ for (Iterator> it = objToCounts.entrySet().iterator(); it
+ .hasNext(); ) {
Map.Entry entry = it.next();
if (shouldBeRemovedFromCounter(entry.getKey())) {
it.remove();
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/DebugMemoryMapState.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/DebugMemoryMapState.java
index 3a7aeb0e53a..842b08b1464 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/DebugMemoryMapState.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/DebugMemoryMapState.java
@@ -45,7 +45,8 @@ public List multiUpdate(List> keys, List updaters)
print(keys, updaters);
if ((updateCount++ % 5) == 0) {
LOG.error("Throwing FailedException");
- throw new FailedException("Enforced State Update Fail. On retrial should replay the exact same batch.");
+ throw new FailedException("Enforced State Update Fail. On retrial should replay the "
+ + "exact same batch.");
}
return super.multiUpdate(keys, updaters);
}
@@ -54,7 +55,8 @@ private void print(List> keys, List updaters) {
for (int i = 0; i < keys.size(); i++) {
ValueUpdater valueUpdater = updaters.get(i);
Object arg = ((CombinerValueUpdater) valueUpdater).getArg();
- LOG.info("updateCount = {}, keys = {} => updaterArgs = {}", updateCount, keys.get(i), arg);
+ LOG.info("updateCount = {}, keys = {} => updaterArgs = {}", updateCount, keys.get(i),
+ arg);
}
}
@@ -66,7 +68,8 @@ public Factory() {
}
@Override
- public State makeState(Map conf, IMetricsContext metrics, int partitionIndex, int numPartitions) {
+ public State makeState(Map conf, IMetricsContext metrics,
+ int partitionIndex, int numPartitions) {
return new DebugMemoryMapState(id + partitionIndex);
}
}
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentMapExample.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentMapExample.java
index 067eeff3d5c..9ec4d2f6a21 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentMapExample.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentMapExample.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -36,7 +42,8 @@
import org.apache.storm.utils.DRPCClient;
/**
- * A simple example that demonstrates the usage of {@link org.apache.storm.trident.Stream#map(MapFunction)} and
+ * A simple example that demonstrates the usage of {@link
+ * org.apache.storm.trident.Stream#map(MapFunction)} and
* {@link org.apache.storm.trident.Stream#flatMap(FlatMapFunction)} functions.
*/
public class TridentMapExample {
@@ -69,7 +76,8 @@ public boolean isKeep(TridentTuple tuple) {
public static StormTopology buildTopology() {
FixedBatchSpout spout = new FixedBatchSpout(
new Fields("word"), 3, new Values("the cow jumped over the moon"),
- new Values("the man went to the store and bought some candy"), new Values("four score and seven years ago"),
+ new Values("the man went to the store and bought some candy"), new Values("four score "
+ + "and seven years ago"),
new Values("how many apples can you eat"), new Values("to be or not to be the person"));
spout.setCycle(true);
@@ -85,7 +93,8 @@ public void accept(TridentTuple input) {
}
})
.groupBy(new Fields("uppercased"))
- .persistentAggregate(new MemoryMapState.Factory(), new Count(), new Fields("count"))
+ .persistentAggregate(new MemoryMapState.Factory(),
+ new Count(), new Fields("count"))
.parallelismHint(16);
topology.newDRPCStream("words")
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentMinMaxOfDevicesTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentMinMaxOfDevicesTopology.java
index 5944408ee71..e1eb5302e0b 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentMinMaxOfDevicesTopology.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentMinMaxOfDevicesTopology.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
+ *
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -37,7 +43,8 @@
public class TridentMinMaxOfDevicesTopology {
/**
- * Creates a topology with device-id and count (which are whole numbers) as tuple fields in a stream and it finally
+ * Creates a topology with device-id and count (which are whole numbers) as tuple fields in a
+ * stream and it finally
* generates result stream based on min amd max with device-id and count values.
*/
public static StormTopology buildDevicesTopology() {
@@ -64,8 +71,10 @@ public static StormTopology buildDevicesTopology() {
}
/**
- * Creates a topology which demonstrates min/max operations on tuples of stream which contain vehicle and driver fields
- * with values {@link TridentMinMaxOfDevicesTopology.Vehicle} and {@link TridentMinMaxOfDevicesTopology.Driver} respectively.
+ * Creates a topology which demonstrates min/max operations on tuples of stream which contain
+ * vehicle and driver fields
+ * with values {@link TridentMinMaxOfDevicesTopology.Vehicle} and {@link
+ * TridentMinMaxOfDevicesTopology.Driver} respectively.
*/
public static StormTopology buildVehiclesTopology() {
Fields driverField = new Fields(Driver.FIELD_NAME);
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentMinMaxOfVehiclesTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentMinMaxOfVehiclesTopology.java
index 3c0ff31ac83..dd58dc28af9 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentMinMaxOfVehiclesTopology.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentMinMaxOfVehiclesTopology.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
+ *
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -38,8 +44,10 @@
public class TridentMinMaxOfVehiclesTopology {
/**
- * Creates a topology which demonstrates min/max operations on tuples of stream which contain vehicle and driver fields
- * with values {@link TridentMinMaxOfVehiclesTopology.Vehicle} and {@link TridentMinMaxOfVehiclesTopology.Driver} respectively.
+ * Creates a topology which demonstrates min/max operations on tuples of stream which contain
+ * vehicle and driver fields
+ * with values {@link TridentMinMaxOfVehiclesTopology.Vehicle} and {@link
+ * TridentMinMaxOfVehiclesTopology.Driver} respectively.
*/
public static StormTopology buildVehiclesTopology() {
Fields driverField = new Fields(Driver.FIELD_NAME);
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentReach.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentReach.java
index 1a31bc71310..14022c1d2cb 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentReach.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentReach.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -41,7 +47,8 @@ public class TridentReach {
public static Map> TWEETERS_DB = new HashMap>() {
{
put("foo.com/blog/1", Arrays.asList("sally", "bob", "tim", "george", "nathan"));
- put("engineering.twitter.com/blog/5", Arrays.asList("adam", "david", "sally", "nathan"));
+ put("engineering.twitter.com/blog/5", Arrays.asList("adam", "david", "sally",
+ "nathan"));
put("tech.backtype.com/blog/123", Arrays.asList("tim", "mike", "john"));
}
};
@@ -51,7 +58,8 @@ public class TridentReach {
put("sally", Arrays.asList("bob", "tim", "alice", "adam", "jim", "chris", "jai"));
put("bob", Arrays.asList("sally", "nathan", "jim", "mary", "david", "vivian"));
put("tim", Arrays.asList("alex"));
- put("nathan", Arrays.asList("sally", "bob", "adam", "harry", "chris", "vivian", "emily", "jordan"));
+ put("nathan", Arrays.asList("sally", "bob", "adam", "harry", "chris", "vivian", "emily",
+ "jordan"));
put("adam", Arrays.asList("david", "carissa"));
put("mike", Arrays.asList("john", "bob"));
put("john", Arrays.asList("alice", "nathan", "jim", "mike", "bob"));
@@ -60,13 +68,18 @@ public class TridentReach {
public static StormTopology buildTopology() {
TridentTopology topology = new TridentTopology();
- TridentState urlToTweeters = topology.newStaticState(new StaticSingleKeyMapState.Factory(TWEETERS_DB));
- TridentState tweetersToFollowers = topology.newStaticState(new StaticSingleKeyMapState.Factory(FOLLOWERS_DB));
-
-
- topology.newDRPCStream("reach").stateQuery(urlToTweeters, new Fields("args"), new MapGet(), new Fields(
- "tweeters")).each(new Fields("tweeters"), new ExpandList(), new Fields("tweeter")).shuffle().stateQuery(
- tweetersToFollowers, new Fields("tweeter"), new MapGet(), new Fields("followers")).each(new Fields("followers"),
+ TridentState urlToTweeters = topology.newStaticState(new StaticSingleKeyMapState
+ .Factory(TWEETERS_DB));
+ TridentState tweetersToFollowers = topology.newStaticState(new StaticSingleKeyMapState
+ .Factory(FOLLOWERS_DB));
+
+
+ topology.newDRPCStream("reach").stateQuery(urlToTweeters, new Fields("args"), new MapGet(),
+ new Fields(
+ "tweeters")).each(new Fields("tweeters"), new ExpandList(), new Fields("tweeter"))
+ .shuffle().stateQuery(
+ tweetersToFollowers, new Fields("tweeter"), new MapGet(), new Fields("followers"))
+ .each(new Fields("followers"),
new ExpandList(),
new Fields("follower"))
.groupBy(new Fields("follower")).aggregate(new One(), new Fields(
@@ -111,7 +124,8 @@ public Factory(Map map) {
}
@Override
- public State makeState(Map conf, IMetricsContext metrics, int partitionIndex, int numPartitions) {
+ public State makeState(Map conf, IMetricsContext metrics,
+ int partitionIndex, int numPartitions) {
return new StaticSingleKeyMapState(map);
}
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentWindowingInmemoryStoreTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentWindowingInmemoryStoreTopology.java
index 862d09a237d..5265c31cd58 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentWindowingInmemoryStoreTopology.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentWindowingInmemoryStoreTopology.java
@@ -1,12 +1,17 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -35,20 +40,26 @@
* Sample application of trident windowing which uses inmemory store for storing tuples in window.
*/
public class TridentWindowingInmemoryStoreTopology {
- private static final Logger LOG = LoggerFactory.getLogger(TridentWindowingInmemoryStoreTopology.class);
+ private static final Logger LOG = LoggerFactory
+ .getLogger(TridentWindowingInmemoryStoreTopology.class);
- public static StormTopology buildTopology(WindowsStoreFactory windowStore, WindowConfig windowConfig) throws Exception {
- FixedBatchSpout spout = new FixedBatchSpout(new Fields("sentence"), 3, new Values("the cow jumped over the moon"),
- new Values("the man went to the store and bought some candy"),
+ public static StormTopology buildTopology(WindowsStoreFactory windowStore,
+ WindowConfig windowConfig) throws Exception {
+ FixedBatchSpout spout = new FixedBatchSpout(new Fields("sentence"), 3,
+ new Values("the cow jumped over the moon"),
+ new Values("the man went to the store and "
+ + "bought some candy"),
new Values("four score and seven years ago"),
new Values("how many apples can you eat"), new Values("to be or not to be the person"));
spout.setCycle(true);
TridentTopology topology = new TridentTopology();
- Stream stream = topology.newStream("spout1", spout).parallelismHint(16).each(new Fields("sentence"),
+ Stream stream = topology.newStream("spout1", spout).parallelismHint(16)
+ .each(new Fields("sentence"),
new Split(), new Fields("word"))
- .window(windowConfig, windowStore, new Fields("word"), new CountAsAggregator(), new Fields("count"))
+ .window(windowConfig, windowStore, new Fields("word"),
+ new CountAsAggregator(), new Fields("count"))
.peek(new Consumer() {
@Override
public void accept(TridentTuple input) {
@@ -68,6 +79,7 @@ public static void main(String[] args) throws Exception {
}
conf.setNumWorkers(3);
- StormSubmitter.submitTopologyWithProgressBar(topoName, conf, buildTopology(mapState, SlidingCountWindow.of(1000, 100)));
+ StormSubmitter.submitTopologyWithProgressBar(topoName, conf, buildTopology(mapState,
+ SlidingCountWindow.of(1000, 100)));
}
}
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentWordCount.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentWordCount.java
index bafeba29c24..9c3e4f8824e 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentWordCount.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentWordCount.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -29,19 +35,22 @@
import org.apache.storm.tuple.Values;
import org.apache.storm.utils.DRPCClient;
-
public class TridentWordCount {
public static StormTopology buildTopology() {
- FixedBatchSpout spout = new FixedBatchSpout(new Fields("sentence"), 3, new Values("the cow jumped over the moon"),
- new Values("the man went to the store and bought some candy"),
+ FixedBatchSpout spout = new FixedBatchSpout(new Fields("sentence"), 3,
+ new Values("the cow jumped over the moon"),
+ new Values("the man went to the store and "
+ + "bought some candy"),
new Values("four score and seven years ago"),
new Values("how many apples can you eat"), new Values("to be or not to be the person"));
spout.setCycle(true);
TridentTopology topology = new TridentTopology();
- TridentState wordCounts = topology.newStream("spout1", spout).parallelismHint(16).each(new Fields("sentence"),
+ TridentState wordCounts = topology.newStream("spout1", spout).parallelismHint(16)
+ .each(new Fields("sentence"),
new Split(), new Fields("word"))
- .groupBy(new Fields("word")).persistentAggregate(new MemoryMapState.Factory(),
+ .groupBy(new Fields("word"))
+ .persistentAggregate(new MemoryMapState.Factory(),
new Count(), new Fields("count"))
.parallelismHint(16);
diff --git a/examples/storm-starter/test/jvm/org/apache/storm/starter/bolt/IntermediateRankingsBoltTest.java b/examples/storm-starter/test/jvm/org/apache/storm/starter/bolt/IntermediateRankingsBoltTest.java
index 18834d3095c..2f45b35c21f 100644
--- a/examples/storm-starter/test/jvm/org/apache/storm/starter/bolt/IntermediateRankingsBoltTest.java
+++ b/examples/storm-starter/test/jvm/org/apache/storm/starter/bolt/IntermediateRankingsBoltTest.java
@@ -1,17 +1,31 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
package org.apache.storm.starter.bolt;
+import static org.fest.assertions.api.Assertions.assertThat;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoInteractions;
+import static org.mockito.Mockito.when;
+
import com.google.common.collect.Lists;
import java.util.Map;
import org.apache.storm.Config;
@@ -24,14 +38,6 @@
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
-import static org.fest.assertions.api.Assertions.assertThat;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.verifyNoInteractions;
-import static org.mockito.Mockito.when;
-
public class IntermediateRankingsBoltTest {
private static final String ANY_NON_SYSTEM_COMPONENT_ID = "irrelevant_component_id";
@@ -41,7 +47,8 @@ public class IntermediateRankingsBoltTest {
private static final long ANY_COUNT = 42;
private Tuple mockRankableTuple(Object obj, long count) {
- Tuple tuple = MockTupleHelpers.mockTuple(ANY_NON_SYSTEM_COMPONENT_ID, ANY_NON_SYSTEM_STREAM_ID);
+ Tuple tuple = MockTupleHelpers.mockTuple(ANY_NON_SYSTEM_COMPONENT_ID,
+ ANY_NON_SYSTEM_STREAM_ID);
when(tuple.getValues()).thenReturn(Lists.newArrayList(ANY_OBJECT, ANY_COUNT));
return tuple;
}
@@ -61,7 +68,8 @@ public Object[][] illegalEmitFrequency() {
return new Object[][]{ { -10 }, { -3 }, { -2 }, { -1 }, { 0 } };
}
- @Test(expectedExceptions = IllegalArgumentException.class, dataProvider = "illegalEmitFrequency")
+ @Test(expectedExceptions = IllegalArgumentException.class,
+ dataProvider = "illegalEmitFrequency")
public void negativeOrZeroEmitFrequencyShouldThrowIAE(int emitFrequencyInSeconds) {
new IntermediateRankingsBolt(ANY_TOPN, emitFrequencyInSeconds);
}
@@ -138,7 +146,8 @@ public void shouldSetTickTupleFrequencyInComponentConfigurationToNonZeroValue()
// then
assertThat(componentConfig).containsKey(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS);
- Integer emitFrequencyInSeconds = (Integer) componentConfig.get(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS);
+ Integer emitFrequencyInSeconds = (Integer) componentConfig
+ .get(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS);
assertThat(emitFrequencyInSeconds).isGreaterThan(0);
}
}
diff --git a/examples/storm-starter/test/jvm/org/apache/storm/starter/bolt/RollingCountBoltTest.java b/examples/storm-starter/test/jvm/org/apache/storm/starter/bolt/RollingCountBoltTest.java
index 56d65606520..f8f5dd3fc15 100644
--- a/examples/storm-starter/test/jvm/org/apache/storm/starter/bolt/RollingCountBoltTest.java
+++ b/examples/storm-starter/test/jvm/org/apache/storm/starter/bolt/RollingCountBoltTest.java
@@ -1,17 +1,31 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
package org.apache.storm.starter.bolt;
+import static org.fest.assertions.api.Assertions.assertThat;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoInteractions;
+import static org.mockito.Mockito.when;
+
import java.util.Map;
import org.apache.storm.Config;
import org.apache.storm.task.OutputCollector;
@@ -23,21 +37,14 @@
import org.apache.storm.utils.MockTupleHelpers;
import org.testng.annotations.Test;
-import static org.fest.assertions.api.Assertions.assertThat;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.verifyNoInteractions;
-import static org.mockito.Mockito.when;
-
public class RollingCountBoltTest {
private static final String ANY_NON_SYSTEM_COMPONENT_ID = "irrelevant_component_id";
private static final String ANY_NON_SYSTEM_STREAM_ID = "irrelevant_stream_id";
private Tuple mockNormalTuple(Object obj) {
- Tuple tuple = MockTupleHelpers.mockTuple(ANY_NON_SYSTEM_COMPONENT_ID, ANY_NON_SYSTEM_STREAM_ID);
+ Tuple tuple = MockTupleHelpers.mockTuple(ANY_NON_SYSTEM_COMPONENT_ID,
+ ANY_NON_SYSTEM_STREAM_ID);
when(tuple.getValue(0)).thenReturn(obj);
return tuple;
}
@@ -105,7 +112,8 @@ public void shouldSetTickTupleFrequencyInComponentConfigurationToNonZeroValue()
// then
assertThat(componentConfig).containsKey(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS);
- Integer emitFrequencyInSeconds = (Integer) componentConfig.get(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS);
+ Integer emitFrequencyInSeconds = (Integer) componentConfig
+ .get(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS);
assertThat(emitFrequencyInSeconds).isGreaterThan(0);
}
}
diff --git a/examples/storm-starter/test/jvm/org/apache/storm/starter/bolt/TotalRankingsBoltTest.java b/examples/storm-starter/test/jvm/org/apache/storm/starter/bolt/TotalRankingsBoltTest.java
index d74e41e5a19..43a3d9b5263 100644
--- a/examples/storm-starter/test/jvm/org/apache/storm/starter/bolt/TotalRankingsBoltTest.java
+++ b/examples/storm-starter/test/jvm/org/apache/storm/starter/bolt/TotalRankingsBoltTest.java
@@ -1,17 +1,31 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
package org.apache.storm.starter.bolt;
+import static org.fest.assertions.api.Assertions.assertThat;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoInteractions;
+import static org.mockito.Mockito.when;
+
import java.util.Map;
import org.apache.storm.Config;
import org.apache.storm.starter.tools.Rankings;
@@ -24,14 +38,6 @@
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
-import static org.fest.assertions.api.Assertions.assertThat;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.verifyNoInteractions;
-import static org.mockito.Mockito.when;
-
public class TotalRankingsBoltTest {
private static final String ANY_NON_SYSTEM_COMPONENT_ID = "irrelevant_component_id";
@@ -41,7 +47,8 @@ public class TotalRankingsBoltTest {
private static final long ANY_COUNT = 42;
private Tuple mockRankingsTuple(Object obj, long count) {
- Tuple tuple = MockTupleHelpers.mockTuple(ANY_NON_SYSTEM_COMPONENT_ID, ANY_NON_SYSTEM_STREAM_ID);
+ Tuple tuple = MockTupleHelpers.mockTuple(ANY_NON_SYSTEM_COMPONENT_ID,
+ ANY_NON_SYSTEM_STREAM_ID);
Rankings rankings = mock(Rankings.class);
when(tuple.getValue(0)).thenReturn(rankings);
return tuple;
@@ -62,7 +69,8 @@ public Object[][] illegalEmitFrequency() {
return new Object[][]{ { -10 }, { -3 }, { -2 }, { -1 }, { 0 } };
}
- @Test(expectedExceptions = IllegalArgumentException.class, dataProvider = "illegalEmitFrequency")
+ @Test(expectedExceptions = IllegalArgumentException.class,
+ dataProvider = "illegalEmitFrequency")
public void negativeOrZeroEmitFrequencyShouldThrowIAE(int emitFrequencyInSeconds) {
new TotalRankingsBolt(ANY_TOPN, emitFrequencyInSeconds);
}
@@ -139,7 +147,8 @@ public void shouldSetTickTupleFrequencyInComponentConfigurationToNonZeroValue()
// then
assertThat(componentConfig).containsKey(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS);
- Integer emitFrequencyInSeconds = (Integer) componentConfig.get(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS);
+ Integer emitFrequencyInSeconds = (Integer) componentConfig
+ .get(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS);
assertThat(emitFrequencyInSeconds).isGreaterThan(0);
}
}
diff --git a/examples/storm-starter/test/jvm/org/apache/storm/starter/tools/NthLastModifiedTimeTrackerTest.java b/examples/storm-starter/test/jvm/org/apache/storm/starter/tools/NthLastModifiedTimeTrackerTest.java
index f4393ed41b4..fa908c9a6da 100644
--- a/examples/storm-starter/test/jvm/org/apache/storm/starter/tools/NthLastModifiedTimeTrackerTest.java
+++ b/examples/storm-starter/test/jvm/org/apache/storm/starter/tools/NthLastModifiedTimeTrackerTest.java
@@ -1,24 +1,30 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
package org.apache.storm.starter.tools;
-import org.apache.storm.utils.Time;
+import static org.fest.assertions.api.Assertions.assertThat;
+
import org.apache.storm.utils.Time.SimulatedTime;
+import org.apache.storm.utils.Time;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
-import static org.fest.assertions.api.Assertions.assertThat;
-
public class NthLastModifiedTimeTrackerTest {
private static final int ANY_NUM_TIMES_TO_TRACK = 3;
@@ -53,7 +59,8 @@ public Object[][] whenNotYetMarkedAsModifiedData() {
public void shouldReturnCorrectModifiedTimeEvenWhenNotYetMarkedAsModified(int secondsToAdvance) {
// given
try (SimulatedTime t = new SimulatedTime()) {
- NthLastModifiedTimeTracker tracker = new NthLastModifiedTimeTracker(ANY_NUM_TIMES_TO_TRACK);
+ NthLastModifiedTimeTracker tracker =
+ new NthLastModifiedTimeTracker(ANY_NUM_TIMES_TO_TRACK);
// when
Time.advanceTimeSecs(secondsToAdvance);
diff --git a/examples/storm-starter/test/jvm/org/apache/storm/starter/tools/RankableObjectWithFieldsTest.java b/examples/storm-starter/test/jvm/org/apache/storm/starter/tools/RankableObjectWithFieldsTest.java
index cf275d26d72..f715e8fe937 100644
--- a/examples/storm-starter/test/jvm/org/apache/storm/starter/tools/RankableObjectWithFieldsTest.java
+++ b/examples/storm-starter/test/jvm/org/apache/storm/starter/tools/RankableObjectWithFieldsTest.java
@@ -1,17 +1,29 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
package org.apache.storm.starter.tools;
+import static org.fest.assertions.api.Assertions.assertThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertTrue;
+
import com.google.common.collect.Lists;
import java.util.ArrayList;
import java.util.List;
@@ -19,12 +31,6 @@
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
-import static org.fest.assertions.api.Assertions.assertThat;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-import static org.testng.Assert.assertFalse;
-import static org.testng.Assert.assertTrue;
-
public class RankableObjectWithFieldsTest {
private static final Object ANY_OBJECT = new Object();
@@ -67,14 +73,15 @@ public Object[][] otherClassesData() {
return new Object[][]{
{"foo"}, { new Object() }, {4}, {
Lists.newArrayList(7, 8, 9)
- }
+ }
};
}
@Test(dataProvider = "otherClassesData")
public void shouldNotBeEqualToInstancesOfOtherClasses(Object notARankable) {
RankableObjectWithFields r = new RankableObjectWithFields(ANY_OBJECT, ANY_COUNT);
- assertFalse(r.equals(notARankable), r + " is equal to " + notARankable + " but it should not be");
+ assertFalse(r.equals(notARankable), r + " is equal to " + notARankable
+ + " but it should not be");
}
@DataProvider
@@ -85,15 +92,17 @@ public Object[][] falseDuplicatesData() {
{ new RankableObjectWithFields("foo", 1), new RankableObjectWithFields("FOO", 1) },
{ new RankableObjectWithFields("foo", 1), new RankableObjectWithFields("bar", 1) },
{ new RankableObjectWithFields("", 0), new RankableObjectWithFields("", 1) }, {
- new RankableObjectWithFields("",
+ new RankableObjectWithFields("",
1), new RankableObjectWithFields("bar", 1)
- }
+ }
};
}
@Test(dataProvider = "falseDuplicatesData")
- public void shouldNotBeEqualToFalseDuplicates(RankableObjectWithFields r, RankableObjectWithFields falseDuplicate) {
- assertFalse(r.equals(falseDuplicate), r + " is equal to " + falseDuplicate + " but it should not be");
+ public void shouldNotBeEqualToFalseDuplicates(RankableObjectWithFields r,
+ RankableObjectWithFields falseDuplicate) {
+ assertFalse(r.equals(falseDuplicate), r + " is equal to " + falseDuplicate
+ + " but it should not be");
}
@Test(dataProvider = "falseDuplicatesData")
@@ -106,17 +115,21 @@ public void shouldHaveDifferentHashCodeThanFalseDuplicates(RankableObjectWithFie
public Object[][] trueDuplicatesData() {
return new Object[][]{
{ new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("foo", 0) },
- { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("foo", 0, "someOtherField") },
+ { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("foo", 0,
+ "someOtherField") },
{
- new RankableObjectWithFields("foo", 0, "someField"), new RankableObjectWithFields("foo", 0,
+ new RankableObjectWithFields("foo", 0,
+ "someField"), new RankableObjectWithFields("foo", 0,
"someOtherField")
}
};
}
@Test(dataProvider = "trueDuplicatesData")
- public void shouldBeEqualToTrueDuplicates(RankableObjectWithFields r, RankableObjectWithFields trueDuplicate) {
- assertTrue(r.equals(trueDuplicate), r + " is not equal to " + trueDuplicate + " but it should be");
+ public void shouldBeEqualToTrueDuplicates(RankableObjectWithFields r,
+ RankableObjectWithFields trueDuplicate) {
+ assertTrue(r.equals(trueDuplicate), r + " is not equal to " + trueDuplicate
+ + " but it should be");
}
@Test(dataProvider = "trueDuplicatesData")
@@ -132,25 +145,32 @@ public Object[][] compareToData() {
new RankableObjectWithFields("foo", 1000), new RankableObjectWithFields("foo", 0),
GREATER_THAN
}, {
- new RankableObjectWithFields("foo", 1), new RankableObjectWithFields("foo", 0),
- GREATER_THAN
- }, {
- new RankableObjectWithFields("foo", 1000), new RankableObjectWithFields("bar", 0),
- GREATER_THAN
- }, {
- new RankableObjectWithFields("foo", 1), new RankableObjectWithFields("bar", 0),
- GREATER_THAN
- }, { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("foo", 0), EQUAL_TO },
- { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("bar", 0), EQUAL_TO },
- { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("foo", 1000), SMALLER_THAN },
- { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("foo", 1), SMALLER_THAN },
- { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("bar", 1), SMALLER_THAN },
- { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("bar", 1000), SMALLER_THAN },
+ new RankableObjectWithFields("foo", 1), new RankableObjectWithFields("foo", 0),
+ GREATER_THAN
+ }, {
+ new RankableObjectWithFields("foo", 1000), new RankableObjectWithFields("bar", 0),
+ GREATER_THAN
+ }, {
+ new RankableObjectWithFields("foo", 1), new RankableObjectWithFields("bar", 0),
+ GREATER_THAN
+ }, { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("foo",
+ 0), EQUAL_TO },
+ { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("bar",
+ 0), EQUAL_TO },
+ { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("foo",
+ 1000), SMALLER_THAN },
+ { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("foo",
+ 1), SMALLER_THAN },
+ { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("bar",
+ 1), SMALLER_THAN },
+ { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("bar",
+ 1000), SMALLER_THAN },
};
}
@Test(dataProvider = "compareToData")
- public void verifyCompareTo(RankableObjectWithFields first, RankableObjectWithFields second, int expCompareToValue) {
+ public void verifyCompareTo(RankableObjectWithFields first, RankableObjectWithFields second,
+ int expCompareToValue) {
assertThat(first.compareTo(second)).isEqualTo(expCompareToValue);
}
@@ -251,13 +271,14 @@ public void shouldCreateRankableObjectFromTuple() {
public Object[][] copyData() {
return new Object[][]{
{ new RankableObjectWithFields("foo", 0) }, {
- new RankableObjectWithFields("foo", 3,
+ new RankableObjectWithFields("foo", 3,
"someOtherField")
- }, { new RankableObjectWithFields("foo", 0, "someField") }
+ }, { new RankableObjectWithFields("foo", 0, "someField") }
};
}
- // TODO: What would be a good test to ensure that RankableObjectWithFields is at least somewhat defensively copied?
+ // TODO: What would be a good test to ensure that RankableObjectWithFields is at least somewhat
+ // defensively copied?
// The contract of Rankable#copy() returns a Rankable value, not a RankableObjectWithFields.
@Test(dataProvider = "copyData")
public void copyShouldReturnCopy(RankableObjectWithFields original) {
diff --git a/examples/storm-starter/test/jvm/org/apache/storm/starter/tools/RankingsTest.java b/examples/storm-starter/test/jvm/org/apache/storm/starter/tools/RankingsTest.java
index 206d45c77d4..565a781ee7d 100644
--- a/examples/storm-starter/test/jvm/org/apache/storm/starter/tools/RankingsTest.java
+++ b/examples/storm-starter/test/jvm/org/apache/storm/starter/tools/RankingsTest.java
@@ -1,17 +1,25 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
package org.apache.storm.starter.tools;
+import static org.fest.assertions.api.Assertions.assertThat;
+
import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
@@ -20,12 +28,11 @@
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
-import static org.fest.assertions.api.Assertions.assertThat;
-
public class RankingsTest {
private static final int ANY_TOPN = 42;
- private static final Rankable ANY_RANKABLE = new RankableObjectWithFields("someObject", ANY_TOPN);
+ private static final Rankable ANY_RANKABLE = new RankableObjectWithFields("someObject",
+ ANY_TOPN);
private static final Rankable ZERO = new RankableObjectWithFields("ZERO_COUNT", 0);
private static final Rankable A = new RankableObjectWithFields("A", 1);
private static final Rankable B = new RankableObjectWithFields("B", 2);
@@ -50,7 +57,8 @@ public void constructorWithNegativeOrZeroTopNShouldThrowIAE(int topN) {
public Object[][] copyRankingsData() {
return new Object[][]{
{ 5, Lists.newArrayList(A, B, C) }, { 2, Lists.newArrayList(A, B, C, D) },
- { 1, Lists.newArrayList() }, { 1, Lists.newArrayList(A) }, { 1, Lists.newArrayList(A, B) }
+ { 1, Lists.newArrayList() }, { 1, Lists.newArrayList(A) }, { 1, Lists.newArrayList(A,
+ B) }
};
}
@@ -74,20 +82,21 @@ public void copyConstructorShouldReturnCopy(int topN, List rankables)
public Object[][] defensiveCopyRankingsData() {
return new Object[][]{
{ 5, Lists.newArrayList(A, B, C), Lists.newArrayList(D) }, {
- 2, Lists.newArrayList(A, B, C,
+ 2, Lists.newArrayList(A, B, C,
D), Lists.newArrayList(E, F)
- }, { 1, Lists.newArrayList(), Lists.newArrayList(A) }, {
- 1, Lists.newArrayList(A),
+ }, { 1, Lists.newArrayList(), Lists.newArrayList(A) }, {
+ 1, Lists.newArrayList(A),
Lists.newArrayList(B)
- }, { 1, Lists.newArrayList(ZERO), Lists.newArrayList(B) }, {
- 1, Lists.newArrayList(ZERO),
+ }, { 1, Lists.newArrayList(ZERO), Lists.newArrayList(B) }, {
+ 1, Lists.newArrayList(ZERO),
Lists.newArrayList()
- }
+ }
};
}
@Test(dataProvider = "defensiveCopyRankingsData")
- public void copyConstructorShouldReturnDefensiveCopy(int topN, List rankables, List changes) {
+ public void copyConstructorShouldReturnDefensiveCopy(int topN, List rankables,
+ List changes) {
// given
Rankings original = new Rankings(topN);
for (Rankable r : rankables) {
@@ -139,14 +148,16 @@ public void defaultConstructorShouldSetPositiveTopN() {
public Object[][] rankingsGrowData() {
return new Object[][]{
{
- 2, Lists.newArrayList(new RankableObjectWithFields("A", 1), new RankableObjectWithFields(
- "B", 2), new RankableObjectWithFields("C", 3))
+ 2, Lists.newArrayList(new RankableObjectWithFields("A", 1),
+ new RankableObjectWithFields(
+ "B", 2), new RankableObjectWithFields("C", 3))
}, {
- 2, Lists.newArrayList(new RankableObjectWithFields("A", 1),
- new RankableObjectWithFields("B", 2), new RankableObjectWithFields("C", 3),
+ 2, Lists.newArrayList(new RankableObjectWithFields("A", 1),
+ new RankableObjectWithFields("B",
+ 2), new RankableObjectWithFields("C", 3),
new RankableObjectWithFields("D",
4))
- }
+ }
};
}
@@ -154,7 +165,8 @@ public Object[][] rankingsGrowData() {
public void sizeOfRankingsShouldNotGrowBeyondTopN(int topN, List rankables) {
// sanity check of the provided test data
assertThat(rankables.size()).overridingErrorMessage(
- "The supplied test data is not correct: the number of rankables <%d> should be greater than <%d>",
+ "The supplied test data is not correct: the number of rankables <%d> should be "
+ + "greater than <%d>",
rankables.size(), topN).isGreaterThan(topN);
// given
@@ -175,15 +187,17 @@ public Object[][] simulatedRankingsData() {
{ Lists.newArrayList(A), Lists.newArrayList(A) }, {
Lists.newArrayList(B, D, A, C),
Lists.newArrayList(D, C, B, A)
- }, {
+ }, {
Lists.newArrayList(B, F, A, C, D, E), Lists.newArrayList(F, E, D, C, B,
A)
- }, { Lists.newArrayList(G, B, F, A, C, D, E, H), Lists.newArrayList(H, G, F, E, D, C, B, A) }
+ }, { Lists.newArrayList(G, B, F, A, C, D, E, H), Lists.newArrayList(H, G, F, E, D, C, B,
+ A) }
};
}
@Test(dataProvider = "simulatedRankingsData")
- public void shouldCorrectlyRankWhenUpdatedWithRankables(List unsorted, List expSorted) {
+ public void shouldCorrectlyRankWhenUpdatedWithRankables(List unsorted,
+ List expSorted) {
// given
Rankings rankings = new Rankings(unsorted.size());
@@ -214,7 +228,8 @@ public void shouldCorrectlyRankWhenEmptyAndUpdatedWithOtherRankings(List unsorted, List expSorted) {
+ public void shouldCorrectlyRankWhenUpdatedWithEmptyOtherRankings(List unsorted,
+ List expSorted) {
// given
Rankings rankings = new Rankings(unsorted.size());
for (Rankable r : unsorted) {
@@ -233,13 +248,15 @@ public void shouldCorrectlyRankWhenUpdatedWithEmptyOtherRankings(List
public Object[][] simulatedRankingsAndOtherRankingsData() {
return new Object[][]{
{ Lists.newArrayList(A), Lists.newArrayList(A), Lists.newArrayList(A) },
- { Lists.newArrayList(A, C), Lists.newArrayList(B, D), Lists.newArrayList(D, C, B, A) }, {
+ { Lists.newArrayList(A, C), Lists.newArrayList(B, D), Lists.newArrayList(D, C, B,
+ A) }, {
Lists.newArrayList(B,
- F, A), Lists.newArrayList(C, D, E), Lists.newArrayList(F, E, D, C, B, A)
- }, {
+ F, A), Lists.newArrayList(C, D, E), Lists
+ .newArrayList(F, E, D, C, B, A)
+ }, {
Lists.newArrayList(G, B, F, A, C),
Lists.newArrayList(D, E, H), Lists.newArrayList(H, G, F, E, D, C, B, A)
- }
+ }
};
}
@@ -273,7 +290,7 @@ public Object[][] duplicatesData() {
{ Lists.newArrayList(ANY_RANKABLE, ANY_RANKABLE, ANY_RANKABLE) }, {
Lists.newArrayList(A1,
A2, A3)
- },
+ },
};
}
@@ -297,18 +314,20 @@ public Object[][] removeZeroRankingsData() {
{ Lists.newArrayList(A, ZERO), Lists.newArrayList(A) }, {
Lists.newArrayList(A),
Lists.newArrayList(A)
- }, { Lists.newArrayList(ZERO, A), Lists.newArrayList(A) }, {
+ }, { Lists.newArrayList(ZERO, A), Lists.newArrayList(A) }, {
Lists.newArrayList(ZERO),
Lists.newArrayList()
- }, {
+ }, {
Lists.newArrayList(ZERO, new RankableObjectWithFields("ZERO2", 0)),
Lists.newArrayList()
- }, {
+ }, {
Lists.newArrayList(B, ZERO, new RankableObjectWithFields("ZERO2", 0), D,
- new RankableObjectWithFields("ZERO3", 0), new RankableObjectWithFields("ZERO4", 0), C), Lists.newArrayList(D,
+ new RankableObjectWithFields("ZERO3",
+ 0), new RankableObjectWithFields("ZERO4", 0), C), Lists
+ .newArrayList(D,
C,
B)
- }, { Lists.newArrayList(A, ZERO, B), Lists.newArrayList(B, A) }
+ }, { Lists.newArrayList(A, ZERO, B), Lists.newArrayList(B, A) }
};
}
@@ -333,8 +352,10 @@ public void updatingWithNewRankablesShouldBeThreadSafe() throws InterruptedExcep
final List entries = ImmutableList.of(A, B, C, D);
final Rankings rankings = new Rankings(entries.size());
- // We are capturing exceptions thrown in Blitzer's child threads into this data structure so that we can properly
- // pass/fail this test. The reason is that Blitzer doesn't report exceptions, which is a known bug in Blitzer
+ // We are capturing exceptions thrown in Blitzer's child threads into this data structure so
+ // that we can properly
+ // pass/fail this test. The reason is that Blitzer doesn't report exceptions, which is a
+ // known bug in Blitzer
// (JMOCK-263). See https://github.com/jmock-developers/jmock-library/issues/22 for more information.
final List exceptions = Lists.newArrayList();
Blitzer blitzer = new Blitzer(1000);
@@ -383,7 +404,8 @@ public void copyShouldReturnCopy(int topN, List rankables) {
}
@Test(dataProvider = "defensiveCopyRankingsData")
- public void copyShouldReturnDefensiveCopy(int topN, List rankables, List changes) {
+ public void copyShouldReturnDefensiveCopy(int topN, List rankables,
+ List changes) {
// given
Rankings original = new Rankings(topN);
for (Rankable r : rankables) {
diff --git a/examples/storm-starter/test/jvm/org/apache/storm/starter/tools/SlidingWindowCounterTest.java b/examples/storm-starter/test/jvm/org/apache/storm/starter/tools/SlidingWindowCounterTest.java
index 3bdb4ff25e6..2c0a5f30073 100644
--- a/examples/storm-starter/test/jvm/org/apache/storm/starter/tools/SlidingWindowCounterTest.java
+++ b/examples/storm-starter/test/jvm/org/apache/storm/starter/tools/SlidingWindowCounterTest.java
@@ -1,23 +1,29 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
package org.apache.storm.starter.tools;
+import static org.fest.assertions.api.Assertions.assertThat;
+
import java.util.Map;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
-import static org.fest.assertions.api.Assertions.assertThat;
-
public class SlidingWindowCounterTest {
private static final int ANY_WINDOW_LENGTH_IN_SLOTS = 2;
@@ -28,7 +34,8 @@ public Object[][] illegalWindowLengths() {
return new Object[][]{ { -10 }, { -3 }, { -2 }, { -1 }, { 0 }, { 1 } };
}
- @Test(expectedExceptions = IllegalArgumentException.class, dataProvider = "illegalWindowLengths")
+ @Test(expectedExceptions = IllegalArgumentException.class,
+ dataProvider = "illegalWindowLengths")
public void lessThanTwoSlotsShouldThrowIAE(int windowLengthInSlots) {
new SlidingWindowCounter(windowLengthInSlots);
}
@@ -46,7 +53,8 @@ public void twoOrMoreSlotsShouldBeValid(int windowLengthInSlots) {
@Test
public void newInstanceShouldHaveEmptyCounts() {
// given
- SlidingWindowCounter counter = new SlidingWindowCounter(ANY_WINDOW_LENGTH_IN_SLOTS);
+ SlidingWindowCounter counter =
+ new SlidingWindowCounter(ANY_WINDOW_LENGTH_IN_SLOTS);
// when
Map counts = counter.getCountsThenAdvanceWindow();
@@ -73,7 +81,8 @@ public Object[][] simulatedCounterIterations() {
public void testCounterWithSimulatedRuns(int windowLengthInSlots, int[] incrementsPerIteration,
long[] expCountsPerIteration) {
// given
- SlidingWindowCounter counter = new SlidingWindowCounter(windowLengthInSlots);
+ SlidingWindowCounter counter =
+ new SlidingWindowCounter(windowLengthInSlots);
int numIterations = incrementsPerIteration.length;
for (int i = 0; i < numIterations; i++) {
@@ -81,7 +90,8 @@ public void testCounterWithSimulatedRuns(int windowLengthInSlots, int[] incremen
long expCounts = expCountsPerIteration[i];
// Objects are absent if they were zero both this iteration
// and the last -- if only this one, we need to report zero.
- boolean expAbsent = ((expCounts == 0) && ((i == 0) || (expCountsPerIteration[i - 1] == 0)));
+ boolean expAbsent = ((expCounts == 0) && ((i == 0)
+ || (expCountsPerIteration[i - 1] == 0)));
// given (for this iteration)
for (int j = 0; j < numIncrements; j++) {
diff --git a/examples/storm-starter/test/jvm/org/apache/storm/starter/tools/SlotBasedCounterTest.java b/examples/storm-starter/test/jvm/org/apache/storm/starter/tools/SlotBasedCounterTest.java
index df2ab21d65c..90d21d1eb0f 100644
--- a/examples/storm-starter/test/jvm/org/apache/storm/starter/tools/SlotBasedCounterTest.java
+++ b/examples/storm-starter/test/jvm/org/apache/storm/starter/tools/SlotBasedCounterTest.java
@@ -1,23 +1,29 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
package org.apache.storm.starter.tools;
+import static org.fest.assertions.api.Assertions.assertThat;
+
import java.util.Map;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
-import static org.fest.assertions.api.Assertions.assertThat;
-
public class SlotBasedCounterTest {
private static final int ANY_NUM_SLOTS = 1;
diff --git a/external/storm-autocreds/pom.xml b/external/storm-autocreds/pom.xml
index 80ecdc4f1ba..bdf1c3cff82 100644
--- a/external/storm-autocreds/pom.xml
+++ b/external/storm-autocreds/pom.xml
@@ -112,6 +112,16 @@
+
+ org.openrewrite.maven
+ rewrite-maven-plugin
+
+
+
+ org.codehaus.gmavenplus
+ gmavenplus-plugin
+
+ org.apache.maven.pluginsmaven-checkstyle-plugin
diff --git a/external/storm-autocreds/src/main/java/org/apache/storm/common/AbstractHadoopAutoCreds.java b/external/storm-autocreds/src/main/java/org/apache/storm/common/AbstractHadoopAutoCreds.java
index e8a7ac791d6..0680b3eea43 100644
--- a/external/storm-autocreds/src/main/java/org/apache/storm/common/AbstractHadoopAutoCreds.java
+++ b/external/storm-autocreds/src/main/java/org/apache/storm/common/AbstractHadoopAutoCreds.java
@@ -7,9 +7,9 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
+ *
Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
@@ -26,7 +26,6 @@
import java.util.Set;
import javax.security.auth.Subject;
import javax.xml.bind.DatatypeConverter;
-
import org.apache.commons.math3.util.Pair;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
@@ -37,7 +36,8 @@
import org.slf4j.LoggerFactory;
/**
- * The base class that for auto credential plugins that abstracts out some of the common functionality.
+ * The base class that for auto credential plugins that abstracts out some of the common
+ * functionality.
*/
public abstract class AbstractHadoopAutoCreds implements IAutoCredentials, CredentialKeyProvider {
private static final Logger LOG = LoggerFactory.getLogger(AbstractHadoopAutoCreds.class);
@@ -118,13 +118,15 @@ private void addTokensToUgi(Subject subject) {
continue;
}
- LOG.debug("Current user: {}", UserGroupInformation.getCurrentUser());
+ LOG.debug("Current user: {}", UserGroupInformation
+ .getCurrentUser());
LOG.debug("Token from Credentials : {}", token);
TokenIdentifier tokenId = token.decodeIdentifier();
if (tokenId != null) {
LOG.debug("Token identifier : {}", tokenId);
- LOG.debug("Username in token identifier : {}", tokenId.getUser());
+ LOG.debug("Username in token identifier : {}", tokenId
+ .getUser());
}
UserGroupInformation.getCurrentUser().addToken(token);
diff --git a/external/storm-autocreds/src/main/java/org/apache/storm/common/AbstractHadoopNimbusPluginAutoCreds.java b/external/storm-autocreds/src/main/java/org/apache/storm/common/AbstractHadoopNimbusPluginAutoCreds.java
index ad3bd17886f..605e6bb16b4 100644
--- a/external/storm-autocreds/src/main/java/org/apache/storm/common/AbstractHadoopNimbusPluginAutoCreds.java
+++ b/external/storm-autocreds/src/main/java/org/apache/storm/common/AbstractHadoopNimbusPluginAutoCreds.java
@@ -7,9 +7,9 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
+ *
Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
@@ -25,7 +25,6 @@
import java.util.Map;
import java.util.Set;
import javax.xml.bind.DatatypeConverter;
-
import org.apache.commons.math3.util.Pair;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
@@ -38,11 +37,13 @@
import org.slf4j.LoggerFactory;
/**
- * The base class that for auto credential plugins that abstracts out some of the common functionality.
+ * The base class that for auto credential plugins that abstracts out some of the common
+ * functionality.
*/
public abstract class AbstractHadoopNimbusPluginAutoCreds
implements INimbusCredentialPlugin, ICredentialsRenewer, CredentialKeyProvider {
- private static final Logger LOG = LoggerFactory.getLogger(AbstractHadoopNimbusPluginAutoCreds.class);
+ private static final Logger LOG = LoggerFactory
+ .getLogger(AbstractHadoopNimbusPluginAutoCreds.class);
public static final String CONFIG_KEY_RESOURCES = "resources";
@Override
@@ -59,11 +60,13 @@ public void populateCredentials(Map credentials,
if (!configKeys.isEmpty()) {
for (String configKey : configKeys) {
credentials.put(getCredentialKey(configKey),
- DatatypeConverter.printBase64Binary(getHadoopCredentials(topologyConf, configKey)));
+ DatatypeConverter.printBase64Binary(getHadoopCredentials(topologyConf,
+ configKey)));
}
} else {
credentials.put(getCredentialKey(""),
- DatatypeConverter.printBase64Binary(getHadoopCredentials(topologyConf, topologyOwnerPrincipal)));
+ DatatypeConverter.printBase64Binary(getHadoopCredentials(topologyConf,
+ topologyOwnerPrincipal)));
}
LOG.info("Tokens added to credentials map.");
} catch (Exception e) {
@@ -72,7 +75,8 @@ public void populateCredentials(Map credentials,
}
@Override
- public void renew(Map credentials, Map topologyConf, final String topologyOwnerPrincipal) {
+ public void renew(Map credentials, Map topologyConf,
+ final String topologyOwnerPrincipal) {
doRenew(credentials, topologyConf, topologyOwnerPrincipal);
}
@@ -81,9 +85,11 @@ protected Set> getCredentials(Map cred
return HadoopCredentialUtil.getCredential(this, credentials, configKeys);
}
- protected void fillHadoopConfiguration(Map topologyConf, String configKey, Configuration configuration) {
+ protected void fillHadoopConfiguration(Map topologyConf, String configKey,
+ Configuration configuration) {
Map config = (Map) topologyConf.get(configKey);
- LOG.info("TopoConf {}, got config {}, for configKey {}", ConfigUtils.maskPasswords(topologyConf),
+ LOG.info("TopoConf {}, got config {}, for configKey {}", ConfigUtils
+ .maskPasswords(topologyConf),
ConfigUtils.maskPasswords(config), configKey);
if (config != null) {
List resourcesToLoad = new ArrayList<>();
@@ -118,11 +124,14 @@ protected void fillHadoopConfiguration(Map topologyConf, String configKey, Confi
*/
protected abstract String getConfigKeyString();
- protected abstract byte[] getHadoopCredentials(Map topologyConf, String configKey, String topologyOwnerPrincipal);
+ protected abstract byte[] getHadoopCredentials(Map topologyConf,
+ String configKey, String topologyOwnerPrincipal);
- protected abstract byte[] getHadoopCredentials(Map topologyConf, String topologyOwnerPrincipal);
+ protected abstract byte[] getHadoopCredentials(Map topologyConf,
+ String topologyOwnerPrincipal);
- protected abstract void doRenew(Map credentials, Map topologyConf, String topologyOwnerPrincipal);
+ protected abstract void doRenew(Map credentials, Map topologyConf, String topologyOwnerPrincipal);
protected List getConfigKeys(Map conf) {
String configKeyString = getConfigKeyString();
diff --git a/external/storm-autocreds/src/main/java/org/apache/storm/common/HadoopCredentialUtil.java b/external/storm-autocreds/src/main/java/org/apache/storm/common/HadoopCredentialUtil.java
index 8e7c64bc2f1..7dfb0c340b6 100644
--- a/external/storm-autocreds/src/main/java/org/apache/storm/common/HadoopCredentialUtil.java
+++ b/external/storm-autocreds/src/main/java/org/apache/storm/common/HadoopCredentialUtil.java
@@ -7,9 +7,9 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
+ *
Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
@@ -25,7 +25,6 @@
import java.util.Map;
import java.util.Set;
import javax.xml.bind.DatatypeConverter;
-
import org.apache.commons.math3.util.Pair;
import org.apache.hadoop.security.Credentials;
import org.slf4j.Logger;
diff --git a/external/storm-autocreds/src/main/java/org/apache/storm/hbase/security/AutoHBase.java b/external/storm-autocreds/src/main/java/org/apache/storm/hbase/security/AutoHBase.java
index 549f2d94f5f..2c9bb0197f8 100644
--- a/external/storm-autocreds/src/main/java/org/apache/storm/hbase/security/AutoHBase.java
+++ b/external/storm-autocreds/src/main/java/org/apache/storm/hbase/security/AutoHBase.java
@@ -7,9 +7,9 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
+ *
Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
@@ -21,7 +21,6 @@
import static org.apache.storm.hbase.security.HBaseSecurityUtil.HBASE_CREDENTIALS;
import java.util.Map;
-
import org.apache.storm.common.AbstractHadoopAutoCreds;
/**
diff --git a/external/storm-autocreds/src/main/java/org/apache/storm/hbase/security/AutoHBaseCommand.java b/external/storm-autocreds/src/main/java/org/apache/storm/hbase/security/AutoHBaseCommand.java
index 518f2130b03..815574b5b79 100644
--- a/external/storm-autocreds/src/main/java/org/apache/storm/hbase/security/AutoHBaseCommand.java
+++ b/external/storm-autocreds/src/main/java/org/apache/storm/hbase/security/AutoHBaseCommand.java
@@ -7,9 +7,9 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
+ *
Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
@@ -24,7 +24,6 @@
import java.util.HashMap;
import java.util.Map;
import javax.security.auth.Subject;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -50,7 +49,8 @@ public static void main(String[] args) throws Exception {
autoHBaseNimbus.prepare(conf);
Map creds = new HashMap<>();
- autoHBaseNimbus.populateCredentials(creds, conf, args[0]); //with realm e.g. storm@WITZEND.COM
+ autoHBaseNimbus.populateCredentials(creds, conf,
+ args[0]); // with realm e.g. storm@WITZEND.COM
LOG.info("Got HBase credentials" + autoHBase.getCredentials(creds));
Subject s = new Subject();
diff --git a/external/storm-autocreds/src/main/java/org/apache/storm/hbase/security/AutoHBaseNimbus.java b/external/storm-autocreds/src/main/java/org/apache/storm/hbase/security/AutoHBaseNimbus.java
index 84ce57e0844..6f102a64954 100644
--- a/external/storm-autocreds/src/main/java/org/apache/storm/hbase/security/AutoHBaseNimbus.java
+++ b/external/storm-autocreds/src/main/java/org/apache/storm/hbase/security/AutoHBaseNimbus.java
@@ -7,9 +7,9 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
+ *
Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
@@ -26,7 +26,6 @@
import java.io.ObjectOutputStream;
import java.net.InetAddress;
import java.util.Map;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.Connection;
@@ -61,37 +60,43 @@ protected String getConfigKeyString() {
@Override
public void shutdown() {
- //no op.
+ // no op.
}
@Override
- protected byte[] getHadoopCredentials(Map conf, String configKey, final String topologyOwnerPrincipal) {
+ protected byte[] getHadoopCredentials(Map conf, String configKey,
+ final String topologyOwnerPrincipal) {
Configuration configuration = getHadoopConfiguration(conf, configKey);
return getHadoopCredentials(conf, configuration, topologyOwnerPrincipal);
}
@Override
- protected byte[] getHadoopCredentials(Map conf, final String topologyOwnerPrincipal) {
+ protected byte[] getHadoopCredentials(Map conf,
+ final String topologyOwnerPrincipal) {
return getHadoopCredentials(conf, HBaseConfiguration.create(), topologyOwnerPrincipal);
}
@SuppressWarnings("unchecked")
- protected byte[] getHadoopCredentials(Map conf, Configuration hbaseConf, final String topologySubmitterUser) {
+ protected byte[] getHadoopCredentials(Map conf, Configuration hbaseConf,
+ final String topologySubmitterUser) {
try {
if (UserGroupInformation.isSecurityEnabled()) {
UserProvider provider = UserProvider.instantiate(hbaseConf);
- provider.login(HBASE_KEYTAB_FILE_KEY, HBASE_PRINCIPAL_KEY, InetAddress.getLocalHost().getCanonicalHostName());
+ provider.login(HBASE_KEYTAB_FILE_KEY, HBASE_PRINCIPAL_KEY, InetAddress
+ .getLocalHost().getCanonicalHostName());
LOG.info("Logged into Hbase as principal = " + hbaseConf.get(HBASE_PRINCIPAL_KEY));
UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
- final UserGroupInformation proxyUser = UserGroupInformation.createProxyUser(topologySubmitterUser, ugi);
+ final UserGroupInformation proxyUser = UserGroupInformation
+ .createProxyUser(topologySubmitterUser, ugi);
User user = User.create(proxyUser);
if (user.isHBaseSecurityEnabled(hbaseConf)) {
- final Connection connection = ConnectionFactory.createConnection(hbaseConf, user);
+ final Connection connection = ConnectionFactory.createConnection(hbaseConf,
+ user);
ClientTokenUtil.obtainAndCacheToken(connection, user);
LOG.info("Obtained HBase tokens, adding to user credentials.");
@@ -127,8 +132,9 @@ private Configuration getHadoopConfiguration(Map topoConf, Strin
}
@Override
- public void doRenew(Map credentials, Map topologyConf, final String topologySubmitterUser) {
- //HBASE tokens are not renewable so we always have to get new ones.
+ public void doRenew(Map credentials, Map topologyConf,
+ final String topologySubmitterUser) {
+ // HBASE tokens are not renewable so we always have to get new ones.
populateCredentials(credentials, topologyConf, topologySubmitterUser);
}
diff --git a/external/storm-autocreds/src/main/java/org/apache/storm/hbase/security/HBaseSecurityUtil.java b/external/storm-autocreds/src/main/java/org/apache/storm/hbase/security/HBaseSecurityUtil.java
index 1d8e5b865a8..323e3a70d84 100644
--- a/external/storm-autocreds/src/main/java/org/apache/storm/hbase/security/HBaseSecurityUtil.java
+++ b/external/storm-autocreds/src/main/java/org/apache/storm/hbase/security/HBaseSecurityUtil.java
@@ -7,9 +7,9 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
+ *
Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
@@ -24,7 +24,6 @@
import java.net.InetAddress;
import java.util.List;
import java.util.Map;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.security.UserProvider;
import org.apache.hadoop.security.UserGroupInformation;
@@ -52,14 +51,17 @@ public class HBaseSecurityUtil {
private HBaseSecurityUtil() {
}
- public static UserProvider login(Map conf, Configuration hbaseConfig) throws IOException {
- //Allowing keytab based login for backward compatibility.
+ public static UserProvider login(Map conf,
+ Configuration hbaseConfig) throws IOException {
+ // Allowing keytab based login for backward compatibility.
if (UserGroupInformation.isSecurityEnabled()) {
List autoCredentials = (List) conf.get(TOPOLOGY_AUTO_CREDENTIALS);
if ((autoCredentials == null)
- || (!autoCredentials.contains(AutoHBase.class.getName()) && !autoCredentials.contains(AutoTGT.class.getName()))) {
- LOG.info("Logging in using keytab as neither AutoHBase or AutoTGT is specified for " + TOPOLOGY_AUTO_CREDENTIALS);
- //insure that if keytab is used only one login per process executed
+ || (!autoCredentials.contains(AutoHBase.class.getName()) && !autoCredentials
+ .contains(AutoTGT.class.getName()))) {
+ LOG.info("Logging in using keytab as neither AutoHBase or AutoTGT is specified "
+ + "for " + TOPOLOGY_AUTO_CREDENTIALS);
+ // insure that if keytab is used only one login per process executed
if (legacyProvider == null) {
synchronized (HBaseSecurityUtil.class) {
if (legacyProvider == null) {
diff --git a/external/storm-autocreds/src/main/java/org/apache/storm/hdfs/security/AutoHDFS.java b/external/storm-autocreds/src/main/java/org/apache/storm/hdfs/security/AutoHDFS.java
index a5d1a03a335..b90fbdf8e49 100644
--- a/external/storm-autocreds/src/main/java/org/apache/storm/hdfs/security/AutoHDFS.java
+++ b/external/storm-autocreds/src/main/java/org/apache/storm/hdfs/security/AutoHDFS.java
@@ -7,9 +7,9 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
+ *
Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
@@ -21,7 +21,6 @@
import static org.apache.storm.hdfs.security.HdfsSecurityUtil.HDFS_CREDENTIALS;
import java.util.Map;
-
import org.apache.storm.common.AbstractHadoopAutoCreds;
/**
diff --git a/external/storm-autocreds/src/main/java/org/apache/storm/hdfs/security/AutoHDFSCommand.java b/external/storm-autocreds/src/main/java/org/apache/storm/hdfs/security/AutoHDFSCommand.java
index b8e4396a930..6e593e10c0f 100644
--- a/external/storm-autocreds/src/main/java/org/apache/storm/hdfs/security/AutoHDFSCommand.java
+++ b/external/storm-autocreds/src/main/java/org/apache/storm/hdfs/security/AutoHDFSCommand.java
@@ -7,9 +7,9 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
+ *
Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
@@ -24,7 +24,6 @@
import java.util.HashMap;
import java.util.Map;
import javax.security.auth.Subject;
-
import org.apache.storm.Config;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -42,7 +41,7 @@ private AutoHDFSCommand() {
@SuppressWarnings("unchecked")
public static void main(String[] args) throws Exception {
Map conf = new HashMap<>();
- conf.put(STORM_USER_NAME_KEY, args[1]); //with realm e.g. hdfs@WITZEND.COM
+ conf.put(STORM_USER_NAME_KEY, args[1]); // with realm e.g. hdfs@WITZEND.COM
conf.put(STORM_KEYTAB_FILE_KEY, args[2]); // /etc/security/keytabs/storm.keytab
AutoHDFS autoHdfs = new AutoHDFS();
diff --git a/external/storm-autocreds/src/main/java/org/apache/storm/hdfs/security/AutoHDFSNimbus.java b/external/storm-autocreds/src/main/java/org/apache/storm/hdfs/security/AutoHDFSNimbus.java
index 200ae4747d8..a91c46ea7fa 100644
--- a/external/storm-autocreds/src/main/java/org/apache/storm/hdfs/security/AutoHDFSNimbus.java
+++ b/external/storm-autocreds/src/main/java/org/apache/storm/hdfs/security/AutoHDFSNimbus.java
@@ -7,9 +7,9 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
+ *
Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
@@ -31,7 +31,6 @@
import java.util.Collection;
import java.util.List;
import java.util.Map;
-
import org.apache.commons.math3.util.Pair;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
@@ -71,22 +70,25 @@ protected String getConfigKeyString() {
@Override
public void shutdown() {
- //no op.
+ // no op.
}
@Override
- protected byte[] getHadoopCredentials(Map conf, String configKey, final String topologyOwnerPrincipal) {
+ protected byte[] getHadoopCredentials(Map conf, String configKey,
+ final String topologyOwnerPrincipal) {
Configuration configuration = getHadoopConfiguration(conf, configKey);
return getHadoopCredentials(conf, configuration, topologyOwnerPrincipal);
}
@Override
- protected byte[] getHadoopCredentials(Map conf, final String topologyOwnerPrincipal) {
+ protected byte[] getHadoopCredentials(Map conf,
+ final String topologyOwnerPrincipal) {
return getHadoopCredentials(conf, new Configuration(), topologyOwnerPrincipal);
}
@SuppressWarnings("unchecked")
- private byte[] getHadoopCredentials(Map conf, final Configuration configuration, final String topologySubmitterUser) {
+ private byte[] getHadoopCredentials(Map conf, final Configuration configuration,
+ final String topologySubmitterUser) {
try {
if (UserGroupInformation.isSecurityEnabled()) {
login(configuration);
@@ -97,7 +99,8 @@ private byte[] getHadoopCredentials(Map conf, final Configuratio
UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
- final UserGroupInformation proxyUser = UserGroupInformation.createProxyUser(topologySubmitterUser, ugi);
+ final UserGroupInformation proxyUser = UserGroupInformation
+ .createProxyUser(topologySubmitterUser, ugi);
Credentials creds = (Credentials) proxyUser.doAs(new PrivilegedAction() {
@Override
@@ -110,8 +113,10 @@ public Object run() {
configuration.set(STORM_USER_NAME_KEY, hdfsPrincipal);
}
- fileSystem.addDelegationTokens(configuration.get(STORM_USER_NAME_KEY), credential);
- LOG.info("Delegation tokens acquired for user {}", topologySubmitterUser);
+ fileSystem.addDelegationTokens(configuration.get(STORM_USER_NAME_KEY),
+ credential);
+ LOG.info("Delegation tokens acquired for user {}",
+ topologySubmitterUser);
return credential;
} catch (IOException e) {
throw new RuntimeException(e);
@@ -146,26 +151,30 @@ private Configuration getHadoopConfiguration(Map topoConf, Strin
* {@inheritDoc}
*/
@Override
- public void doRenew(Map credentials, Map topologyConf, final String topologyOwnerPrincipal) {
+ public void doRenew(Map credentials, Map topologyConf,
+ final String topologyOwnerPrincipal) {
List confKeys = getConfigKeys(topologyConf);
for (Pair cred : getCredentials(credentials, confKeys)) {
try {
Configuration configuration = getHadoopConfiguration(topologyConf, cred.getFirst());
- Collection> tokens = cred.getSecond().getAllTokens();
+ Collection> tokens = cred.getSecond()
+ .getAllTokens();
if (tokens != null && !tokens.isEmpty()) {
for (Token extends TokenIdentifier> token : tokens) {
- //We need to re-login some other thread might have logged into hadoop using
+ // We need to re-login some other thread might have logged into hadoop using
// their credentials (e.g. AutoHBase might be also part of nimbu auto creds)
login(configuration);
long expiration = token.renew(configuration);
- LOG.info("HDFS delegation token renewed, new expiration time {}", expiration);
+ LOG.info("HDFS delegation token renewed, new expiration time {}",
+ expiration);
}
} else {
LOG.debug("No tokens found for credentials, skipping renewal.");
}
} catch (Exception e) {
- LOG.warn("could not renew the credentials, one of the possible reason is tokens are beyond "
+ LOG.warn("could not renew the credentials, one of the possible reason is tokens "
+ + "are beyond "
+ "renewal period so attempting to get new tokens.",
e);
populateCredentials(credentials, topologyConf, topologyOwnerPrincipal);
diff --git a/external/storm-autocreds/src/main/java/org/apache/storm/hdfs/security/HdfsSecurityUtil.java b/external/storm-autocreds/src/main/java/org/apache/storm/hdfs/security/HdfsSecurityUtil.java
index c89b845273d..1737c77ea64 100644
--- a/external/storm-autocreds/src/main/java/org/apache/storm/hdfs/security/HdfsSecurityUtil.java
+++ b/external/storm-autocreds/src/main/java/org/apache/storm/hdfs/security/HdfsSecurityUtil.java
@@ -7,9 +7,9 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
+ *
Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
@@ -24,13 +24,10 @@
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
-
import org.apache.storm.security.auth.kerberos.AutoTGT;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -51,15 +48,20 @@ public final class HdfsSecurityUtil {
private HdfsSecurityUtil() {
}
- public static void login(Map conf, Configuration hdfsConfig) throws IOException {
- //If AutoHDFS is specified, do not attempt to login using keytabs, only kept for backward compatibility.
+ public static void login(Map conf,
+ Configuration hdfsConfig) throws IOException {
+ // If AutoHDFS is specified, do not attempt to login using keytabs, only kept for backward
+ // compatibility.
if (conf.get(TOPOLOGY_AUTO_CREDENTIALS) == null
- || (!(((List) conf.get(TOPOLOGY_AUTO_CREDENTIALS)).contains(AutoHDFS.class.getName()))
- && !(((List) conf.get(TOPOLOGY_AUTO_CREDENTIALS)).contains(AutoTGT.class.getName())))) {
+ || (!(((List) conf.get(TOPOLOGY_AUTO_CREDENTIALS)).contains(AutoHDFS.class
+ .getName()))
+ && !(((List) conf.get(TOPOLOGY_AUTO_CREDENTIALS)).contains(AutoTGT.class
+ .getName())))) {
if (UserGroupInformation.isSecurityEnabled()) {
// compareAndSet added because of https://issues.apache.org/jira/browse/STORM-1535
if (isLoggedIn.compareAndSet(false, true)) {
- LOG.info("Logging in using keytab as AutoHDFS is not specified for " + TOPOLOGY_AUTO_CREDENTIALS);
+ LOG.info("Logging in using keytab as AutoHDFS is not specified for "
+ + TOPOLOGY_AUTO_CREDENTIALS);
String keytab = (String) conf.get(STORM_KEYTAB_FILE_KEY);
if (keytab != null) {
hdfsConfig.set(STORM_KEYTAB_FILE_KEY, keytab);
diff --git a/external/storm-blobstore-migration/pom.xml b/external/storm-blobstore-migration/pom.xml
index ae3bfa3b8a3..9150aa0a220 100644
--- a/external/storm-blobstore-migration/pom.xml
+++ b/external/storm-blobstore-migration/pom.xml
@@ -146,6 +146,16 @@ limitations under the License.
+
+ org.openrewrite.maven
+ rewrite-maven-plugin
+
+
+
+ org.codehaus.gmavenplus
+ gmavenplus-plugin
+
+ org.apache.maven.pluginsmaven-checkstyle-plugin
diff --git a/external/storm-blobstore-migration/src/main/java/org/apache/storm/blobstore/ListHDFS.java b/external/storm-blobstore-migration/src/main/java/org/apache/storm/blobstore/ListHDFS.java
index cfa71312c24..78bf6e31642 100644
--- a/external/storm-blobstore-migration/src/main/java/org/apache/storm/blobstore/ListHDFS.java
+++ b/external/storm-blobstore-migration/src/main/java/org/apache/storm/blobstore/ListHDFS.java
@@ -7,9 +7,9 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
+ *
Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
@@ -19,9 +19,7 @@
package org.apache.storm.blobstore;
import java.util.Map;
-
import javax.security.auth.Subject;
-
import org.apache.storm.Config;
import org.apache.storm.blobstore.ClientBlobStore;
import org.apache.storm.hdfs.blobstore.HdfsBlobStore;
@@ -33,12 +31,14 @@ public class ListHDFS {
public static void main(String[] args) throws Exception {
if (args.length < 1) {
- System.out.println("Need at least 1 argument (hdfs_blobstore_path), but have " + Integer.toString(args.length));
+ System.out.println("Need at least 1 argument (hdfs_blobstore_path), but have " + Integer
+ .toString(args.length));
System.out.println("listHDFS ");
System.out.println("Lists blobs in HdfsBlobStore");
System.out.println("Example: listHDFS "
+ "'hdfs://some-hdfs-namenode:8080/srv/storm/my-storm-blobstore' "
- + "'stormUser/my-nimbus-host.example.com@STORM.EXAMPLE.COM' '/srv/my-keytab/stormUser.kt'");
+ + "'stormUser/my-nimbus-host.example.com@STORM.EXAMPLE.COM' "
+ + "'/srv/my-keytab/stormUser.kt'");
System.exit(1);
}
@@ -46,7 +46,8 @@ public static void main(String[] args) throws Exception {
String hdfsBlobstorePath = args[0];
hdfsConf.put(Config.BLOBSTORE_DIR, hdfsBlobstorePath);
- hdfsConf.put(Config.STORM_PRINCIPAL_TO_LOCAL_PLUGIN, "org.apache.storm.security.auth.DefaultPrincipalToLocal");
+ hdfsConf.put(Config.STORM_PRINCIPAL_TO_LOCAL_PLUGIN,
+ "org.apache.storm.security.auth.DefaultPrincipalToLocal");
if (args.length >= 2) {
System.out.println("SETTING HDFS PRINCIPAL!");
hdfsConf.put(Config.STORM_HDFS_LOGIN_PRINCIPAL, args[1]);
diff --git a/external/storm-blobstore-migration/src/main/java/org/apache/storm/blobstore/ListLocalFs.java b/external/storm-blobstore-migration/src/main/java/org/apache/storm/blobstore/ListLocalFs.java
index aefddb8869b..e3721b0e03b 100644
--- a/external/storm-blobstore-migration/src/main/java/org/apache/storm/blobstore/ListLocalFs.java
+++ b/external/storm-blobstore-migration/src/main/java/org/apache/storm/blobstore/ListLocalFs.java
@@ -7,9 +7,9 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
+ *
Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
@@ -19,9 +19,7 @@
package org.apache.storm.blobstore;
import java.util.Map;
-
import javax.security.auth.Subject;
-
import org.apache.storm.Config;
import org.apache.storm.blobstore.LocalFsBlobStore;
import org.apache.storm.nimbus.NimbusInfo;
@@ -41,7 +39,8 @@ public static void main(String[] args) throws Exception {
Map lfsConf = Utils.readStormConfig();
lfsConf.put(Config.BLOBSTORE_DIR, args[0]);
- lfsConf.put(Config.STORM_PRINCIPAL_TO_LOCAL_PLUGIN, "org.apache.storm.security.auth.DefaultPrincipalToLocal");
+ lfsConf.put(Config.STORM_PRINCIPAL_TO_LOCAL_PLUGIN,
+ "org.apache.storm.security.auth.DefaultPrincipalToLocal");
/* CREATE THE BLOBSTORE */
LocalFsBlobStore lfsBlobStore = new LocalFsBlobStore();
diff --git a/external/storm-blobstore-migration/src/main/java/org/apache/storm/blobstore/MigrateBlobs.java b/external/storm-blobstore-migration/src/main/java/org/apache/storm/blobstore/MigrateBlobs.java
index e7a3581637d..9ea2435f105 100644
--- a/external/storm-blobstore-migration/src/main/java/org/apache/storm/blobstore/MigrateBlobs.java
+++ b/external/storm-blobstore-migration/src/main/java/org/apache/storm/blobstore/MigrateBlobs.java
@@ -7,9 +7,9 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
+ *
Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
@@ -21,10 +21,8 @@
import java.io.IOException;
import java.io.InputStream;
import java.util.Map;
-
import javax.security.auth.Subject;
import javax.security.auth.login.LoginContext;
-
import org.apache.storm.Config;
import org.apache.storm.blobstore.BlobStore;
import org.apache.storm.blobstore.LocalFsBlobStore;
@@ -39,7 +37,8 @@
public class MigrateBlobs {
- protected static void deleteAllBlobStoreKeys(BlobStore bs, Subject who) throws AuthorizationException, KeyNotFoundException {
+ protected static void deleteAllBlobStoreKeys(BlobStore bs,
+ Subject who) throws AuthorizationException, KeyNotFoundException {
Iterable hdfsKeys = () -> bs.listKeys();
for (String key : hdfsKeys) {
System.out.println(key);
@@ -63,25 +62,29 @@ protected static void copyBlobStoreKeys(BlobStore bsFrom,
System.out.println("DONE CREATING BLOB " + key);
}
}
-
-
+
public static void main(String[] args) throws Exception {
Map hdfsConf = Utils.readStormConfig();
if (args.length < 2) {
- System.out.println("Need at least 2 arguments, but have " + Integer.toString(args.length));
- System.out.println("migrate ");
+ System.out.println("Need at least 2 arguments, but have " + Integer
+ .toString(args.length));
+ System.out
+ .println("migrate "
+ + "");
System.out.println("Migrates blobs from LocalFsBlobStore to HdfsBlobStore");
System.out.println("Example: migrate '/srv/storm' "
+ "'hdfs://some-hdfs-namenode:8080/srv/storm/my-storm-blobstore' "
- + "'stormUser/my-nimbus-host.example.com@STORM.EXAMPLE.COM' '/srv/my-keytab/stormUser.kt'");
+ + "'stormUser/my-nimbus-host.example.com@STORM.EXAMPLE.COM' "
+ + "'/srv/my-keytab/stormUser.kt'");
System.exit(1);
}
String hdfsBlobstorePath = args[1];
hdfsConf.put(Config.BLOBSTORE_DIR, hdfsBlobstorePath);
- hdfsConf.put(Config.STORM_PRINCIPAL_TO_LOCAL_PLUGIN, "org.apache.storm.security.auth.DefaultPrincipalToLocal");
+ hdfsConf.put(Config.STORM_PRINCIPAL_TO_LOCAL_PLUGIN,
+ "org.apache.storm.security.auth.DefaultPrincipalToLocal");
if (args.length >= 3) {
System.out.println("SETTING HDFS PRINCIPAL!");
hdfsConf.put(Config.STORM_HDFS_LOGIN_PRINCIPAL, args[2]);
@@ -95,7 +98,8 @@ public static void main(String[] args) throws Exception {
Map lfsConf = Utils.readStormConfig();
String localBlobstoreDir = args[0];
lfsConf.put(Config.BLOBSTORE_DIR, localBlobstoreDir);
- lfsConf.put(Config.STORM_PRINCIPAL_TO_LOCAL_PLUGIN, "org.apache.storm.security.auth.DefaultPrincipalToLocal");
+ lfsConf.put(Config.STORM_PRINCIPAL_TO_LOCAL_PLUGIN,
+ "org.apache.storm.security.auth.DefaultPrincipalToLocal");
/* CREATE THE BLOBSTORES */
@@ -117,7 +121,9 @@ public static void main(String[] args) throws Exception {
System.out.println("Done listing HDFS blobstore keys.");
- System.out.println("Going to delete everything in HDFS, then copy all local blobs to HDFS. Continue? [Y/n]");
+ System.out
+ .println("Going to delete everything in HDFS, then copy all local blobs to HDFS. "
+ + "Continue? [Y/n]");
String resp = System.console().readLine().toLowerCase().trim();
if (!(resp.equals("y") || resp.equals(""))) {
System.out.println("Not copying blobs. Exiting. [" + resp.toLowerCase().trim() + "]");
diff --git a/external/storm-blobstore-migration/src/main/java/org/apache/storm/blobstore/MigratorMain.java b/external/storm-blobstore-migration/src/main/java/org/apache/storm/blobstore/MigratorMain.java
index 03d163aad27..678bb420f0f 100644
--- a/external/storm-blobstore-migration/src/main/java/org/apache/storm/blobstore/MigratorMain.java
+++ b/external/storm-blobstore-migration/src/main/java/org/apache/storm/blobstore/MigratorMain.java
@@ -7,9 +7,9 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
+ *
Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
@@ -19,7 +19,6 @@
package org.apache.storm.blobstore;
import java.util.Arrays;
-
import javax.security.auth.Subject;
public class MigratorMain {
diff --git a/external/storm-hdfs-blobstore/pom.xml b/external/storm-hdfs-blobstore/pom.xml
index a33bbca96e6..fe31d7fabe3 100644
--- a/external/storm-hdfs-blobstore/pom.xml
+++ b/external/storm-hdfs-blobstore/pom.xml
@@ -153,6 +153,16 @@
+
+ org.openrewrite.maven
+ rewrite-maven-plugin
+
+
+
+ org.codehaus.gmavenplus
+ gmavenplus-plugin
+
+ org.apache.maven.pluginsmaven-checkstyle-plugin
diff --git a/external/storm-hdfs-blobstore/src/main/java/org/apache/storm/hdfs/blobstore/HdfsBlobStore.java b/external/storm-hdfs-blobstore/src/main/java/org/apache/storm/hdfs/blobstore/HdfsBlobStore.java
index 7d957186a21..dea33e29175 100644
--- a/external/storm-hdfs-blobstore/src/main/java/org/apache/storm/hdfs/blobstore/HdfsBlobStore.java
+++ b/external/storm-hdfs-blobstore/src/main/java/org/apache/storm/hdfs/blobstore/HdfsBlobStore.java
@@ -7,9 +7,9 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
+ *
Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
@@ -59,18 +59,24 @@
* Note that this provides an api for having HDFS be the backing store for the blobstore,
* it is not a service/daemon.
*
- *
We currently have NIMBUS_ADMINS and SUPERVISOR_ADMINS configuration. NIMBUS_ADMINS are given READ, WRITE and ADMIN
- * access whereas the SUPERVISOR_ADMINS are given READ access in order to read and download the blobs form the nimbus.
+ *
We currently have NIMBUS_ADMINS and SUPERVISOR_ADMINS configuration. NIMBUS_ADMINS are given
+ * READ, WRITE and ADMIN
+ * access whereas the SUPERVISOR_ADMINS are given READ access in order to read and download the
+ * blobs form the nimbus.
*
- *
The ACLs for the blob store are validated against whether the subject is a NIMBUS_ADMIN, SUPERVISOR_ADMIN or USER
+ *
The ACLs for the blob store are validated against whether the subject is a NIMBUS_ADMIN,
+ * SUPERVISOR_ADMIN or USER
* who has read, write or admin privileges in order to perform respective operations on the blob.
*
*
For hdfs blob store
- * 1. The USER interacts with nimbus to upload and access blobs through NimbusBlobStore Client API. Here, unlike
+ * 1. The USER interacts with nimbus to upload and access blobs through NimbusBlobStore Client API.
+ * Here, unlike
* local blob store which stores the blobs locally, the nimbus talks to HDFS to upload the blobs.
* 2. The USER sets the ACLs, and the blob access is validated against these ACLs.
- * 3. The SUPERVISOR interacts with nimbus through HdfsClientBlobStore to download the blobs. Here, unlike local
- * blob store the supervisor interacts with HDFS directly to download the blobs. The call to HdfsBlobStore is made as a "null"
+ * 3. The SUPERVISOR interacts with nimbus through HdfsClientBlobStore to download the blobs. Here,
+ * unlike local
+ * blob store the supervisor interacts with HDFS directly to download the blobs. The call to
+ * HdfsBlobStore is made as a "null"
* subject. The blobstore gets the hadoop user and validates permissions for the supervisor.
*/
public class HdfsBlobStore extends BlobStore {
@@ -82,8 +88,10 @@ public class HdfsBlobStore extends BlobStore {
private HdfsBlobStoreImpl hbs;
private Subject localSubject;
private Map conf;
- private Cache cacheMetas = CacheBuilder.newBuilder().expireAfterWrite(10, TimeUnit.MINUTES).build();
- private Cache cachedReplicationCount = CacheBuilder.newBuilder().expireAfterWrite(10, TimeUnit.MINUTES).build();
+ private Cache cacheMetas = CacheBuilder.newBuilder()
+ .expireAfterWrite(10, TimeUnit.MINUTES).build();
+ private Cache cachedReplicationCount = CacheBuilder.newBuilder()
+ .expireAfterWrite(10, TimeUnit.MINUTES).build();
/**
* If who is null then we want to use the user hadoop says we are.
@@ -98,7 +106,8 @@ private Subject checkAndGetSubject(Subject who) {
}
@Override
- public void prepare(Map conf, String overrideBase, NimbusInfo nimbusInfo, ILeaderElector leaderElector) {
+ public void prepare(Map conf, String overrideBase, NimbusInfo nimbusInfo,
+ ILeaderElector leaderElector) {
this.conf = conf;
prepareInternal(conf, overrideBase, null);
}
@@ -107,7 +116,8 @@ public void prepare(Map conf, String overrideBase, NimbusInfo ni
* Allow a Hadoop Configuration to be passed for testing. If it's null then the hadoop configs
* must be in your classpath.
*/
- protected void prepareInternal(Map conf, String overrideBase, Configuration hadoopConf) {
+ protected void prepareInternal(Map conf, String overrideBase,
+ Configuration hadoopConf) {
this.conf = conf;
if (overrideBase == null) {
overrideBase = (String) conf.get(Config.BLOBSTORE_DIR);
@@ -117,7 +127,7 @@ protected void prepareInternal(Map conf, String overrideBase, Co
}
LOG.debug("directory is: {}", overrideBase);
- //Login to hdfs
+ // Login to hdfs
localSubject = HadoopLoginUtil.loginHadoop(conf);
aclHandler = new BlobStoreAclHandler(conf);
@@ -166,7 +176,7 @@ public AtomicOutputStream createBlob(String key, SettableBlobMeta meta, Subject
try {
outputStream.cancel();
} catch (IOException e) {
- //Ignored
+ // Ignored
}
}
}
@@ -205,7 +215,8 @@ private SettableBlobMeta getStoredBlobMeta(String key) throws KeyNotFoundExcepti
}
in.close();
in = null;
- SettableBlobMeta blobMeta = Utils.thriftDeserialize(SettableBlobMeta.class, out.toByteArray());
+ SettableBlobMeta blobMeta = Utils.thriftDeserialize(SettableBlobMeta.class, out
+ .toByteArray());
return blobMeta;
} catch (IOException e) {
throw new RuntimeException(e);
@@ -214,7 +225,7 @@ private SettableBlobMeta getStoredBlobMeta(String key) throws KeyNotFoundExcepti
try {
in.close();
} catch (IOException e) {
- //Ignored
+ // Ignored
}
}
}
@@ -324,11 +335,12 @@ public Iterator listKeys() {
@Override
public void shutdown() {
- //Empty
+ // Empty
}
@Override
- public int getBlobReplication(String key, Subject who) throws AuthorizationException, KeyNotFoundException {
+ public int getBlobReplication(String key,
+ Subject who) throws AuthorizationException, KeyNotFoundException {
who = checkAndGetSubject(who);
validateKey(key);
SettableBlobMeta meta = extractBlobMeta(key);
@@ -361,7 +373,8 @@ private SettableBlobMeta extractBlobMeta(String key) throws KeyNotFoundException
}
@Override
- public int updateBlobReplication(String key, int replication, Subject who) throws AuthorizationException, KeyNotFoundException {
+ public int updateBlobReplication(String key, int replication,
+ Subject who) throws AuthorizationException, KeyNotFoundException {
who = checkAndGetSubject(who);
validateKey(key);
SettableBlobMeta meta = extractBlobMeta(key);
@@ -395,7 +408,7 @@ public void writeMetadata(String key, SettableBlobMeta meta)
try {
outputStream.cancel();
} catch (IOException e) {
- //Ignored
+ // Ignored
}
}
}
diff --git a/external/storm-hdfs-blobstore/src/main/java/org/apache/storm/hdfs/blobstore/HdfsBlobStoreFile.java b/external/storm-hdfs-blobstore/src/main/java/org/apache/storm/hdfs/blobstore/HdfsBlobStoreFile.java
index f124cdfd0fe..737f8ea23eb 100644
--- a/external/storm-hdfs-blobstore/src/main/java/org/apache/storm/hdfs/blobstore/HdfsBlobStoreFile.java
+++ b/external/storm-hdfs-blobstore/src/main/java/org/apache/storm/hdfs/blobstore/HdfsBlobStoreFile.java
@@ -7,9 +7,9 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
+ *
Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
@@ -22,7 +22,6 @@
import java.io.InputStream;
import java.io.OutputStream;
import java.util.regex.Matcher;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileSystem;
@@ -56,7 +55,8 @@ public HdfsBlobStoreFile(Path base, String name, Configuration hconf) {
} else {
Matcher m = TMP_NAME_PATTERN.matcher(name);
if (!m.matches()) {
- throw new IllegalArgumentException("File name does not match '" + name + "' !~ " + TMP_NAME_PATTERN);
+ throw new IllegalArgumentException("File name does not match '" + name + "' !~ "
+ + TMP_NAME_PATTERN);
}
isTmp = true;
}
@@ -136,14 +136,16 @@ public OutputStream getOutputStream() throws IOException {
fileSystem.setPermission(path, fileperms);
fileSystem.setReplication(path, (short) this.getMetadata().get_replication_factor());
} catch (IOException e) {
- //Try to create the parent directory, may not work
+ // Try to create the parent directory, may not work
FsPermission dirperms = new FsPermission(HdfsBlobStoreImpl.BLOBSTORE_DIR_PERMISSION);
if (!fileSystem.mkdirs(path.getParent(), dirperms)) {
LOG.warn("error creating parent dir: " + path.getParent());
}
if (!fileSystem.getFileStatus(path.getParent()).getPermission().equals(dirperms)) {
- LOG.warn("Directory {} created with unexpected permission {}.Set permission {} for this directory.",
- path.getParent(), fileSystem.getFileStatus(path.getParent()).getPermission(), dirperms);
+ LOG.warn("Directory {} created with unexpected permission {}.Set permission {} "
+ + "for this directory.",
+ path.getParent(), fileSystem.getFileStatus(path.getParent())
+ .getPermission(), dirperms);
fileSystem.setPermission(path.getParent(), dirperms);
}
out = fileSystem.create(path, (short) this.getMetadata().get_replication_factor());
diff --git a/external/storm-hdfs-blobstore/src/main/java/org/apache/storm/hdfs/blobstore/HdfsBlobStoreImpl.java b/external/storm-hdfs-blobstore/src/main/java/org/apache/storm/hdfs/blobstore/HdfsBlobStoreImpl.java
index 455cdd78569..c9f40fc69d3 100644
--- a/external/storm-hdfs-blobstore/src/main/java/org/apache/storm/hdfs/blobstore/HdfsBlobStoreImpl.java
+++ b/external/storm-hdfs-blobstore/src/main/java/org/apache/storm/hdfs/blobstore/HdfsBlobStoreImpl.java
@@ -7,9 +7,9 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
+ *
Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
@@ -29,7 +29,6 @@
import java.util.NoSuchElementException;
import java.util.Timer;
import java.util.TimerTask;
-
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
@@ -45,7 +44,6 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
/**
* HDFS blob store impl.
*/
@@ -144,7 +142,8 @@ public HdfsBlobStoreImpl(Path path, Map conf,
FsPermission perms = new FsPermission(BLOBSTORE_DIR_PERMISSION);
boolean success = fileSystem.mkdirs(fullPath, perms);
if (!fileSystem.getFileStatus(fullPath).getPermission().equals(perms)) {
- LOG.warn("Directory {} created with unexpected permission {}.Set permission {} for this directory.",
+ LOG.warn("Directory {} created with unexpected permission {}.Set permission {} "
+ + "for this directory.",
fullPath, fileSystem.getFileStatus(fullPath).getPermission(), perms);
fileSystem.setPermission(fullPath, perms);
}
@@ -189,7 +188,7 @@ protected Iterator listKeys(Path path) throws IOException {
try {
ret.add(sub.getPath().getName().toString());
} catch (IllegalArgumentException e) {
- //Ignored the file did not match
+ // Ignored the file did not match
LOG.debug("Found an unexpected file in {} {}", path, sub.getPath().getName());
}
}
@@ -269,7 +268,7 @@ public void fullCleanup(long age) throws IOException {
Path keyDir = getKeyDir(key);
Iterator i = listBlobStoreFiles(keyDir);
if (!i.hasNext()) {
- //The dir is empty, so try to delete it, may fail, but that is OK
+ // The dir is empty, so try to delete it, may fail, but that is OK
try {
fileSystem.delete(keyDir, true);
} catch (Exception e) {
@@ -293,10 +292,11 @@ protected Iterator listBlobStoreFiles(Path path) throws IOExcepti
if (files != null) {
for (FileStatus sub : files) {
try {
- ret.add(new HdfsBlobStoreFile(sub.getPath().getParent(), sub.getPath().getName(),
+ ret.add(new HdfsBlobStoreFile(sub.getPath().getParent(), sub.getPath()
+ .getName(),
hadoopConf));
} catch (IllegalArgumentException e) {
- //Ignored the file did not match
+ // Ignored the file did not match
LOG.warn("Found an unexpected file in {} {}", path, sub.getPath().getName());
}
}
@@ -359,14 +359,17 @@ public synchronized void updateLastBlobUpdateTime() throws IOException {
Long timestamp = Time.currentTimeMillis();
Path updateTimeFile = new Path(fullPath, BLOBSTORE_UPDATE_TIME_FILE);
FSDataOutputStream fsDataOutputStream = fileSystem.create(updateTimeFile, true);
- BufferedWriter bufferedWriter = new BufferedWriter(new OutputStreamWriter(fsDataOutputStream, StandardCharsets.UTF_8));
+ BufferedWriter bufferedWriter =
+ new BufferedWriter(new OutputStreamWriter(fsDataOutputStream,
+ StandardCharsets.UTF_8));
bufferedWriter.write(timestamp.toString());
bufferedWriter.close();
LOG.debug("Updated blobstore update time of {} to {}", updateTimeFile, timestamp);
}
/**
- * Validates that the last updated blob time of the blobstore is up to date with the current existing blobs.
+ * Validates that the last updated blob time of the blobstore is up to date with the current
+ * existing blobs.
*
* @throws IOException on any error
*/
diff --git a/external/storm-hdfs-blobstore/src/main/java/org/apache/storm/hdfs/blobstore/HdfsClientBlobStore.java b/external/storm-hdfs-blobstore/src/main/java/org/apache/storm/hdfs/blobstore/HdfsClientBlobStore.java
index fbdc1866100..12eb7e4b5a4 100644
--- a/external/storm-hdfs-blobstore/src/main/java/org/apache/storm/hdfs/blobstore/HdfsClientBlobStore.java
+++ b/external/storm-hdfs-blobstore/src/main/java/org/apache/storm/hdfs/blobstore/HdfsClientBlobStore.java
@@ -7,9 +7,9 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
+ *
Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
@@ -21,7 +21,6 @@
import java.io.IOException;
import java.util.Iterator;
import java.util.Map;
-
import org.apache.storm.blobstore.AtomicOutputStream;
import org.apache.storm.blobstore.ClientBlobStore;
import org.apache.storm.blobstore.InputStreamWithMeta;
@@ -107,7 +106,8 @@ public int getBlobReplication(String key) throws AuthorizationException, KeyNotF
}
@Override
- public int updateBlobReplication(String key, int replication) throws AuthorizationException, KeyNotFoundException {
+ public int updateBlobReplication(String key,
+ int replication) throws AuthorizationException, KeyNotFoundException {
return blobStore.updateBlobReplication(key, replication, null);
}
diff --git a/external/storm-hdfs-blobstore/src/test/java/org/apache/storm/hdfs/blobstore/BlobStoreTest.java b/external/storm-hdfs-blobstore/src/test/java/org/apache/storm/hdfs/blobstore/BlobStoreTest.java
index 206e6518a0c..c734368ea18 100644
--- a/external/storm-hdfs-blobstore/src/test/java/org/apache/storm/hdfs/blobstore/BlobStoreTest.java
+++ b/external/storm-hdfs-blobstore/src/test/java/org/apache/storm/hdfs/blobstore/BlobStoreTest.java
@@ -18,7 +18,21 @@
*/
package org.apache.storm.hdfs.blobstore;
-import org.apache.storm.hdfs.testing.MiniDFSClusterExtension;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import javax.security.auth.Subject;
import org.apache.storm.Config;
import org.apache.storm.blobstore.AtomicOutputStream;
import org.apache.storm.blobstore.BlobStore;
@@ -28,25 +42,10 @@
import org.apache.storm.generated.AuthorizationException;
import org.apache.storm.generated.KeyNotFoundException;
import org.apache.storm.generated.SettableBlobMeta;
+import org.apache.storm.hdfs.testing.MiniDFSClusterExtension;
import org.apache.storm.security.auth.FixedGroupsMapping;
import org.apache.storm.security.auth.NimbusPrincipal;
import org.apache.storm.security.auth.SingleUserPrincipal;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.security.auth.Subject;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import static org.junit.jupiter.api.Assertions.*;
-
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -54,11 +53,14 @@
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.EnumSource;
import org.junit.jupiter.params.provider.ValueSource;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class BlobStoreTest {
@RegisterExtension
- public static final MiniDFSClusterExtension DFS_CLUSTER_EXTENSION = new MiniDFSClusterExtension();
+ public static final MiniDFSClusterExtension DFS_CLUSTER_EXTENSION =
+ new MiniDFSClusterExtension();
private static final Logger LOG = LoggerFactory.getLogger(BlobStoreTest.class);
private static final Map CONF = new HashMap<>();
@@ -88,19 +90,21 @@ public static void initializeConfigs() {
// Now create a params map to put it in to our conf
Map paramMap = new HashMap<>();
paramMap.put(FixedGroupsMapping.STORM_FIXED_GROUP_MAPPING, groupsMapping);
- CONF.put(Config.STORM_GROUP_MAPPING_SERVICE_PROVIDER_PLUGIN, "org.apache.storm.security.auth.FixedGroupsMapping");
+ CONF.put(Config.STORM_GROUP_MAPPING_SERVICE_PROVIDER_PLUGIN,
+ "org.apache.storm.security.auth.FixedGroupsMapping");
CONF.put(Config.STORM_GROUP_MAPPING_SERVICE_PARAMS, paramMap);
CONF.put(Config.NIMBUS_SUPERVISOR_USERS, "supervisor");
}
- //Gets Nimbus Subject with NimbusPrincipal set on it
+ // Gets Nimbus Subject with NimbusPrincipal set on it
public static Subject getNimbusSubject() {
Subject nimbus = new Subject();
nimbus.getPrincipals().add(new NimbusPrincipal());
return nimbus;
}
- // Overloading the assertStoreHasExactly method accommodate Subject in order to check for authorization
+ // Overloading the assertStoreHasExactly method accommodate Subject in order to check for
+ // authorization
public static void assertStoreHasExactly(BlobStore store, Subject who, String... keys) {
Set expected = new HashSet<>(Arrays.asList(keys));
Set found = new HashSet<>();
@@ -121,8 +125,10 @@ public static void assertStoreHasExactly(BlobStore store, String... keys) {
assertStoreHasExactly(store, null, keys);
}
- // Overloading the readInt method accommodate Subject in order to check for authorization (security turned on)
- public static int readInt(BlobStore store, Subject who, String key) throws IOException, KeyNotFoundException, AuthorizationException {
+ // Overloading the readInt method accommodate Subject in order to check for authorization
+ // (security turned on)
+ public static int readInt(BlobStore store, Subject who,
+ String key) throws IOException, KeyNotFoundException, AuthorizationException {
try (InputStream in = store.getBlob(key, who)) {
return in.read();
}
@@ -147,10 +153,12 @@ public void readAssertEqualsWithAuth(BlobStore store, Subject who, String key, i
private AutoCloseableBlobStoreContainer initHdfs(String dirName) {
Map conf = new HashMap<>();
conf.put(Config.BLOBSTORE_DIR, dirName);
- conf.put(Config.STORM_PRINCIPAL_TO_LOCAL_PLUGIN, "org.apache.storm.security.auth.DefaultPrincipalToLocal");
+ conf.put(Config.STORM_PRINCIPAL_TO_LOCAL_PLUGIN,
+ "org.apache.storm.security.auth.DefaultPrincipalToLocal");
conf.put(Config.STORM_BLOBSTORE_REPLICATION_FACTOR, 3);
HdfsBlobStore store = new HdfsBlobStore();
- store.prepareInternal(conf, null, DFS_CLUSTER_EXTENSION.getDfscluster().getConfiguration(0));
+ store.prepareInternal(conf, null, DFS_CLUSTER_EXTENSION.getDfscluster()
+ .getConfiguration(0));
return new AutoCloseableBlobStoreContainer(store);
}
@@ -203,10 +211,11 @@ public void testReplication(String path, BlobStore store)
out.write(1);
}
assertStoreHasExactly(store, "test");
- assertEquals(store.getBlobReplication("test", null), 4, "Blobstore replication not matching");
+ assertEquals(store.getBlobReplication("test", null), 4,
+ "Blobstore replication not matching");
store.deleteBlob("test", null);
- //Test for replication with NIMBUS as user
+ // Test for replication with NIMBUS as user
Subject admin = getSubject("admin");
metadata = new SettableBlobMeta(BlobStoreAclHandler.DEFAULT);
metadata.set_replication_factor(4);
@@ -214,12 +223,14 @@ public void testReplication(String path, BlobStore store)
out.write(1);
}
assertStoreHasExactly(store, "test");
- assertEquals(store.getBlobReplication("test", admin), 4, "Blobstore replication not matching");
+ assertEquals(store.getBlobReplication("test", admin), 4,
+ "Blobstore replication not matching");
store.updateBlobReplication("test", 5, admin);
- assertEquals(store.getBlobReplication("test", admin), 5, "Blobstore replication not matching");
+ assertEquals(store.getBlobReplication("test", admin), 5,
+ "Blobstore replication not matching");
store.deleteBlob("test", admin);
- //Test for replication using SUPERVISOR access
+ // Test for replication using SUPERVISOR access
Subject supervisor = getSubject("supervisor");
metadata = new SettableBlobMeta(BlobStoreAclHandler.DEFAULT);
metadata.set_replication_factor(4);
@@ -227,9 +238,11 @@ public void testReplication(String path, BlobStore store)
out.write(1);
}
assertStoreHasExactly(store, "test");
- assertEquals(store.getBlobReplication("test", supervisor), 4, "Blobstore replication not matching");
+ assertEquals(store.getBlobReplication("test", supervisor), 4,
+ "Blobstore replication not matching");
store.updateBlobReplication("test", 5, supervisor);
- assertEquals(store.getBlobReplication("test", supervisor), 5, "Blobstore replication not matching");
+ assertEquals(store.getBlobReplication("test", supervisor), 5,
+ "Blobstore replication not matching");
store.deleteBlob("test", supervisor);
Subject adminsGroupsUser = getSubject("adminsGroupsUser");
@@ -239,12 +252,14 @@ public void testReplication(String path, BlobStore store)
out.write(1);
}
assertStoreHasExactly(store, "test");
- assertEquals(store.getBlobReplication("test", adminsGroupsUser), 4, "Blobstore replication not matching");
+ assertEquals(store.getBlobReplication("test", adminsGroupsUser), 4,
+ "Blobstore replication not matching");
store.updateBlobReplication("test", 5, adminsGroupsUser);
- assertEquals(store.getBlobReplication("test", adminsGroupsUser), 5, "Blobstore replication not matching");
+ assertEquals(store.getBlobReplication("test", adminsGroupsUser), 5,
+ "Blobstore replication not matching");
store.deleteBlob("test", adminsGroupsUser);
- //Test for a user having read or write or admin access to read replication for a blob
+ // Test for a user having read or write or admin access to read replication for a blob
String createSubject = "createSubject";
String writeSubject = "writeSubject";
String adminSubject = "adminSubject";
@@ -261,12 +276,14 @@ public void testReplication(String path, BlobStore store)
}
assertStoreHasExactly(store, "test");
who = getSubject(writeSubject);
- assertEquals(store.getBlobReplication("test", who), 4, "Blobstore replication not matching");
+ assertEquals(store.getBlobReplication("test", who), 4,
+ "Blobstore replication not matching");
- //Test for a user having WRITE or ADMIN privileges to change replication of a blob
+ // Test for a user having WRITE or ADMIN privileges to change replication of a blob
who = getSubject(adminSubject);
store.updateBlobReplication("test", 5, who);
- assertEquals(store.getBlobReplication("test", who), 5, "Blobstore replication not matching");
+ assertEquals(store.getBlobReplication("test", who), 5,
+ "Blobstore replication not matching");
store.deleteBlob("test", getSubject(createSubject));
}
@@ -278,13 +295,13 @@ public static Subject getSubject(String name) {
}
enum AuthenticationTestSubject {
- //Nimbus Admin
+ // Nimbus Admin
ADMIN(getSubject("admin")),
- //Nimbus groups admin
+ // Nimbus groups admin
ADMIN_GROUPS_USER(getSubject("adminGroupsUser")),
- //Supervisor admin
+ // Supervisor admin
SUPERVISOR(getSubject("supervisor")),
- //Nimbus itself
+ // Nimbus itself
NIMBUS(getNimbusSubject());
private final Subject subject;
@@ -297,7 +314,8 @@ enum AuthenticationTestSubject {
@ParameterizedTest
@EnumSource(value = AuthenticationTestSubject.class)
void testWithAuthentication(AuthenticationTestSubject testSubject) throws Exception {
- try (AutoCloseableBlobStoreContainer container = initHdfs("/storm/blobstore-auth-" + testSubject.name())) {
+ try (AutoCloseableBlobStoreContainer container = initHdfs("/storm/blobstore-auth-"
+ + testSubject.name())) {
BlobStore store = container.blobStore;
assertStoreHasExactly(store);
SettableBlobMeta metadata = new SettableBlobMeta(BlobStoreAclHandler.DEFAULT);
@@ -312,26 +330,32 @@ void testWithAuthentication(AuthenticationTestSubject testSubject) throws Except
@ParameterizedTest
@ValueSource(booleans = {true, false})
void testWithAuthenticationDummy(boolean securityEnabled) throws Exception {
- try (AutoCloseableBlobStoreContainer container = initHdfs("/storm/blobstore-auth-dummy-sec-" + securityEnabled)) {
+ try (AutoCloseableBlobStoreContainer container =
+ initHdfs("/storm/blobstore-auth-dummy-sec-" + securityEnabled)) {
BlobStore store = container.blobStore;
Subject who = getSubject("test_subject");
assertStoreHasExactly(store);
// Tests for case when subject != null (security turned on) and
// acls for the blob are set to WORLD_EVERYTHING
- SettableBlobMeta metadata = new SettableBlobMeta(securityEnabled ? BlobStoreAclHandler.DEFAULT : BlobStoreAclHandler.WORLD_EVERYTHING);
+ SettableBlobMeta metadata = new SettableBlobMeta(securityEnabled
+ ? BlobStoreAclHandler.DEFAULT : BlobStoreAclHandler.WORLD_EVERYTHING);
try (AtomicOutputStream out = store.createBlob("test", metadata, who)) {
out.write(1);
}
assertStoreHasExactly(store, "test");
if (securityEnabled) {
- // Testing whether acls are set to WORLD_EVERYTHING. Here the acl should not contain WORLD_EVERYTHING because
- // the subject is neither null nor empty. The ACL should however contain USER_EVERYTHING as user needs to have
+ // Testing whether acls are set to WORLD_EVERYTHING. Here the acl should not contain
+ // WORLD_EVERYTHING because
+ // the subject is neither null nor empty. The ACL should however contain
+ // USER_EVERYTHING as user needs to have
// complete access to the blob
- assertFalse(metadata.toString().contains("AccessControl(type:OTHER, access:7)"), "ACL contains WORLD_EVERYTHING");
+ assertFalse(metadata.toString().contains("AccessControl(type:OTHER, access:7)"),
+ "ACL contains WORLD_EVERYTHING");
} else {
// Testing whether acls are set to WORLD_EVERYTHING
- assertTrue(metadata.toString().contains("AccessControl(type:OTHER, access:7)"), "ACL does not contain WORLD_EVERYTHING");
+ assertTrue(metadata.toString().contains("AccessControl(type:OTHER, access:7)"),
+ "ACL does not contain WORLD_EVERYTHING");
}
readAssertEqualsWithAuth(store, who, "test", 1);
@@ -377,21 +401,24 @@ void testWithAuthenticationUpdate() throws Exception {
@ParameterizedTest
@ValueSource(booleans = {true, false})
void testWithAuthenticationNoPrincipal(boolean securityEnabled) throws Exception {
- try (AutoCloseableBlobStoreContainer container = initHdfs("/storm/blobstore-auth-no-principal-sec-" + securityEnabled)) {
+ try (AutoCloseableBlobStoreContainer container =
+ initHdfs("/storm/blobstore-auth-no-principal-sec-" + securityEnabled)) {
BlobStore store = container.blobStore;
- //Test for subject with no principals
+ // Test for subject with no principals
Subject who = new Subject();
assertStoreHasExactly(store);
// Tests for case when subject != null (security turned on) and
// acls for the blob are set to WORLD_EVERYTHING
- SettableBlobMeta metadata = new SettableBlobMeta(securityEnabled ? BlobStoreAclHandler.DEFAULT : BlobStoreAclHandler.WORLD_EVERYTHING);
+ SettableBlobMeta metadata = new SettableBlobMeta(securityEnabled
+ ? BlobStoreAclHandler.DEFAULT : BlobStoreAclHandler.WORLD_EVERYTHING);
try (AtomicOutputStream out = store.createBlob("test", metadata, who)) {
out.write(1);
}
assertStoreHasExactly(store, "test");
// With no principals in the subject ACL should always be set to WORLD_EVERYTHING
- assertTrue(metadata.toString().contains("AccessControl(type:OTHER, access:7)"), "ACL does not contain WORLD_EVERYTHING");
+ assertTrue(metadata.toString().contains("AccessControl(type:OTHER, access:7)"),
+ "ACL does not contain WORLD_EVERYTHING");
readAssertEqualsWithAuth(store, who, "test", 1);
}
@@ -409,7 +436,8 @@ public void testBasic(BlobStore store)
}
assertStoreHasExactly(store, "test");
// Testing whether acls are set to WORLD_EVERYTHING
- assertTrue(metadata.toString().contains("AccessControl(type:OTHER, access:7)"), "ACL does not contain WORLD_EVERYTHING");
+ assertTrue(metadata.toString().contains("AccessControl(type:OTHER, access:7)"),
+ "ACL does not contain WORLD_EVERYTHING");
readAssertEquals(store, "test", 1);
LOG.info("Deleting test");
@@ -450,14 +478,16 @@ public void testMultiple(BlobStore store)
throws Exception {
assertStoreHasExactly(store);
LOG.info("Creating test");
- try (AtomicOutputStream out = store.createBlob("test", new SettableBlobMeta(BlobStoreAclHandler.WORLD_EVERYTHING), null)) {
+ try (AtomicOutputStream out = store.createBlob("test",
+ new SettableBlobMeta(BlobStoreAclHandler.WORLD_EVERYTHING), null)) {
out.write(1);
}
assertStoreHasExactly(store, "test");
readAssertEquals(store, "test", 1);
LOG.info("Creating other");
- try (AtomicOutputStream out = store.createBlob("other", new SettableBlobMeta(BlobStoreAclHandler.WORLD_EVERYTHING),
+ try (AtomicOutputStream out = store.createBlob("other",
+ new SettableBlobMeta(BlobStoreAclHandler.WORLD_EVERYTHING),
null)) {
out.write(2);
}
@@ -479,7 +509,8 @@ public void testMultiple(BlobStore store)
readAssertEquals(store, "other", 5);
LOG.info("Creating test again");
- try (AtomicOutputStream out = store.createBlob("test", new SettableBlobMeta(BlobStoreAclHandler.WORLD_EVERYTHING),
+ try (AtomicOutputStream out = store.createBlob("test",
+ new SettableBlobMeta(BlobStoreAclHandler.WORLD_EVERYTHING),
null)) {
out.write(2);
}
diff --git a/external/storm-hdfs-blobstore/src/test/java/org/apache/storm/hdfs/blobstore/HdfsBlobStoreImplTest.java b/external/storm-hdfs-blobstore/src/test/java/org/apache/storm/hdfs/blobstore/HdfsBlobStoreImplTest.java
index f596e4591df..b75323d13b0 100644
--- a/external/storm-hdfs-blobstore/src/test/java/org/apache/storm/hdfs/blobstore/HdfsBlobStoreImplTest.java
+++ b/external/storm-hdfs-blobstore/src/test/java/org/apache/storm/hdfs/blobstore/HdfsBlobStoreImplTest.java
@@ -8,9 +8,9 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
+ *
Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
@@ -18,6 +18,20 @@
*/
package org.apache.storm.hdfs.blobstore;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
@@ -30,22 +44,11 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
-import static org.junit.jupiter.api.Assertions.*;
-
public class HdfsBlobStoreImplTest {
@RegisterExtension
- public static final MiniDFSClusterExtensionClassLevel DFS_CLUSTER_EXTENSION = new MiniDFSClusterExtensionClassLevel();
+ public static final MiniDFSClusterExtensionClassLevel DFS_CLUSTER_EXTENSION =
+ new MiniDFSClusterExtensionClassLevel();
private static final Logger LOG = LoggerFactory.getLogger(HdfsBlobStoreImplTest.class);
public static final String CONCURRENT_TEST_KEY_PREFIX = "concurrent-test-key";
@@ -62,6 +65,7 @@ public class HdfsBlobStoreImplTest {
public class TestHdfsBlobStoreImpl extends HdfsBlobStoreImpl implements AutoCloseable {
Path basePath;
+
public TestHdfsBlobStoreImpl(Path path, Map conf) throws IOException {
super(path, conf);
basePath = path;
@@ -90,14 +94,16 @@ public void testMultiple() throws Exception {
String testString = "testingblob";
String validKey = "validkeyBasic";
- //Will be closed automatically when shutting down the DFS cluster
+ // Will be closed automatically when shutting down the DFS cluster
FileSystem fs = DFS_CLUSTER_EXTENSION.getDfscluster().getFileSystem();
Map conf = new HashMap<>();
- try (TestHdfsBlobStoreImpl hbs = new TestHdfsBlobStoreImpl(blobDir, conf, DFS_CLUSTER_EXTENSION.getHadoopConf())) {
+ try (TestHdfsBlobStoreImpl hbs = new TestHdfsBlobStoreImpl(blobDir, conf,
+ DFS_CLUSTER_EXTENSION.getHadoopConf())) {
// should have created blobDir
assertTrue(fs.exists(blobDir), "BlobStore dir wasn't created");
- assertEquals(HdfsBlobStoreImpl.BLOBSTORE_DIR_PERMISSION, fs.getFileStatus(blobDir).getPermission(),
+ assertEquals(HdfsBlobStoreImpl.BLOBSTORE_DIR_PERMISSION, fs.getFileStatus(blobDir)
+ .getPermission(),
"BlobStore dir was created with wrong permissions");
// test exist with non-existent key
@@ -126,7 +132,8 @@ public void testMultiple() throws Exception {
pfile.commit();
Path dataFile = new Path(new Path(fullKeyDir, validKey), BLOBSTORE_DATA);
assertTrue(fs.exists(dataFile), "blob data not committed");
- assertEquals(HdfsBlobStoreFile.BLOBSTORE_FILE_PERMISSION, fs.getFileStatus(dataFile).getPermission(),
+ assertEquals(HdfsBlobStoreFile.BLOBSTORE_FILE_PERMISSION, fs.getFileStatus(dataFile)
+ .getPermission(),
"BlobStore dir was created with wrong permissions");
assertTrue(hbs.exists(validKey), "key doesn't exist but should");
@@ -162,7 +169,8 @@ public void testMultiple() throws Exception {
assertTrue(fs.exists(fullKeyDir), "BlobStore key dir wasn't created");
pfile.commit();
assertTrue(fs.exists(dataFile), "blob data not committed");
- assertEquals(HdfsBlobStoreFile.BLOBSTORE_FILE_PERMISSION, fs.getFileStatus(dataFile).getPermission(),
+ assertEquals(HdfsBlobStoreFile.BLOBSTORE_FILE_PERMISSION, fs.getFileStatus(dataFile)
+ .getPermission(),
"BlobStore dir was created with wrong permissions");
assertTrue(hbs.exists(validKey), "key doesn't exist but should");
@@ -177,7 +185,8 @@ public void testMultiple() throws Exception {
pfile.commit();
Path dataFile2 = new Path(new Path(fullKeyDir, validKey2), BLOBSTORE_DATA);
assertTrue(fs.exists(dataFile2), "blob data not committed");
- assertEquals(HdfsBlobStoreFile.BLOBSTORE_FILE_PERMISSION, fs.getFileStatus(dataFile2).getPermission(),
+ assertEquals(HdfsBlobStoreFile.BLOBSTORE_FILE_PERMISSION, fs.getFileStatus(dataFile2)
+ .getPermission(),
"BlobStore dir was created with wrong permissions");
assertTrue(hbs.exists(validKey2), "key doesn't exist but should");
@@ -227,7 +236,8 @@ public void testGetFileLength() throws Exception {
Map conf = new HashMap<>();
String validKey = "validkeyBasic";
String testString = "testingblob";
- try (TestHdfsBlobStoreImpl hbs = new TestHdfsBlobStoreImpl(blobDir, conf, DFS_CLUSTER_EXTENSION.getHadoopConf())) {
+ try (TestHdfsBlobStoreImpl hbs = new TestHdfsBlobStoreImpl(blobDir, conf,
+ DFS_CLUSTER_EXTENSION.getHadoopConf())) {
BlobStoreFile pfile = hbs.write(validKey, false);
// Adding metadata to avoid null pointer exception
SettableBlobMeta meta = new SettableBlobMeta();
@@ -241,7 +251,8 @@ public void testGetFileLength() throws Exception {
}
/**
- * Test by listing keys {@link HdfsBlobStoreImpl#listKeys()} in multiple concurrent threads and then ensure that
+ * Test by listing keys {@link HdfsBlobStoreImpl#listKeys()} in multiple concurrent threads and
+ * then ensure that
* same keys are retrived in all the threads without any exceptions.
*/
@Test
@@ -273,9 +284,10 @@ public void run() {
}
Map conf = new HashMap<>();
- try (TestHdfsBlobStoreImpl hbs = new TestHdfsBlobStoreImpl(concurrentTestBlobDir, conf, DFS_CLUSTER_EXTENSION.getHadoopConf())) {
+ try (TestHdfsBlobStoreImpl hbs = new TestHdfsBlobStoreImpl(concurrentTestBlobDir, conf,
+ DFS_CLUSTER_EXTENSION.getHadoopConf())) {
// test write again
- for (int i = 0 ; i < keyCount ; i++) {
+ for (int i = 0; i < keyCount; i++) {
String key = CONCURRENT_TEST_KEY_PREFIX + i;
String val = "This is string " + i;
BlobStoreFile pfile = hbs.write(key, false);
@@ -290,27 +302,28 @@ public void run() {
ConcurrentListerRunnable[] runnables = new ConcurrentListerRunnable[concurrency];
Thread[] threads = new Thread[concurrency];
- for (int i = 0 ; i < concurrency ; i++) {
+ for (int i = 0; i < concurrency; i++) {
runnables[i] = new ConcurrentListerRunnable(hbs, i);
threads[i] = new Thread(runnables[i]);
}
- for (int i = 0 ; i < concurrency ; i++) {
+ for (int i = 0; i < concurrency; i++) {
threads[i].start();
}
- for (int i = 0 ; i < concurrency ; i++) {
+ for (int i = 0; i < concurrency; i++) {
threads[i].join();
}
List keys = runnables[0].keys;
assertEquals(keyCount, keys.size(), "Number of keys (values=" + keys + ")");
- for (int i = 1 ; i < concurrency ; i++) {
+ for (int i = 1; i < concurrency; i++) {
ConcurrentListerRunnable otherRunnable = runnables[i];
assertEquals(keys, otherRunnable.keys);
}
- for (int i = 0 ; i < keyCount ; i++) {
+ for (int i = 0; i < keyCount; i++) {
String key = CONCURRENT_TEST_KEY_PREFIX + i;
hbs.deleteKey(key);
}
- LOG.info("All %d threads have %d keys=[%s]\n", concurrency, keys.size(), String.join(",", keys));
+ LOG.info("All %d threads have %d keys=[%s]\n", concurrency, keys.size(), String
+ .join(",", keys));
}
}
}
diff --git a/external/storm-hdfs-blobstore/src/test/java/org/apache/storm/hdfs/testing/MiniDFSClusterExtension.java b/external/storm-hdfs-blobstore/src/test/java/org/apache/storm/hdfs/testing/MiniDFSClusterExtension.java
index 8bf6b0b3c5b..4e2791ed4c5 100644
--- a/external/storm-hdfs-blobstore/src/test/java/org/apache/storm/hdfs/testing/MiniDFSClusterExtension.java
+++ b/external/storm-hdfs-blobstore/src/test/java/org/apache/storm/hdfs/testing/MiniDFSClusterExtension.java
@@ -7,9 +7,9 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
+ *
Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
@@ -17,19 +17,18 @@
*/
package org.apache.storm.hdfs.testing;
+import static org.apache.hadoop.test.GenericTestUtils.DEFAULT_TEST_DATA_DIR;
+import static org.apache.hadoop.test.GenericTestUtils.SYSPROP_TEST_DATA_DIR;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.File;
+import java.util.function.Supplier;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.junit.jupiter.api.extension.AfterEachCallback;
import org.junit.jupiter.api.extension.BeforeEachCallback;
import org.junit.jupiter.api.extension.ExtensionContext;
-import java.io.File;
-import java.util.function.Supplier;
-
-import static org.apache.hadoop.test.GenericTestUtils.DEFAULT_TEST_DATA_DIR;
-import static org.apache.hadoop.test.GenericTestUtils.SYSPROP_TEST_DATA_DIR;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-
public class MiniDFSClusterExtension implements BeforeEachCallback, AfterEachCallback {
private static final String TEST_BUILD_DATA = "test.build.data";
@@ -74,6 +73,7 @@ public void afterEach(ExtensionContext arg0) throws Exception {
* Get an uncreated directory for tests.
* We use this method to get rid of getTestDir() in GenericTestUtils in Hadoop code
* which uses assert from junit4.
+ *
* @return the absolute directory for tests. Caller is expected to create it.
*/
public static File getTestDir(String subdir) {
@@ -82,6 +82,7 @@ public static File getTestDir(String subdir) {
/**
* Get the (created) base directory for tests.
+ *
* @return the absolute directory
*/
public static File getTestDir() {
diff --git a/external/storm-hdfs-blobstore/src/test/java/org/apache/storm/hdfs/testing/MiniDFSClusterExtensionClassLevel.java b/external/storm-hdfs-blobstore/src/test/java/org/apache/storm/hdfs/testing/MiniDFSClusterExtensionClassLevel.java
index 1fd13d930ce..b0c992e01c6 100644
--- a/external/storm-hdfs-blobstore/src/test/java/org/apache/storm/hdfs/testing/MiniDFSClusterExtensionClassLevel.java
+++ b/external/storm-hdfs-blobstore/src/test/java/org/apache/storm/hdfs/testing/MiniDFSClusterExtensionClassLevel.java
@@ -7,9 +7,9 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
+ *
Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
@@ -17,17 +17,16 @@
*/
package org.apache.storm.hdfs.testing;
+import static org.apache.storm.hdfs.testing.MiniDFSClusterExtension.getTestDir;
+
+import java.io.File;
+import java.util.function.Supplier;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.junit.jupiter.api.extension.AfterAllCallback;
import org.junit.jupiter.api.extension.BeforeAllCallback;
import org.junit.jupiter.api.extension.ExtensionContext;
-import java.io.File;
-import java.util.function.Supplier;
-
-import static org.apache.storm.hdfs.testing.MiniDFSClusterExtension.getTestDir;
-
public class MiniDFSClusterExtensionClassLevel implements BeforeAllCallback, AfterAllCallback {
private static final String TEST_BUILD_DATA = "test.build.data";
diff --git a/external/storm-hdfs-oci/pom.xml b/external/storm-hdfs-oci/pom.xml
index 73b3f0c95fc..e0d4990a92c 100644
--- a/external/storm-hdfs-oci/pom.xml
+++ b/external/storm-hdfs-oci/pom.xml
@@ -104,6 +104,16 @@
+
+ org.openrewrite.maven
+ rewrite-maven-plugin
+
+
+
+ org.codehaus.gmavenplus
+ gmavenplus-plugin
+
+ org.apache.maven.pluginsmaven-checkstyle-plugin
diff --git a/external/storm-hdfs-oci/src/main/java/org/apache/storm/container/oci/HdfsManifestToResourcesPlugin.java b/external/storm-hdfs-oci/src/main/java/org/apache/storm/container/oci/HdfsManifestToResourcesPlugin.java
index 0ad78032c22..87fb8304cbc 100644
--- a/external/storm-hdfs-oci/src/main/java/org/apache/storm/container/oci/HdfsManifestToResourcesPlugin.java
+++ b/external/storm-hdfs-oci/src/main/java/org/apache/storm/container/oci/HdfsManifestToResourcesPlugin.java
@@ -42,9 +42,11 @@ public class HdfsManifestToResourcesPlugin implements OciManifestToResourcesPlug
private FileSystem fs;
private LoadingCache statCache;
- private static final String CONFIG_MEDIA_TYPE = "application/vnd.docker.container.image.v1+json";
+ private static final String CONFIG_MEDIA_TYPE =
+ "application/vnd.docker.container.image.v1+json";
- private static final String LAYER_TAR_GZIP_MEDIA_TYPE = "application/vnd.docker.image.rootfs.diff.tar.gzip";
+ private static final String LAYER_TAR_GZIP_MEDIA_TYPE =
+ "application/vnd.docker.image.rootfs.diff.tar.gzip";
private static final String SHA_256 = "sha256";
@@ -59,10 +61,11 @@ public class HdfsManifestToResourcesPlugin implements OciManifestToResourcesPlug
@Override
public void init(Map conf) throws IOException {
- //login to hdfs
+ // login to hdfs
HadoopLoginUtil.loginHadoop(conf);
- String topLevelDir = ObjectReader.getString(conf.get(DaemonConfig.STORM_OCI_IMAGE_HDFS_TOPLEVEL_DIR));
+ String topLevelDir = ObjectReader.getString(conf
+ .get(DaemonConfig.STORM_OCI_IMAGE_HDFS_TOPLEVEL_DIR));
this.layersDir = topLevelDir + "/layers/";
this.configDir = topLevelDir + "/config/";
@@ -108,7 +111,8 @@ public List getLayerResources(ImageManifest manifest) throws IOExce
FileStatus stat = statCache.get(path);
long timestamp = stat.getModificationTime();
- OciResource ociResource = new OciResource(path.toString(), fileName, size, timestamp, OciResource.OciResourceType.LAYER);
+ OciResource ociResource = new OciResource(path.toString(), fileName, size,
+ timestamp, OciResource.OciResourceType.LAYER);
ociResources.add(ociResource);
} catch (ExecutionException e) {
throw new IOException(e);
@@ -146,7 +150,8 @@ public OciResource getConfigResource(ImageManifest manifest) throws IOException
try {
FileStatus stat = statCache.get(path);
long timestamp = stat.getModificationTime();
- ociResource = new OciResource(path.toString(), hash, size, timestamp, OciResource.OciResourceType.CONFIG);
+ ociResource = new OciResource(path.toString(), hash, size, timestamp,
+ OciResource.OciResourceType.CONFIG);
} catch (ExecutionException e) {
throw new IOException(e);
}
diff --git a/external/storm-hdfs-oci/src/main/java/org/apache/storm/container/oci/HdfsOciResourcesLocalizer.java b/external/storm-hdfs-oci/src/main/java/org/apache/storm/container/oci/HdfsOciResourcesLocalizer.java
index ef749c25f7a..8f2bd4b2380 100644
--- a/external/storm-hdfs-oci/src/main/java/org/apache/storm/container/oci/HdfsOciResourcesLocalizer.java
+++ b/external/storm-hdfs-oci/src/main/java/org/apache/storm/container/oci/HdfsOciResourcesLocalizer.java
@@ -21,7 +21,6 @@
import java.io.File;
import java.io.IOException;
import java.util.Map;
-
import org.apache.commons.io.FileDeleteStrategy;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
@@ -31,7 +30,6 @@
import org.apache.storm.utils.ConfigUtils;
import org.apache.storm.utils.HadoopLoginUtil;
import org.apache.storm.utils.ObjectReader;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -44,25 +42,29 @@ public class HdfsOciResourcesLocalizer implements OciResourcesLocalizerInterface
/**
* Initialization.
+ *
* @param conf the storm conf.
* @throws IOException on I/O exception
*/
@Override
public void init(Map conf) throws IOException {
- //login to hdfs
+ // login to hdfs
HadoopLoginUtil.loginHadoop(conf);
- String resourcesLocalDir = ObjectReader.getString(conf.get(DaemonConfig.STORM_OCI_RESOURCES_LOCAL_DIR),
+ String resourcesLocalDir = ObjectReader.getString(conf
+ .get(DaemonConfig.STORM_OCI_RESOURCES_LOCAL_DIR),
ConfigUtils.supervisorLocalDir(conf) + "/oci-resources");
FileUtils.forceMkdir(new File(resourcesLocalDir));
this.layersLocalDir = resourcesLocalDir + "/layers/";
this.configLocalDir = resourcesLocalDir + "/config/";
- String topLevelDir = ObjectReader.getString(conf.get(DaemonConfig.STORM_OCI_IMAGE_HDFS_TOPLEVEL_DIR));
+ String topLevelDir = ObjectReader.getString(conf
+ .get(DaemonConfig.STORM_OCI_IMAGE_HDFS_TOPLEVEL_DIR));
this.fs = new Path(topLevelDir).getFileSystem(new Configuration());
}
/**
* Download the resources from HDFS to local dir.
+ *
* @param ociResource The oci resource to download
* @return the destination of the oci resource
* @throws IOException on I/O exception
@@ -101,9 +103,10 @@ public synchronized String localize(OciResource ociResource) throws IOException
LOG.info("Starting to copy {} from hdfs to {}", ociResource.getPath(), workingDst);
copyFileLocallyWithRetry(ociResource, workingDst);
- LOG.info("Successfully finished copying {} from hdfs to {}", ociResource.getPath(), workingDst);
+ LOG.info("Successfully finished copying {} from hdfs to {}", ociResource.getPath(),
+ workingDst);
- //set to readable by anyone
+ // set to readable by anyone
boolean setReadable = workingDst.setReadable(true, false);
if (!setReadable) {
throw new IOException("Couldn't set " + workingDst + " to be world-readable");
@@ -117,7 +120,8 @@ public synchronized String localize(OciResource ociResource) throws IOException
return dst.toString();
}
- private synchronized void copyFileLocallyWithRetry(OciResource ociResource, File dst) throws IOException {
+ private synchronized void copyFileLocallyWithRetry(OciResource ociResource,
+ File dst) throws IOException {
IOException lastIoException = null;
for (int retryCount = 0; retryCount < LOCALIZE_MAX_RETRY; retryCount++) {
@@ -129,7 +133,8 @@ private synchronized void copyFileLocallyWithRetry(OciResource ociResource, File
if (dst.exists()) {
FileDeleteStrategy.FORCE.delete(dst);
}
- LOG.warn("{} occurred at attempt {}, deleted corrupt file {} if present", e.toString(), retryCount, dst);
+ LOG.warn("{} occurred at attempt {}, deleted corrupt file {} if present", e
+ .toString(), retryCount, dst);
lastIoException = e;
try {
Thread.sleep(1500);
@@ -140,7 +145,8 @@ private synchronized void copyFileLocallyWithRetry(OciResource ociResource, File
}
}
if (lastIoException != null) {
- LOG.error("Resource localization of {} to {} failed after {} retries", ociResource, dst, LOCALIZE_MAX_RETRY, lastIoException);
+ LOG.error("Resource localization of {} to {} failed after {} retries", ociResource, dst,
+ LOCALIZE_MAX_RETRY, lastIoException);
throw lastIoException;
}
diff --git a/external/storm-hdfs-oci/src/main/java/org/apache/storm/container/oci/LocalOrHdfsImageTagToManifestPlugin.java b/external/storm-hdfs-oci/src/main/java/org/apache/storm/container/oci/LocalOrHdfsImageTagToManifestPlugin.java
index c3244436531..928eeac5473 100644
--- a/external/storm-hdfs-oci/src/main/java/org/apache/storm/container/oci/LocalOrHdfsImageTagToManifestPlugin.java
+++ b/external/storm-hdfs-oci/src/main/java/org/apache/storm/container/oci/LocalOrHdfsImageTagToManifestPlugin.java
@@ -23,8 +23,8 @@
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
-import java.io.UncheckedIOException;
import java.io.InputStreamReader;
+import java.io.UncheckedIOException;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.Map;
@@ -41,7 +41,8 @@
import org.slf4j.LoggerFactory;
public class LocalOrHdfsImageTagToManifestPlugin implements OciImageTagToManifestPluginInterface {
- private static final Logger LOG = LoggerFactory.getLogger(LocalOrHdfsImageTagToManifestPlugin.class);
+ private static final Logger LOG = LoggerFactory
+ .getLogger(LocalOrHdfsImageTagToManifestPlugin.class);
private Map manifestCache;
private ObjectMapper objMapper;
@@ -56,7 +57,8 @@ public class LocalOrHdfsImageTagToManifestPlugin implements OciImageTagToManifes
private int ociCacheRefreshIntervalSecs;
private long lastRefreshTime;
- private static final String LOCAL_OR_HDFS_IMAGE_TAG_TO_MANIFEST_PLUGIN_PREFIX = "storm.oci.local.or.hdfs.image.tag.to.manifest.plugin.";
+ private static final String LOCAL_OR_HDFS_IMAGE_TAG_TO_MANIFEST_PLUGIN_PREFIX =
+ "storm.oci.local.or.hdfs.image.tag.to.manifest.plugin.";
/**
* The HDFS location where the oci image-tag-to-hash file exists.
@@ -79,7 +81,8 @@ public class LocalOrHdfsImageTagToManifestPlugin implements OciImageTagToManifes
/**
* The number of manifests to cache.
*/
- private static final String OCI_NUM_MANIFESTS_TO_CACHE = LOCAL_OR_HDFS_IMAGE_TAG_TO_MANIFEST_PLUGIN_PREFIX + "num.manifests.to.cache";
+ private static final String OCI_NUM_MANIFESTS_TO_CACHE =
+ LOCAL_OR_HDFS_IMAGE_TAG_TO_MANIFEST_PLUGIN_PREFIX + "num.manifests.to.cache";
private static final int SHA256_HASH_LENGTH = 64;
@@ -88,6 +91,7 @@ public class LocalOrHdfsImageTagToManifestPlugin implements OciImageTagToManifes
/**
* Check that a string can be used as an image hash, i.e. it consists of exactly
* {@link #SHA256_HASH_LENGTH} alphanumeric characters.
+ *
* @param hash the string to check
* @return true if the string has the shape of an image hash
*/
@@ -99,7 +103,7 @@ private static boolean isValidHash(String hash) {
public void init(Map conf) throws IOException {
this.conf = conf;
- //login to hdfs
+ // login to hdfs
HadoopLoginUtil.loginHadoop(conf);
localImageTagToHashFile = (String) conf.get(LOCAL_OCI_IMAGE_TAG_TO_HASH_FILE);
@@ -113,7 +117,8 @@ public void init(Map conf) throws IOException {
if (hdfsImageToHashFile == null && localImageTagToHashFile == null) {
throw new IllegalArgumentException("No valid image-tag-to-hash files");
}
- manifestDir = ObjectReader.getString(conf.get(DaemonConfig.STORM_OCI_IMAGE_HDFS_TOPLEVEL_DIR)) + "/manifests/";
+ manifestDir = ObjectReader.getString(conf
+ .get(DaemonConfig.STORM_OCI_IMAGE_HDFS_TOPLEVEL_DIR)) + "/manifests/";
int numManifestsToCache = ObjectReader.getInt(conf.get(OCI_NUM_MANIFESTS_TO_CACHE), 10);
this.objMapper = new ObjectMapper();
this.manifestCache = new LruCache(numManifestsToCache, 0.75f);
@@ -123,7 +128,8 @@ public void init(Map conf) throws IOException {
private boolean loadImageToHashFiles() throws IOException {
boolean ret = false;
try (BufferedReader localBr = getLocalImageToHashReader()) {
- Map localImageToHash = readImageToHashFile(localBr, localImageTagToHashFile);
+ Map localImageToHash = readImageToHashFile(localBr,
+ localImageTagToHashFile);
if (localImageToHash != null && !localImageToHash.equals(localImageToHashCache)) {
localImageToHashCache = localImageToHash;
LOG.info("Reloaded local image tag to hash cache");
@@ -173,7 +179,8 @@ private BufferedReader getHdfsImageToHashReader() throws IOException {
Path imageToHash = new Path(hdfsImageToHashFile);
FileSystem fs = imageToHash.getFileSystem(new Configuration());
if (!fs.exists(imageToHash)) {
- String message = "Could not load hdfs image to hash file, " + hdfsImageToHashFile + " doesn't exist";
+ String message = "Could not load hdfs image to hash file, " + hdfsImageToHashFile
+ + " doesn't exist";
LOG.error(message);
throw new IOException(message);
}
@@ -199,7 +206,8 @@ private BufferedReader getHdfsImageToHashReader() throws IOException {
*
*
This will map both foo/bar:current and fizz/gig:latest to 123456789
*/
- private static Map readImageToHashFile(BufferedReader br, String filePath) throws IOException {
+ private static Map readImageToHashFile(BufferedReader br,
+ String filePath) throws IOException {
if (br == null) {
return null;
}
@@ -236,7 +244,6 @@ private static Map readImageToHashFile(BufferedReader br, String
return imageToHashCache;
}
-
@Override
public synchronized ImageManifest getManifestFromImageTag(String imageTag) throws IOException {
String hash = getHashFromImageTag(imageTag);
@@ -274,7 +281,7 @@ public synchronized String getHashFromImageTag(String imageTag) {
LOG.debug("Refreshing local and hdfs image-tag-to-hash cache");
try {
boolean loaded = loadImageToHashFiles();
- //If this is the first time trying to load the files and yet it failed
+ // If this is the first time trying to load the files and yet it failed
if (!loaded && lastRefreshTime == 0) {
throw new RuntimeException("Couldn't load any image-tag-to-hash-files");
}
diff --git a/external/storm-hdfs-oci/src/test/java/org/apache/storm/container/oci/LocalOrHdfsImageTagToManifestPluginTest.java b/external/storm-hdfs-oci/src/test/java/org/apache/storm/container/oci/LocalOrHdfsImageTagToManifestPluginTest.java
index 2cda2b06378..955e0956b20 100644
--- a/external/storm-hdfs-oci/src/test/java/org/apache/storm/container/oci/LocalOrHdfsImageTagToManifestPluginTest.java
+++ b/external/storm-hdfs-oci/src/test/java/org/apache/storm/container/oci/LocalOrHdfsImageTagToManifestPluginTest.java
@@ -34,9 +34,11 @@
public class LocalOrHdfsImageTagToManifestPluginTest {
- private static final String KNOWN_HASH = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
- private static final String UNKNOWN_HASH = "fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210";
- //same length as a hash, but made of characters that would still escape the manifest directory
+ private static final String KNOWN_HASH =
+ "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
+ private static final String UNKNOWN_HASH =
+ "fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210";
+ // same length as a hash, but made of characters that would still escape the manifest directory
private static final String HASH_LENGTH_PATH = "../../../user/foo/bar/" + "a".repeat(42);
@TempDir
@@ -44,10 +46,12 @@ public class LocalOrHdfsImageTagToManifestPluginTest {
private LocalOrHdfsImageTagToManifestPlugin createPlugin() throws IOException {
Path hashFile = tempDir.resolve("image-tag-to-hash");
- Files.write(hashFile, ("busybox:latest:" + KNOWN_HASH + "\n").getBytes(StandardCharsets.UTF_8));
+ Files.write(hashFile, ("busybox:latest:" + KNOWN_HASH + "\n")
+ .getBytes(StandardCharsets.UTF_8));
Map conf = new HashMap<>();
- conf.put("storm.oci.local.or.hdfs.image.tag.to.manifest.plugin.local.hash.file", hashFile.toString());
+ conf.put("storm.oci.local.or.hdfs.image.tag.to.manifest.plugin.local.hash.file", hashFile
+ .toString());
conf.put(DaemonConfig.STORM_OCI_IMAGE_HDFS_TOPLEVEL_DIR, "/storm/oci");
LocalOrHdfsImageTagToManifestPlugin plugin = new LocalOrHdfsImageTagToManifestPlugin();
@@ -68,18 +72,24 @@ public void testUnmappedImageTagIsUsedAsHashWhenItLooksLikeOne() throws Exceptio
@Test
public void testUnmappedImageTagThatIsNotAHashIsRejected() throws Exception {
LocalOrHdfsImageTagToManifestPlugin plugin = createPlugin();
- assertThrows(UncheckedIOException.class, () -> plugin.getHashFromImageTag("../../../user/foo/bar"));
- assertThrows(UncheckedIOException.class, () -> plugin.getHashFromImageTag("busybox:unknown"));
+ assertThrows(UncheckedIOException.class, () -> plugin
+ .getHashFromImageTag("../../../user/foo/bar"));
+ assertThrows(UncheckedIOException.class, () -> plugin
+ .getHashFromImageTag("busybox:unknown"));
assertThrows(UncheckedIOException.class, () -> plugin.getHashFromImageTag(".."));
assertThrows(UncheckedIOException.class, () -> plugin.getHashFromImageTag("/etc/passwd"));
- assertThrows(UncheckedIOException.class, () -> plugin.getHashFromImageTag(HASH_LENGTH_PATH));
- assertThrows(UncheckedIOException.class, () -> plugin.getHashFromImageTag(UNKNOWN_HASH + "a"));
+ assertThrows(UncheckedIOException.class, () -> plugin
+ .getHashFromImageTag(HASH_LENGTH_PATH));
+ assertThrows(UncheckedIOException.class, () -> plugin.getHashFromImageTag(UNKNOWN_HASH
+ + "a"));
}
@Test
public void testGetManifestFromImageTagRejectsUnmappedNonHashTag() throws Exception {
LocalOrHdfsImageTagToManifestPlugin plugin = createPlugin();
- assertThrows(UncheckedIOException.class, () -> plugin.getManifestFromImageTag("../../../user/foo/bar"));
- assertThrows(UncheckedIOException.class, () -> plugin.getManifestFromImageTag(HASH_LENGTH_PATH));
+ assertThrows(UncheckedIOException.class, () -> plugin
+ .getManifestFromImageTag("../../../user/foo/bar"));
+ assertThrows(UncheckedIOException.class, () -> plugin
+ .getManifestFromImageTag(HASH_LENGTH_PATH));
}
}
diff --git a/external/storm-hdfs/pom.xml b/external/storm-hdfs/pom.xml
index ca1207cfb56..565ddd0e620 100644
--- a/external/storm-hdfs/pom.xml
+++ b/external/storm-hdfs/pom.xml
@@ -145,6 +145,16 @@
1
+
+ org.openrewrite.maven
+ rewrite-maven-plugin
+
+
+
+ org.codehaus.gmavenplus
+ gmavenplus-plugin
+
+ org.apache.maven.pluginsmaven-checkstyle-plugin
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/AbstractAvroSerializer.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/AbstractAvroSerializer.java
index 04cef636714..9b23b19915a 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/AbstractAvroSerializer.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/AbstractAvroSerializer.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -26,15 +32,18 @@
import org.apache.avro.io.DecoderFactory;
import org.apache.avro.io.EncoderFactory;
-//Generously adapted from:
-//https://github.com/kijiproject/kiji-express/blob/master/kiji-express/src/main/scala/org/kiji/express/flow/framework/serialization
+// Generously adapted from:
+// https://github.com/kijiproject/kiji-express/blob/master/kiji-express/src/main/scala/org/kiji/express/flow/framework/serialization
// /AvroSerializer.scala
-//Which has as an ASL2.0 license
+// Which has as an ASL2.0 license
/**
- * This abstract class can be extended to implement concrete classes capable of (de)serializing generic avro objects
- * across a Topology. The methods in the AvroSchemaRegistry interface specify how schemas can be mapped to unique
- * identifiers and vice versa. Implementations based on pre-defining schemas or utilizing an external schema registry
+ * This abstract class can be extended to implement concrete classes capable of (de)serializing
+ * generic avro objects
+ * across a Topology. The methods in the AvroSchemaRegistry interface specify how schemas can be
+ * mapped to unique
+ * identifiers and vice versa. Implementations based on pre-defining schemas or utilizing an
+ * external schema registry
* are provided.
*/
public abstract class AbstractAvroSerializer extends Serializer implements AvroSchemaRegistry {
@@ -57,7 +66,8 @@ public void write(Kryo kryo, Output output, GenericContainer record) {
}
@Override
- public GenericContainer read(Kryo kryo, Input input, Class extends GenericContainer> someClass) {
+ public GenericContainer read(Kryo kryo, Input input,
+ Class extends GenericContainer> someClass) {
Schema theSchema = this.getSchema(input.readString());
GenericDatumReader reader = new GenericDatumReader<>(theSchema);
Decoder decoder = DecoderFactory
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/AvroSchemaRegistry.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/AvroSchemaRegistry.java
index cca5099df45..23a41f8e13e 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/AvroSchemaRegistry.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/AvroSchemaRegistry.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/AvroUtils.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/AvroUtils.java
index 13798bb24f5..35f5aa01374 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/AvroUtils.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/AvroUtils.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -17,9 +23,12 @@
public class AvroUtils {
/**
- * A helper method to extract avro serialization configurations from the topology configuration and register
- * specific kryo serializers as necessary. A default serializer will be provided if none is specified in the
- * configuration. "avro.serializer" should specify the complete class name of the serializer, e.g.
+ * A helper method to extract avro serialization configurations from the topology configuration
+ * and register
+ * specific kryo serializers as necessary. A default serializer will be provided if none is
+ * specified in the
+ * configuration. "avro.serializer" should specify the complete class name of the serializer,
+ * e.g.
* "org.apache.stgorm.hdfs.avro.GenericAvroSerializer"
*
* @param conf The topology configuration
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/FixedAvroSerializer.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/FixedAvroSerializer.java
index 94607b37cb5..f5f5f423b82 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/FixedAvroSerializer.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/FixedAvroSerializer.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -24,8 +30,10 @@
import org.apache.commons.codec.binary.Base64;
/**
- * A class to help (de)serialize a pre-defined set of Avro schemas. Schemas should be listed, one per line, in a file
- * called "FixedAvroSerializer.config", which must be part of the Storm topology jar file. Any schemas intended to be
+ * A class to help (de)serialize a pre-defined set of Avro schemas. Schemas should be listed, one
+ * per line, in a file
+ * called "FixedAvroSerializer.config", which must be part of the Storm topology jar file. Any
+ * schemas intended to be
* used with this class **MUST** be defined in that file.
*/
public class FixedAvroSerializer extends AbstractAvroSerializer {
@@ -35,7 +43,8 @@ public class FixedAvroSerializer extends AbstractAvroSerializer {
final Map schema2fingerprintMap = new HashMap<>();
public FixedAvroSerializer() throws IOException, NoSuchAlgorithmException {
- InputStream in = this.getClass().getClassLoader().getResourceAsStream("FixedAvroSerializer.config");
+ InputStream in = this.getClass().getClassLoader()
+ .getResourceAsStream("FixedAvroSerializer.config");
BufferedReader reader = new BufferedReader(new InputStreamReader(in));
String line;
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/GenericAvroSerializer.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/GenericAvroSerializer.java
index 6bb0e26b411..a1ca4eadb0a 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/GenericAvroSerializer.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/GenericAvroSerializer.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -15,7 +21,8 @@
import org.apache.avro.Schema;
/**
- * A default implementation of the AvroSerializer that will just pass literal schemas back and forth. This should
+ * A default implementation of the AvroSerializer that will just pass literal schemas back and
+ * forth. This should
* only be used if no other serializer will fit a use case.
*/
public class GenericAvroSerializer extends AbstractAvroSerializer {
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/AbstractHdfsBolt.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/AbstractHdfsBolt.java
index ee820663350..d0c6201769e 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/AbstractHdfsBolt.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/AbstractHdfsBolt.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -45,7 +51,7 @@ public abstract class AbstractHdfsBolt extends BaseRichBolt {
private static final Logger LOG = LoggerFactory.getLogger(AbstractHdfsBolt.class);
private static final Integer DEFAULT_RETRY_COUNT = 3;
/**
- * Half of the default Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS
+ * Half of the default Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS.
*/
private static final int DEFAULT_TICK_TUPLE_INTERVAL_SECS = 15;
private static final Integer DEFAULT_MAX_OPEN_FILES = 50;
@@ -89,7 +95,8 @@ protected void rotateOutputFile(Writer writer) throws IOException {
* Marked as final to prevent override. Subclasses should implement the doPrepare() method.
*/
@Override
- public final void prepare(Map conf, TopologyContext topologyContext, OutputCollector collector) {
+ public final void prepare(Map conf, TopologyContext topologyContext,
+ OutputCollector collector) {
this.writeLock = new Object();
if (this.syncPolicy == null) {
throw new IllegalStateException("SyncPolicy must be specified.");
@@ -146,7 +153,7 @@ public final void execute(Tuple tuple) {
this.offset = writer.write(tuple);
tupleBatch.add(tuple);
} catch (IOException e) {
- //If the write failed, try to sync anything already written
+ // If the write failed, try to sync anything already written
LOG.info("Tuple failed to write, forcing a flush of existing data.");
this.collector.reportError(e);
forceSync = true;
@@ -158,13 +165,15 @@ public final void execute(Tuple tuple) {
int attempts = 0;
boolean success = false;
IOException lastException = null;
- // Make every attempt to sync the data we have. If it can't be done then kill the bolt with
+ // Make every attempt to sync the data we have. If it can't be done then kill the
+ // bolt with
// a runtime exception. The filesystem is presumably in a very bad state.
while (success == false && attempts < fileRetryCount) {
attempts += 1;
try {
syncAllWriters();
- LOG.debug("Data synced to filesystem. Ack'ing [{}] tuples", tupleBatch.size());
+ LOG.debug("Data synced to filesystem. Ack'ing [{}] tuples", tupleBatch
+ .size());
for (Tuple t : tupleBatch) {
this.collector.ack(t);
}
@@ -172,7 +181,8 @@ public final void execute(Tuple tuple) {
syncPolicy.reset();
success = true;
} catch (IOException e) {
- LOG.warn("Data could not be synced to filesystem on attempt [{}]", attempts);
+ LOG.warn("Data could not be synced to filesystem on attempt [{}]",
+ attempts);
this.collector.reportError(e);
lastException = e;
}
@@ -186,7 +196,8 @@ public final void execute(Tuple tuple) {
}
tupleBatch.clear();
- throw new RuntimeException("Sync failed [" + attempts + "] times.", lastException);
+ throw new RuntimeException("Sync failed [" + attempts + "] times.",
+ lastException);
}
}
@@ -210,7 +221,8 @@ private Writer getOrCreateWriter(String writerKey, Tuple tuple) throws IOExcepti
/**
* A tuple must be mapped to a writer based on two factors.
- * - bolt specific logic that must separate tuples into different files in the same directory (see the avro bolt
+ * - bolt specific logic that must separate tuples into different files in the same directory
+ * (see the avro bolt
* for an example of this)
* - the directory the tuple will be partioned into
*/
@@ -226,18 +238,23 @@ void doRotationAndRemoveWriter(String writerKey, Writer writer) {
} catch (IOException e) {
this.collector.reportError(e);
LOG.error("File could not be rotated");
- //At this point there is nothing to do. In all likelihood any filesystem operations will fail.
- //The next tuple will almost certainly fail to write and/or sync, which force a rotation. That
- //will give rotateAndReset() a chance to work which includes creating a fresh file handle.
+ // At this point there is nothing to do. In all likelihood any filesystem operations
+ // will fail.
+ // The next tuple will almost certainly fail to write and/or sync, which force a
+ // rotation. That
+ // will give rotateAndReset() a chance to work which includes creating a fresh file
+ // handle.
} finally {
- //rotateOutputFile(writer) has closed the writer. It's safe to remove the writer from the map here.
+ // rotateOutputFile(writer) has closed the writer. It's safe to remove the writer from
+ // the map here.
writers.remove(writerKey);
}
}
@Override
public Map getComponentConfiguration() {
- return TupleUtils.putTickFrequencyIntoComponentConfig(super.getComponentConfiguration(), tickTupleInterval);
+ return TupleUtils.putTickFrequencyIntoComponentConfig(super.getComponentConfiguration(),
+ tickTupleInterval);
}
@Override
@@ -262,7 +279,7 @@ private void doRotationAndRemoveAllWriters() {
LOG.warn("IOException during scheduled file rotation.", e);
}
}
- //above for-loop has closed all the writers. It's safe to clear the map here.
+ // above for-loop has closed all the writers. It's safe to clear the map here.
writers.clear();
}
}
@@ -300,7 +317,8 @@ protected Path getBasePathForNextFile(Tuple tuple) {
this.fileNameFormat.getName(rotation, System.currentTimeMillis()));
}
- protected abstract void doPrepare(Map conf, TopologyContext topologyContext, OutputCollector collector) throws
+ protected abstract void doPrepare(Map conf, TopologyContext topologyContext,
+ OutputCollector collector) throws
IOException;
protected abstract String getWriterKey(Tuple tuple);
@@ -320,8 +338,8 @@ static class WritersMap extends LinkedHashMap {
@Override
protected boolean removeEldestEntry(Map.Entry eldest) {
if (this.size() > this.maxWriters) {
- //The writer must be closed before removed from the map.
- //If it failed, we might lose some data.
+ // The writer must be closed before removed from the map.
+ // If it failed, we might lose some data.
try {
eldest.getValue().close();
} catch (IOException e) {
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/AvroGenericRecordBolt.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/AvroGenericRecordBolt.java
index f00df3a744c..dadfc1bcf66 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/AvroGenericRecordBolt.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/AvroGenericRecordBolt.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -82,14 +88,17 @@ public AvroGenericRecordBolt withPartitioner(Partitioner partitioner) {
}
@Override
- protected void doPrepare(Map conf, TopologyContext topologyContext, OutputCollector collector) throws IOException {
+ protected void doPrepare(Map conf, TopologyContext topologyContext,
+ OutputCollector collector) throws IOException {
LOG.info("Preparing AvroGenericRecord Bolt...");
this.fs = FileSystem.get(URI.create(this.fsUrl), hdfsConfig);
}
/**
- * AvroGenericRecordBolt must override this method because messages with different schemas cannot be written to the
- * same file. By treating the complete schema as the "key" AbstractHdfsBolt will associate a different writer for
+ * AvroGenericRecordBolt must override this method because messages with different schemas
+ * cannot be written to the
+ * same file. By treating the complete schema as the "key" AbstractHdfsBolt will associate a
+ * different writer for
* every distinct schema.
*/
@Override
@@ -101,6 +110,7 @@ protected String getWriterKey(Tuple tuple) {
@Override
protected AbstractHDFSWriter makeNewWriter(Path path, Tuple tuple) throws IOException {
Schema recordSchema = ((GenericRecord) tuple.getValue(0)).getSchema();
- return new AvroGenericRecordHDFSWriter(this.rotationPolicy, path, this.fs.create(path), recordSchema);
+ return new AvroGenericRecordHDFSWriter(this.rotationPolicy, path, this.fs.create(path),
+ recordSchema);
}
}
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/HdfsBolt.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/HdfsBolt.java
index 6677c7b9f74..133460ae695 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/HdfsBolt.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/HdfsBolt.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -94,7 +100,8 @@ public HdfsBolt withMaxOpenFiles(int maxOpenFiles) {
}
@Override
- public void doPrepare(Map conf, TopologyContext topologyContext, OutputCollector collector) throws IOException {
+ public void doPrepare(Map conf, TopologyContext topologyContext,
+ OutputCollector collector) throws IOException {
LOG.info("Preparing HDFS Bolt...");
this.fs = FileSystem.get(URI.create(this.fsUrl), hdfsConfig);
}
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/SequenceFileBolt.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/SequenceFileBolt.java
index 991a23cb905..f90ed88d62d 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/SequenceFileBolt.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/SequenceFileBolt.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -112,7 +118,8 @@ public SequenceFileBolt withMaxOpenFiles(int maxOpenFiles) {
}
@Override
- public void doPrepare(Map conf, TopologyContext topologyContext, OutputCollector collector) throws IOException {
+ public void doPrepare(Map conf, TopologyContext topologyContext,
+ OutputCollector collector) throws IOException {
LOG.info("Preparing Sequence File Bolt...");
if (this.format == null) {
throw new IllegalStateException("SequenceFormat must be specified.");
@@ -134,7 +141,8 @@ protected AbstractHDFSWriter makeNewWriter(Path path, Tuple tuple) throws IOExce
SequenceFile.Writer.file(path),
SequenceFile.Writer.keyClass(this.format.keyClass()),
SequenceFile.Writer.valueClass(this.format.valueClass()),
- SequenceFile.Writer.compression(this.compressionType, this.codecFactory.getCodecByName(this.compressionCodec))
+ SequenceFile.Writer.compression(this.compressionType, this.codecFactory
+ .getCodecByName(this.compressionCodec))
);
return new SequenceFileWriter(this.rotationPolicy, path, writer, this.format);
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/Writer.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/Writer.java
index c6b3fb3d249..e58f964c805 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/Writer.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/Writer.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
+ *
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/DefaultFileNameFormat.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/DefaultFileNameFormat.java
index 157929c3579..0ba0c635fcb 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/DefaultFileNameFormat.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/DefaultFileNameFormat.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -64,7 +70,8 @@ public void prepare(Map conf, TopologyContext topologyContext) {
@Override
public String getName(long rotation, long timeStamp) {
- return this.prefix + this.componentId + "-" + this.taskId + "-" + rotation + "-" + timeStamp + this.extension;
+ return this.prefix + this.componentId + "-" + this.taskId + "-" + rotation + "-"
+ + timeStamp + this.extension;
}
@Override
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/DelimitedRecordFormat.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/DelimitedRecordFormat.java
index f8cdad92923..07e09d6d792 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/DelimitedRecordFormat.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/DelimitedRecordFormat.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/FileNameFormat.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/FileNameFormat.java
index 70210a934b0..9e79d6633f9 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/FileNameFormat.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/FileNameFormat.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -26,6 +32,7 @@ public interface FileNameFormat extends Serializable {
/**
* Returns the filename the HdfsBolt will create.
+ *
* @param rotation the current file rotation number (incremented on every rotation)
* @param timeStamp current time in milliseconds when the rotation occurs
*/
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/RecordFormat.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/RecordFormat.java
index 5102f381863..b759808121a 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/RecordFormat.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/RecordFormat.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/SimpleFileNameFormat.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/SimpleFileNameFormat.java
index d80aaa8cd9f..19d6a160889 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/SimpleFileNameFormat.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/SimpleFileNameFormat.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -66,7 +72,7 @@ public SimpleFileNameFormat withPath(String path) {
}
/**
- * support parameters:
+ * Support parameters:
* $TIME - current time. use withTimeFormat to format.
* $NUM - rotation number
* $HOST - local host name
@@ -81,7 +87,7 @@ public SimpleFileNameFormat withName(String name) {
}
public SimpleFileNameFormat withTimeFormat(String timeFormat) {
- //check format
+ // check format
try {
new SimpleDateFormat(timeFormat);
} catch (Exception e) {
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/rotation/FileRotationPolicy.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/rotation/FileRotationPolicy.java
index 13229dd125d..0bd76fa56b1 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/rotation/FileRotationPolicy.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/rotation/FileRotationPolicy.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -35,7 +41,6 @@ public interface FileRotationPolicy extends Serializable {
*/
boolean mark(Tuple tuple, long offset);
-
/**
* Called after the HdfsBolt rotates a file.
*/
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/rotation/FileSizeRotationPolicy.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/rotation/FileSizeRotationPolicy.java
index eca10d67101..aa05344398b 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/rotation/FileSizeRotationPolicy.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/rotation/FileSizeRotationPolicy.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/rotation/NoRotationPolicy.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/rotation/NoRotationPolicy.java
index f25be14da9f..048ad2bc5bd 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/rotation/NoRotationPolicy.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/rotation/NoRotationPolicy.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/sync/CountSyncPolicy.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/sync/CountSyncPolicy.java
index a048dc3002e..f87eae1d219 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/sync/CountSyncPolicy.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/sync/CountSyncPolicy.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/sync/SyncPolicy.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/sync/SyncPolicy.java
index 12c673f3499..067549131f9 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/sync/SyncPolicy.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/sync/SyncPolicy.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -30,7 +36,6 @@ public interface SyncPolicy extends Serializable {
*/
boolean mark(Tuple tuple, long offset);
-
/**
* Called after the HdfsBolt performs a sync.
*
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/AbstractHDFSWriter.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/AbstractHDFSWriter.java
index caf6b49f925..d326e0277ac 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/AbstractHDFSWriter.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/AbstractHDFSWriter.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -27,7 +33,8 @@ public abstract class AbstractHDFSWriter implements Writer {
protected boolean needsRotation;
public AbstractHDFSWriter(FileRotationPolicy policy, Path path) {
- //This must be defensively copied, because a bolt probably has only one rotation policy object
+ // This must be defensively copied, because a bolt probably has only one rotation policy
+ // object
this.rotationPolicy = policy.copy();
this.filePath = path;
}
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/AvroGenericRecordHDFSWriter.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/AvroGenericRecordHDFSWriter.java
index 713aa587a8b..24b2f741ebe 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/AvroGenericRecordHDFSWriter.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/AvroGenericRecordHDFSWriter.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -36,7 +42,8 @@ public class AvroGenericRecordHDFSWriter extends AbstractHDFSWriter {
private Schema schema;
private DataFileWriter avroWriter;
- public AvroGenericRecordHDFSWriter(FileRotationPolicy policy, Path path, FSDataOutputStream stream, Schema schema) throws IOException {
+ public AvroGenericRecordHDFSWriter(FileRotationPolicy policy, Path path,
+ FSDataOutputStream stream, Schema schema) throws IOException {
super(policy, path);
this.out = stream;
this.schema = schema;
@@ -60,7 +67,8 @@ protected void doSync() throws IOException {
LOG.debug("Attempting to sync all data to filesystem");
if (this.out instanceof HdfsDataOutputStream) {
- ((HdfsDataOutputStream) this.out).hsync(EnumSet.of(HdfsDataOutputStream.SyncFlag.UPDATE_LENGTH));
+ ((HdfsDataOutputStream) this.out).hsync(EnumSet
+ .of(HdfsDataOutputStream.SyncFlag.UPDATE_LENGTH));
} else {
this.out.hsync();
}
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/HDFSWriter.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/HDFSWriter.java
index 8b3dcd1bf2d..db3aaa53b2a 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/HDFSWriter.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/HDFSWriter.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -31,7 +37,8 @@ public class HDFSWriter extends AbstractHDFSWriter {
private FSDataOutputStream out;
private RecordFormat format;
- public HDFSWriter(FileRotationPolicy policy, Path path, FSDataOutputStream out, RecordFormat format) {
+ public HDFSWriter(FileRotationPolicy policy, Path path, FSDataOutputStream out,
+ RecordFormat format) {
super(policy, path);
this.out = out;
this.format = format;
@@ -48,7 +55,8 @@ protected void doWrite(Tuple tuple) throws IOException {
protected void doSync() throws IOException {
LOG.info("Attempting to sync all data to filesystem");
if (this.out instanceof HdfsDataOutputStream) {
- ((HdfsDataOutputStream) this.out).hsync(EnumSet.of(HdfsDataOutputStream.SyncFlag.UPDATE_LENGTH));
+ ((HdfsDataOutputStream) this.out).hsync(EnumSet
+ .of(HdfsDataOutputStream.SyncFlag.UPDATE_LENGTH));
} else {
this.out.hsync();
}
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/HdfsUtils.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/HdfsUtils.java
index 462087eebc4..f7000cdb6d0 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/HdfsUtils.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/HdfsUtils.java
@@ -1,12 +1,17 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -25,9 +30,11 @@
import org.apache.hadoop.ipc.RemoteException;
public class HdfsUtils {
- /** list files sorted by modification time that have not been modified since 'olderThan'. if
+ /**
+ * List files sorted by modification time that have not been modified since 'olderThan'. if
* 'olderThan' is <= 0 then the filtering is disabled */
- public static ArrayList listFilesByModificationTime(FileSystem fs, Path directory, long olderThan)
+ public static ArrayList listFilesByModificationTime(FileSystem fs, Path directory,
+ long olderThan)
throws IOException {
ArrayList fstats = new ArrayList<>();
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/ModifTimeComparator.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/ModifTimeComparator.java
index 47ebdfe304e..d9f3420e276 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/ModifTimeComparator.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/ModifTimeComparator.java
@@ -1,12 +1,17 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -15,7 +20,6 @@
import java.util.Comparator;
import org.apache.hadoop.fs.FileStatus;
-
public class ModifTimeComparator
implements Comparator {
@Override
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/NullPartitioner.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/NullPartitioner.java
index 3137f48b099..c7785c1454b 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/NullPartitioner.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/NullPartitioner.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -15,7 +21,8 @@
import org.apache.storm.tuple.Tuple;
/**
- * The NullPartitioner partitions every tuple to the empty string. In otherwords, no partition sub directories will
+ * The NullPartitioner partitions every tuple to the empty string. In otherwords, no partition sub
+ * directories will
* be added to the path.
*/
public class NullPartitioner implements Partitioner {
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/Partitioner.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/Partitioner.java
index 92f674733b1..de83f2b9f14 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/Partitioner.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/Partitioner.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -18,8 +24,10 @@
public interface Partitioner extends Serializable {
/**
- * Return a relative path that the tuple should be written to. For example, if an HdfsBolt were configured to write
- * to /common/output and a partitioner returned "/foo" then the bolt should open a file in "/common/output/foo"
+ * Return a relative path that the tuple should be written to. For example, if an HdfsBolt were
+ * configured to write
+ * to /common/output and a partitioner returned "/foo" then the bolt should open a file in
+ * "/common/output/foo"
*
*
A best practice is to use Path.SEPARATOR instead of a literal "/"
*
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/SequenceFileWriter.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/SequenceFileWriter.java
index d0507b84ee5..e09da85774e 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/SequenceFileWriter.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/SequenceFileWriter.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -28,7 +34,8 @@ public class SequenceFileWriter extends AbstractHDFSWriter {
private SequenceFile.Writer writer;
private SequenceFormat format;
- public SequenceFileWriter(FileRotationPolicy policy, Path path, SequenceFile.Writer writer, SequenceFormat format) {
+ public SequenceFileWriter(FileRotationPolicy policy, Path path, SequenceFile.Writer writer,
+ SequenceFormat format) {
super(policy, path);
this.writer = writer;
this.format = format;
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/AbstractFileReader.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/AbstractFileReader.java
index 614d2409510..44425a8e4be 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/AbstractFileReader.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/AbstractFileReader.java
@@ -1,12 +1,17 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -15,7 +20,6 @@
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-
abstract class AbstractFileReader implements FileReader {
private final Path file;
@@ -35,7 +39,6 @@ public Path getFilePath() {
return file;
}
-
@Override
public boolean equals(Object o) {
if (this == o) {
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/Configs.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/Configs.java
index 9c54a19b6d8..07a194d4715 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/Configs.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/Configs.java
@@ -1,12 +1,17 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -25,6 +30,7 @@
public class Configs implements Validated {
/**
* Required - chose the file type being consumed.
+ *
* @deprecated please use {@link HdfsSpout#setReaderType(String)}
*/
@Deprecated
@@ -35,6 +41,7 @@ public class Configs implements Validated {
public static final String SEQ = "seq";
/**
* Required - HDFS name node.
+ *
* @deprecated please use {@link HdfsSpout#setHdfsUri(String)}
*/
@Deprecated
@@ -42,6 +49,7 @@ public class Configs implements Validated {
public static final String HDFS_URI = "hdfsspout.hdfs";
/**
* Required - dir from which to read files.
+ *
* @deprecated please use {@link HdfsSpout#setSourceDir(String)}
*/
@Deprecated
@@ -49,6 +57,7 @@ public class Configs implements Validated {
public static final String SOURCE_DIR = "hdfsspout.source.dir";
/**
* Required - completed files will be moved here.
+ *
* @deprecated please use {@link HdfsSpout#setArchiveDir(String)}
*/
@Deprecated
@@ -56,6 +65,7 @@ public class Configs implements Validated {
public static final String ARCHIVE_DIR = "hdfsspout.archive.dir";
/**
* Required - unparsable files will be moved here.
+ *
* @deprecated please use {@link HdfsSpout#setBadFilesDir(String)}
*/
@Deprecated
@@ -63,6 +73,7 @@ public class Configs implements Validated {
public static final String BAD_DIR = "hdfsspout.badfiles.dir";
/**
* Directory in which lock files will be created.
+ *
* @deprecated please use {@link HdfsSpout#setLockDir(String)}
*/
@Deprecated
@@ -70,6 +81,7 @@ public class Configs implements Validated {
public static final String LOCK_DIR = "hdfsspout.lock.dir";
/**
* Commit after N records. 0 disables this.
+ *
* @deprecated please use {@link HdfsSpout#setCommitFrequencyCount(int)}
*/
@Deprecated
@@ -78,6 +90,7 @@ public class Configs implements Validated {
public static final String COMMIT_FREQ_COUNT = "hdfsspout.commit.count";
/**
* Commit after N secs. cannot be disabled.
+ *
* @deprecated please use {@link HdfsSpout#setCommitFrequencySec(int)}
*/
@Deprecated
@@ -86,6 +99,7 @@ public class Configs implements Validated {
public static final String COMMIT_FREQ_SEC = "hdfsspout.commit.sec";
/**
* Max outstanding.
+ *
* @deprecated please use {@link HdfsSpout#setMaxOutstanding(int)}
*/
@Deprecated
@@ -94,6 +108,7 @@ public class Configs implements Validated {
public static final String MAX_OUTSTANDING = "hdfsspout.max.outstanding";
/**
* Lock timeout.
+ *
* @deprecated please use {@link HdfsSpout#setLockTimeoutSec(int)}
*/
@Deprecated
@@ -101,7 +116,8 @@ public class Configs implements Validated {
@IsPositiveNumber
public static final String LOCK_TIMEOUT = "hdfsspout.lock.timeout.sec";
/**
- * If clocks on machines in the Storm cluster are in sync inactivity duration after which locks are considered
+ * If clocks on machines in the Storm cluster are in sync inactivity duration after which locks
+ * are considered
* candidates for being reassigned to another spout.
*
* @deprecated please use {@link HdfsSpout#setClocksInSync(boolean)}
@@ -111,6 +127,7 @@ public class Configs implements Validated {
public static final String CLOCKS_INSYNC = "hdfsspout.clocks.insync";
/**
* Ignore suffix.
+ *
* @deprecated please use {@link HdfsSpout#setIgnoreSuffix(String)}
*/
@Deprecated
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/DirLock.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/DirLock.java
index 488531a040d..d58f07fa5f4 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/DirLock.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/DirLock.java
@@ -1,12 +1,17 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -39,10 +44,12 @@ private DirLock(FileSystem fs, Path lockFile) throws IOException {
this.lockFile = lockFile;
}
- /** Get a lock on file if not already locked.
+ /**
+ * Get a lock on file if not already locked.
*
* @param dir the dir on which to get a lock
- * @return The lock object if it the lock was acquired. Returns null if the dir is already locked.
+ * @return The lock object if it the lock was acquired. Returns null if the dir is already
+ * locked.
* @throws IOException if there were errors
*/
public static DirLock tryLock(FileSystem fs, Path dir) throws IOException {
@@ -55,7 +62,8 @@ public static DirLock tryLock(FileSystem fs, Path dir) throws IOException {
ostream.close();
return new DirLock(fs, lockFile);
} else {
- LOG.debug("Thread ({}) cannot lock dir {} as its already locked.", threadInfo(), dir);
+ LOG.debug("Thread ({}) cannot lock dir {} as its already locked.", threadInfo(),
+ dir);
return null;
}
} catch (IOException e) {
@@ -74,7 +82,7 @@ private static String threadInfo() {
}
/**
- * if the lock on the directory is stale, take ownership.
+ * If the lock on the directory is stale, take ownership.
*/
public static DirLock takeOwnershipIfStale(FileSystem fs, Path dirToLock, int lockTimeoutSec) {
Path dirLockFile = getDirLockFile(dirToLock);
@@ -103,7 +111,8 @@ private static DirLock takeOwnership(FileSystem fs, Path dirLockFile) throws IOE
}
// delete and recreate lock file
- if (fs.delete(dirLockFile, false)) { // returns false if somebody else already deleted it (to take ownership)
+ if (fs.delete(dirLockFile,
+ false)) { // returns false if somebody else already deleted it (to take ownership)
FSDataOutputStream ostream = HdfsUtils.tryCreateFile(fs, dirLockFile);
if (ostream != null) {
ostream.close();
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/FileLock.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/FileLock.java
index 7ff5aaaa29c..3d78c995b5b 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/FileLock.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/FileLock.java
@@ -1,12 +1,17 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -28,8 +33,10 @@
import org.slf4j.LoggerFactory;
/**
- * Facility to synchronize access to HDFS files. Thread gains exclusive access to a file by acquiring
- * a FileLock object. The lock itself is represented as file on HDFS. Relies on atomic file creation.
+ * Facility to synchronize access to HDFS files. Thread gains exclusive access to a file by
+ * acquiring
+ * a FileLock object. The lock itself is represented as file on HDFS. Relies on atomic file
+ * creation.
* Owning thread must heartbeat periodically on the lock to prevent the lock from being deemed as
* stale (i.e. lock whose owning thread have died).
*/
@@ -42,7 +49,8 @@ public class FileLock {
private final FSDataOutputStream lockFileStream;
private LogEntry lastEntry;
- private FileLock(FileSystem fs, Path lockFile, FSDataOutputStream lockFileStream, String spoutId)
+ private FileLock(FileSystem fs, Path lockFile, FSDataOutputStream lockFileStream,
+ String spoutId)
throws IOException {
this.fs = fs;
this.lockFile = lockFile;
@@ -62,7 +70,7 @@ private FileLock(FileSystem fs, Path lockFile, String spoutId, LogEntry entry)
}
/**
- * returns lock on file or null if file is already locked. throws if unexpected problem
+ * Returns lock on file or null if file is already locked. throws if unexpected problem
*/
public static FileLock tryLock(FileSystem fs, Path fileToLock, Path lockDirPath, String spoutId)
throws IOException {
@@ -71,10 +79,12 @@ public static FileLock tryLock(FileSystem fs, Path fileToLock, Path lockDirPath,
try {
FSDataOutputStream ostream = HdfsUtils.tryCreateFile(fs, lockFile);
if (ostream != null) {
- LOG.debug("Acquired lock on file {}. LockFile= {}, Spout = {}", fileToLock, lockFile, spoutId);
+ LOG.debug("Acquired lock on file {}. LockFile= {}, Spout = {}", fileToLock,
+ lockFile, spoutId);
return new FileLock(fs, lockFile, ostream, spoutId);
} else {
- LOG.debug("Cannot lock file {} as its already locked. Spout = {}", fileToLock, spoutId);
+ LOG.debug("Cannot lock file {} as its already locked. Spout = {}", fileToLock,
+ spoutId);
return null;
}
} catch (IOException e) {
@@ -84,7 +94,7 @@ public static FileLock tryLock(FileSystem fs, Path fileToLock, Path lockDirPath,
}
/**
- * checks if lockFile is older than 'olderThan' UTC time by examining the modification time
+ * Checks if lockFile is older than 'olderThan' UTC time by examining the modification time
* on file and (if necessary) the timestamp in last log entry in the file. If its stale, then
* returns the last log entry, else returns null.
*
@@ -95,7 +105,7 @@ public static LogEntry getLastEntryIfStale(FileSystem fs, Path lockFile, long ol
throws IOException {
long modifiedTime = fs.getFileStatus(lockFile).getModificationTime();
if (modifiedTime <= olderThan) { // look
- //Impt: HDFS timestamp may not reflect recent appends, so we double check the
+ // Impt: HDFS timestamp may not reflect recent appends, so we double check the
// timestamp in last line of file to see when the last update was made
LogEntry lastEntry = getLastEntry(fs, lockFile);
if (lastEntry == null) {
@@ -116,7 +126,7 @@ public static LogEntry getLastEntryIfStale(FileSystem fs, Path lockFile, long ol
}
/**
- * returns the last log entry.
+ * Returns the last log entry.
*/
public static LogEntry getLastEntry(FileSystem fs, Path lockFile)
throws IOException {
@@ -131,6 +141,7 @@ public static LogEntry getLastEntry(FileSystem fs, Path lockFile)
/**
* Takes ownership of the lock file if possible.
+ *
* @param lastEntry last entry in the lock file. this param is an optimization.
* we dont scan the lock file again to find its last entry here since
* its already been done once by the logic used to check if the lock
@@ -139,13 +150,15 @@ public static LogEntry getLastEntry(FileSystem fs, Path lockFile)
* @return null if lock File is not recoverable
* @throws IOException if unable to acquire
*/
- public static FileLock takeOwnership(FileSystem fs, Path lockFile, LogEntry lastEntry, String spoutId)
+ public static FileLock takeOwnership(FileSystem fs, Path lockFile, LogEntry lastEntry,
+ String spoutId)
throws IOException {
try {
if (fs instanceof DistributedFileSystem) {
if (!((DistributedFileSystem) fs).recoverLease(lockFile)) {
LOG.warn(
- "Unable to recover lease on lock file {} right now. Cannot transfer ownership. Will need to try later. Spout = {}",
+ "Unable to recover lease on lock file {} right now. Cannot transfer "
+ + "ownership. Will need to try later. Spout = {}",
lockFile, spoutId);
return null;
}
@@ -153,14 +166,18 @@ public static FileLock takeOwnership(FileSystem fs, Path lockFile, LogEntry last
return new FileLock(fs, lockFile, spoutId, lastEntry);
} catch (IOException e) {
if (e instanceof RemoteException
- && ((RemoteException) e).unwrapRemoteException() instanceof AlreadyBeingCreatedException) {
+ && ((RemoteException) e)
+ .unwrapRemoteException() instanceof AlreadyBeingCreatedException) {
LOG.warn(
- "Lock file " + lockFile + "is currently open. Cannot transfer ownership now. Will need to try later. Spout= "
+ "Lock file " + lockFile
+ + "is currently open. Cannot transfer ownership now. Will need to try "
+ + "later. Spout= "
+ spoutId,
e);
return null;
} else { // unexpected error
- LOG.warn("Cannot transfer ownership now for lock file " + lockFile + ". Will need to try later. Spout =" + spoutId, e);
+ LOG.warn("Cannot transfer ownership now for lock file " + lockFile
+ + ". Will need to try later. Spout =" + spoutId, e);
throw e;
}
}
@@ -172,12 +189,14 @@ public static FileLock takeOwnership(FileSystem fs, Path lockFile, LogEntry last
* Impt: Assumes access to lockFilesDir has been externally synchronized such that
* only one thread accessing the same thread
*/
- public static FileLock acquireOldestExpiredLock(FileSystem fs, Path lockFilesDir, int locktimeoutSec, String spoutId)
+ public static FileLock acquireOldestExpiredLock(FileSystem fs, Path lockFilesDir,
+ int locktimeoutSec, String spoutId)
throws IOException {
// list files
long now = System.currentTimeMillis();
long olderThan = now - (locktimeoutSec * 1000);
- Collection listing = HdfsUtils.listFilesByModificationTime(fs, lockFilesDir, olderThan);
+ Collection listing = HdfsUtils.listFilesByModificationTime(fs, lockFilesDir,
+ olderThan);
// locate expired lock files (if any). Try to take ownership (oldest lock first)
for (Path file : listing) {
@@ -206,12 +225,14 @@ public static FileLock acquireOldestExpiredLock(FileSystem fs, Path lockFilesDir
*
* @return a Pair<lock file path, last entry in lock file> .. if expired lock file found
*/
- public static HdfsUtils.Pair locateOldestExpiredLock(FileSystem fs, Path lockFilesDir, int locktimeoutSec)
+ public static HdfsUtils.Pair locateOldestExpiredLock(FileSystem fs,
+ Path lockFilesDir, int locktimeoutSec)
throws IOException {
// list files
long now = System.currentTimeMillis();
long olderThan = now - (locktimeoutSec * 1000);
- Collection listing = HdfsUtils.listFilesByModificationTime(fs, lockFilesDir, olderThan);
+ Collection listing = HdfsUtils.listFilesByModificationTime(fs, lockFilesDir,
+ olderThan);
// locate oldest expired lock file (if any) and take ownership
for (Path file : listing) {
@@ -250,6 +271,7 @@ private void logProgress(String fileOffset, boolean prefixNewLine)
/**
* Release lock by deleting file.
+ *
* @throws IOException if lock file could not be deleted
*/
public void release() throws IOException {
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/FileOffset.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/FileOffset.java
index a8b354a8dcf..665825297fd 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/FileOffset.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/FileOffset.java
@@ -1,12 +1,17 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -25,7 +30,7 @@
interface FileOffset extends Comparable, Cloneable {
/**
- * tests if rhs == currOffset+1.
+ * Tests if rhs == currOffset+1.
*/
boolean isNextOffset(FileOffset rhs);
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/FileReader.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/FileReader.java
index b6e08f4523b..b019aedde5f 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/FileReader.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/FileReader.java
@@ -1,12 +1,17 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/HdfsSpout.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/HdfsSpout.java
index c7a2bb16138..619835b34f9 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/HdfsSpout.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/HdfsSpout.java
@@ -1,12 +1,17 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -44,7 +49,8 @@ public class HdfsSpout extends BaseRichSpout {
private static final Logger LOG = LoggerFactory.getLogger(HdfsSpout.class);
private final AtomicBoolean commitTimeElapsed = new AtomicBoolean(false);
HashMap> inflight = new HashMap<>();
- LinkedBlockingQueue>> retryList = new LinkedBlockingQueue<>();
+ LinkedBlockingQueue>> retryList =
+ new LinkedBlockingQueue<>();
HdfsUtils.Pair lastExpiredLock = null;
// user configurable
private String hdfsUri; // required
@@ -63,7 +69,8 @@ public class HdfsSpout extends BaseRichSpout {
private int maxOutstanding = Configs.DEFAULT_MAX_OUTSTANDING;
private int lockTimeoutSec = Configs.DEFAULT_LOCK_TIMEOUT;
private boolean clocksInSync = true;
- private String inprogressSuffix = ".inprogress"; // not configurable to prevent change between topology restarts
+ private String inprogressSuffix =
+ ".inprogress"; // not configurable to prevent change between topology restarts
private String ignoreSuffix = ".ignore";
private String outputStreamName = null;
private ProgressTracker tracker = null;
@@ -94,10 +101,12 @@ private static void releaseLockAndLog(FileLock fileLock, String spoutId) {
try {
if (fileLock != null) {
fileLock.release();
- LOG.debug("Spout {} released FileLock. SpoutId = {}", fileLock.getLockFile(), spoutId);
+ LOG.debug("Spout {} released FileLock. SpoutId = {}", fileLock.getLockFile(),
+ spoutId);
}
} catch (IOException e) {
- LOG.error("Unable to delete lock file : " + fileLock.getLockFile() + " SpoutId =" + spoutId, e);
+ LOG.error("Unable to delete lock file : " + fileLock.getLockFile() + " SpoutId ="
+ + spoutId, e);
}
}
@@ -106,15 +115,18 @@ private static void validateOrMakeDir(FileSystem fs, Path dir, String dirDescrip
if (fs.exists(dir)) {
if (!fs.isDirectory(dir)) {
LOG.error(dirDescription + " directory is a file, not a dir. " + dir);
- throw new RuntimeException(dirDescription + " directory is a file, not a dir. " + dir);
+ throw new RuntimeException(dirDescription + " directory is a file, not a dir. "
+ + dir);
}
} else if (!fs.mkdirs(dir)) {
LOG.error("Unable to create " + dirDescription + " directory " + dir);
- throw new RuntimeException("Unable to create " + dirDescription + " directory " + dir);
+ throw new RuntimeException("Unable to create " + dirDescription + " directory "
+ + dir);
}
} catch (IOException e) {
LOG.error("Unable to create " + dirDescription + " directory " + dir, e);
- throw new RuntimeException("Unable to create " + dirDescription + " directory " + dir, e);
+ throw new RuntimeException("Unable to create " + dirDescription + " directory " + dir,
+ e);
}
}
@@ -135,7 +147,8 @@ static void checkValidReader(String readerType) {
throw new IllegalArgumentException(readerType + " not found in classpath.", e);
} catch (NoSuchMethodException e) {
LOG.error(readerType + " is missing the expected constructor for Readers.", e);
- throw new IllegalArgumentException(readerType + " is missing the expected constuctor for Readers.");
+ throw new IllegalArgumentException(readerType
+ + " is missing the expected constuctor for Readers.");
}
}
@@ -208,7 +221,8 @@ public HdfsSpout withOutputFields(String... fields) {
}
/**
- * set key name under which HDFS options are placed. (similar to HDFS bolt). default key name is 'hdfs.config'
+ * Set key name under which HDFS options are placed. (similar to HDFS bolt). default key name is
+ * 'hdfs.config'
*/
public HdfsSpout withConfigKey(String configKey) {
this.configKey = configKey;
@@ -273,7 +287,8 @@ public void nextTuple() {
if (tuple != null) {
fileReadCompletely = false;
++tupleCounter;
- MessageId msgId = new MessageId(tupleCounter, reader.getFilePath(), reader.getFileOffset());
+ MessageId msgId = new MessageId(tupleCounter, reader.getFilePath(), reader
+ .getFileOffset());
emitData(tuple, msgId);
if (!ackEnabled) {
@@ -295,7 +310,8 @@ public void nextTuple() {
// don't emit anything .. allow configured spout wait strategy to kick in
return;
} catch (ParseException e) {
- LOG.error("Parsing error when processing at file location " + getFileProgress(reader)
+ LOG.error("Parsing error when processing at file location "
+ + getFileProgress(reader)
+ ". Skipping remainder of file.", e);
markFileAsBad(reader.getFilePath());
// Note: We don't return from this method on ParseException to avoid triggering the
@@ -353,14 +369,18 @@ private void markFileAsBad(Path file) {
String originalName = new Path(fileNameMinusSuffix).getName();
Path newFile = new Path(badFilesDirPath + Path.SEPARATOR + originalName);
- LOG.info("Moving bad file {} to {}. Processed it till offset {}. SpoutID= {}", originalName, newFile, tracker.getCommitPosition(),
+ LOG.info("Moving bad file {} to {}. Processed it till offset {}. SpoutID= {}", originalName,
+ newFile, tracker.getCommitPosition(),
spoutId);
try {
- if (!hdfs.rename(file, newFile)) { // seems this can fail by returning false or throwing exception
- throw new IOException("Move failed for bad file: " + file); // convert false ret value to exception
+ if (!hdfs.rename(file,
+ newFile)) { // seems this can fail by returning false or throwing exception
+ throw new IOException("Move failed for bad file: "
+ + file); // convert false ret value to exception
}
} catch (IOException e) {
- LOG.warn("Error moving bad file: " + file + " to destination " + newFile + " SpoutId =" + spoutId, e);
+ LOG.warn("Error moving bad file: " + file + " to destination " + newFile + " SpoutId ="
+ + spoutId, e);
}
closeReaderAndResetTrackers();
}
@@ -390,7 +410,8 @@ protected void emitData(List tuple, MessageId id) {
@SuppressWarnings("deprecation")
@Override
- public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
+ public void open(Map conf, TopologyContext context,
+ SpoutOutputCollector collector) {
LOG.info("Opening HDFS Spout");
this.conf = conf;
this.commitTimer = new Timer(context.getThisTaskId() + "-commit-timer", true);
@@ -418,7 +439,8 @@ public void open(Map conf, TopologyContext context, SpoutOutputC
if (map != null) {
for (String keyName : map.keySet()) {
LOG.info("HDFS Config override : {} = {} ", keyName,
- ConfigUtils.isCredentialKey(keyName) ? "*****" : String.valueOf(map.get(keyName)));
+ ConfigUtils.isCredentialKey(keyName) ? "*****" : String.valueOf(map
+ .get(keyName)));
this.hdfsConfig.set(keyName, String.valueOf(map.get(keyName)));
}
try {
@@ -510,7 +532,8 @@ public void open(Map conf, TopologyContext context, SpoutOutputC
if (conf.get(Configs.COMMIT_FREQ_SEC) != null) {
commitFrequencySec = Integer.parseInt(conf.get(Configs.COMMIT_FREQ_SEC).toString());
if (commitFrequencySec <= 0) {
- throw new RuntimeException(Configs.COMMIT_FREQ_SEC + " setting must be greater than 0");
+ throw new RuntimeException(Configs.COMMIT_FREQ_SEC
+ + " setting must be greater than 0");
}
}
@@ -571,7 +594,8 @@ public void fail(Object msgId) {
LOG.trace("Fail received for msg id {} on spout {}", msgId, spoutId);
super.fail(msgId);
if (ackEnabled) {
- HdfsUtils.Pair> item = HdfsUtils.Pair.of(msgId, inflight.remove(msgId));
+ HdfsUtils.Pair> item = HdfsUtils.Pair.of(msgId, inflight
+ .remove(msgId));
retryList.add(item);
}
}
@@ -581,15 +605,18 @@ private FileReader pickNextFile() {
// 1) If there are any abandoned files, pick oldest one
lock = getOldestExpiredLock();
if (lock != null) {
- LOG.debug("Spout {} now took over ownership of abandoned FileLock {}", spoutId, lock.getLockFile());
+ LOG.debug("Spout {} now took over ownership of abandoned FileLock {}", spoutId, lock
+ .getLockFile());
Path file = getFileForLockFile(lock.getLockFile(), sourceDirPath);
String resumeFromOffset = lock.getLastLogEntry().fileOffset;
LOG.info("Resuming processing of abandoned file : {}", file);
return createFileReader(file, resumeFromOffset);
}
- // 2) If no abandoned files, then pick oldest file in sourceDirPath, lock it and rename it
- Collection listing = HdfsUtils.listFilesByModificationTime(hdfs, sourceDirPath, 0);
+ // 2) If no abandoned files, then pick oldest file in sourceDirPath, lock it and rename
+ // it
+ Collection listing = HdfsUtils.listFilesByModificationTime(hdfs, sourceDirPath,
+ 0);
for (Path file : listing) {
if (file.getName().endsWith(inprogressSuffix)) {
@@ -623,7 +650,8 @@ private FileReader pickNextFile() {
}
/**
- * If clocks in sync, then acquires the oldest expired lock Else, on first call, just remembers the oldest expired lock, on next call
+ * If clocks in sync, then acquires the oldest expired lock Else, on first call, just remembers
+ * the oldest expired lock, on next call
* check if the lock is updated. if not updated then acquires the lock
*
* @return a lock object
@@ -634,10 +662,12 @@ private FileLock getOldestExpiredLock() throws IOException {
if (dirlock == null) {
dirlock = DirLock.takeOwnershipIfStale(hdfs, lockDirPath, lockTimeoutSec);
if (dirlock == null) {
- LOG.debug("Spout {} could not take over ownership of DirLock for {}", spoutId, lockDirPath);
+ LOG.debug("Spout {} could not take over ownership of DirLock for {}", spoutId,
+ lockDirPath);
return null;
}
- LOG.debug("Spout {} now took over ownership of abandoned DirLock for {}", spoutId, lockDirPath);
+ LOG.debug("Spout {} now took over ownership of abandoned DirLock for {}", spoutId,
+ lockDirPath);
} else {
LOG.debug("Spout {} now owns DirLock for {}", spoutId, lockDirPath);
}
@@ -645,13 +675,16 @@ private FileLock getOldestExpiredLock() throws IOException {
try {
// 2 - if clocks are in sync then simply take ownership of the oldest expired lock
if (clocksInSync) {
- return FileLock.acquireOldestExpiredLock(hdfs, lockDirPath, lockTimeoutSec, spoutId);
+ return FileLock.acquireOldestExpiredLock(hdfs, lockDirPath, lockTimeoutSec,
+ spoutId);
}
// 3 - if clocks are not in sync ..
if (lastExpiredLock == null) {
- // just make a note of the oldest expired lock now and check if its still unmodified after lockTimeoutSec
- lastExpiredLock = FileLock.locateOldestExpiredLock(hdfs, lockDirPath, lockTimeoutSec);
+ // just make a note of the oldest expired lock now and check if its still unmodified
+ // after lockTimeoutSec
+ lastExpiredLock = FileLock.locateOldestExpiredLock(hdfs, lockDirPath,
+ lockTimeoutSec);
lastExpiredLockTime = System.currentTimeMillis();
return null;
}
@@ -663,7 +696,8 @@ private FileLock getOldestExpiredLock() throws IOException {
// If lock file has expired, then own it
FileLock.LogEntry lastEntry = FileLock.getLastEntry(hdfs, lastExpiredLock.getKey());
if (lastEntry.equals(lastExpiredLock.getValue())) {
- FileLock result = FileLock.takeOwnership(hdfs, lastExpiredLock.getKey(), lastEntry, spoutId);
+ FileLock result = FileLock.takeOwnership(hdfs, lastExpiredLock.getKey(), lastEntry,
+ spoutId);
lastExpiredLock = null;
return result;
} else {
@@ -696,7 +730,8 @@ private FileReader createFileReader(Path file)
}
try {
Class> clsType = Class.forName(readerType);
- Constructor> constructor = clsType.getConstructor(FileSystem.class, Path.class, Map.class);
+ Constructor> constructor = clsType.getConstructor(FileSystem.class, Path.class,
+ Map.class);
return (FileReader) constructor.newInstance(this.hdfs, file, conf);
} catch (Exception e) {
LOG.error(e.getMessage(), e);
@@ -721,7 +756,8 @@ private FileReader createFileReader(Path file, String offset)
try {
Class> clsType = Class.forName(readerType);
- Constructor> constructor = clsType.getConstructor(FileSystem.class, Path.class, Map.class, String.class);
+ Constructor> constructor = clsType.getConstructor(FileSystem.class, Path.class,
+ Map.class, String.class);
return (FileReader) constructor.newInstance(this.hdfs, file, conf, offset);
} catch (Exception e) {
LOG.error(e.getMessage(), e);
@@ -749,7 +785,8 @@ private Path renameToInProgressFile(Path file)
}
/**
- * Returns the corresponding input file in the 'sourceDirPath' for the specified lock file. If no such file is found then returns null
+ * Returns the corresponding input file in the 'sourceDirPath' for the specified lock file. If
+ * no such file is found then returns null
*/
private Path getFileForLockFile(Path lockFile, Path sourceDirPath)
throws IOException {
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/ParseException.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/ParseException.java
index a7845ea24c2..6461e6b300a 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/ParseException.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/ParseException.java
@@ -1,12 +1,17 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/ProgressTracker.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/ProgressTracker.java
index 93a9b093009..b185cd9277b 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/ProgressTracker.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/ProgressTracker.java
@@ -1,12 +1,17 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/SequenceFileReader.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/SequenceFileReader.java
index cf9dc1767b6..6c2201de1ba 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/SequenceFileReader.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/SequenceFileReader.java
@@ -1,12 +1,17 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -35,16 +40,16 @@ public class SequenceFileReader
private final SequenceFileReader.Offset offset;
-
private final KeyT key;
private final ValueT value;
-
public SequenceFileReader(FileSystem fs, Path file, Map conf)
throws IOException {
super(fs, file);
- int bufferSize = !conf.containsKey(BUFFER_SIZE) ? DEFAULT_BUFF_SIZE : Integer.parseInt(conf.get(BUFFER_SIZE).toString());
- this.reader = new SequenceFile.Reader(fs.getConf(), SequenceFile.Reader.file(file), SequenceFile.Reader.bufferSize(bufferSize));
+ int bufferSize = !conf.containsKey(BUFFER_SIZE) ? DEFAULT_BUFF_SIZE : Integer.parseInt(conf
+ .get(BUFFER_SIZE).toString());
+ this.reader = new SequenceFile.Reader(fs.getConf(), SequenceFile.Reader.file(file),
+ SequenceFile.Reader.bufferSize(bufferSize));
this.key = (KeyT) ReflectionUtils.newInstance(reader.getKeyClass(), fs.getConf());
this.value = (ValueT) ReflectionUtils.newInstance(reader.getValueClass(), fs.getConf());
this.offset = new SequenceFileReader.Offset(0, 0, 0);
@@ -53,15 +58,18 @@ public SequenceFileReader(FileSystem fs, Path file, Map conf)
public SequenceFileReader(FileSystem fs, Path file, Map conf, String offset)
throws IOException {
super(fs, file);
- int bufferSize = !conf.containsKey(BUFFER_SIZE) ? DEFAULT_BUFF_SIZE : Integer.parseInt(conf.get(BUFFER_SIZE).toString());
+ int bufferSize = !conf.containsKey(BUFFER_SIZE) ? DEFAULT_BUFF_SIZE : Integer.parseInt(conf
+ .get(BUFFER_SIZE).toString());
this.offset = new SequenceFileReader.Offset(offset);
- this.reader = new SequenceFile.Reader(fs.getConf(), SequenceFile.Reader.file(file), SequenceFile.Reader.bufferSize(bufferSize));
+ this.reader = new SequenceFile.Reader(fs.getConf(), SequenceFile.Reader.file(file),
+ SequenceFile.Reader.bufferSize(bufferSize));
this.key = (KeyT) ReflectionUtils.newInstance(reader.getKeyClass(), fs.getConf());
this.value = (ValueT) ReflectionUtils.newInstance(reader.getValueClass(), fs.getConf());
skipToOffset(this.reader, this.offset, this.key);
}
- private static void skipToOffset(SequenceFile.Reader reader, Offset offset, K key) throws IOException {
+ private static void skipToOffset(SequenceFile.Reader reader, Offset offset,
+ K key) throws IOException {
reader.sync(offset.lastSyncPoint);
for (int i = 0; i < offset.recordsSinceLastSync; ++i) {
reader.next(key);
@@ -93,7 +101,6 @@ public Offset getFileOffset() {
return offset;
}
-
public static class Offset implements FileOffset {
public long lastSyncPoint;
public long recordsSinceLastSync;
@@ -138,7 +145,8 @@ public Offset(String offset) {
}
} catch (Exception e) {
throw new IllegalArgumentException("'" + offset
- + "' cannot be interpreted. It is not in expected format for SequenceFileReader."
+ + "' cannot be interpreted. It is not in expected format for "
+ + "SequenceFileReader."
+ " Format e.g. {sync=123:afterSync=345:record=67}");
}
}
@@ -206,8 +214,9 @@ void increment(boolean syncSeen, long newBytePosition) {
@Override
public Offset clone() {
- return new Offset(lastSyncPoint, recordsSinceLastSync, currentRecord, currRecordEndOffset, prevRecordEndOffset);
+ return new Offset(lastSyncPoint, recordsSinceLastSync, currentRecord,
+ currRecordEndOffset, prevRecordEndOffset);
}
- } //class Offset
-} //class
+ } // class Offset
+} // class
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/TextFileReader.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/TextFileReader.java
index 42924c91d18..1b96d3c2605 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/TextFileReader.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/TextFileReader.java
@@ -1,12 +1,17 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -39,19 +44,23 @@ public TextFileReader(FileSystem fs, Path file, Map conf) throws
this(fs, file, conf, new TextFileReader.Offset(0, 0));
}
- public TextFileReader(FileSystem fs, Path file, Map conf, String startOffset) throws IOException {
+ public TextFileReader(FileSystem fs, Path file, Map conf,
+ String startOffset) throws IOException {
this(fs, file, conf, new TextFileReader.Offset(startOffset));
}
- private TextFileReader(FileSystem fs, Path file, Map conf, TextFileReader.Offset startOffset)
+ private TextFileReader(FileSystem fs, Path file, Map conf,
+ TextFileReader.Offset startOffset)
throws IOException {
super(fs, file);
offset = startOffset;
FSDataInputStream in = fs.open(file);
- String charSet = (conf == null || !conf.containsKey(CHARSET)) ? "UTF-8" : conf.get(CHARSET).toString();
+ String charSet = (conf == null || !conf.containsKey(CHARSET)) ? "UTF-8" : conf.get(CHARSET)
+ .toString();
int buffSz =
- (conf == null || !conf.containsKey(BUFFER_SIZE)) ? DEFAULT_BUFF_SIZE : Integer.parseInt(conf.get(BUFFER_SIZE).toString());
+ (conf == null || !conf.containsKey(BUFFER_SIZE)) ? DEFAULT_BUFF_SIZE : Integer
+ .parseInt(conf.get(BUFFER_SIZE).toString());
reader = new BufferedReader(new InputStreamReader(in, charSet), buffSz);
if (offset.charOffset > 0) {
reader.skip(offset.charOffset);
@@ -125,7 +134,8 @@ public Offset(String offset) {
}
} catch (Exception e) {
throw new IllegalArgumentException("'" + offset
- + "' cannot be interpreted. It is not in expected format for TextFileReader."
+ + "' cannot be interpreted. It is not in expected format for "
+ + "TextFileReader."
+ " Format e.g. {char=123:line=5}");
}
}
@@ -188,5 +198,5 @@ public int hashCode() {
public Offset clone() {
return new Offset(charOffset, lineNumber);
}
- } //class Offset
+ } // class Offset
}
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/HdfsState.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/HdfsState.java
index 118a113a204..a35ab4b1eda 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/HdfsState.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/HdfsState.java
@@ -62,12 +62,12 @@ public class HdfsState implements State {
private volatile TxnRecord lastSeenTxn;
private Path indexFilePath;
-
HdfsState(Options options) {
this.options = options;
}
- void prepare(Map conf, IMetricsContext metrics, int partitionIndex, int numPartitions) {
+ void prepare(Map conf, IMetricsContext metrics, int partitionIndex,
+ int numPartitions) {
this.options.prepare(conf, partitionIndex, numPartitions);
initLastTxn(conf, partitionIndex);
}
@@ -114,8 +114,10 @@ private TxnRecord getTxnRecord(Path indexFilePath) throws IOException {
}
private void initLastTxn(Map conf, int partition) {
- // include partition id in the file name so that index for different partitions are independent.
- String indexFileName = String.format(".index.%s.%d", conf.get(Config.TOPOLOGY_NAME), partition);
+ // include partition id in the file name so that index for different partitions are
+ // independent.
+ String indexFileName = String.format(".index.%s.%d", conf.get(Config.TOPOLOGY_NAME),
+ partition);
this.indexFilePath = new Path(options.fileNameFormat.getPath(), indexFileName);
try {
this.lastSeenTxn = getTxnRecord(indexFilePath);
@@ -132,7 +134,8 @@ private void updateIndex(long txId) {
try (FSDataOutputStream out = this.options.fs.create(tmpPath, true);
BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(out))) {
- TxnRecord txnRecord = new TxnRecord(txId, options.currentFile.toString(), this.options.getCurrentOffset());
+ TxnRecord txnRecord = new TxnRecord(txId, options.currentFile.toString(), this.options
+ .getCurrentOffset());
bw.write(txnRecord.toString());
bw.newLine();
bw.flush();
@@ -156,7 +159,8 @@ private void updateIndex(long txId) {
@Override
public void beginCommit(Long txId) {
if (txId <= lastSeenTxn.txnid) {
- LOG.info("txID {} is already processed, lastSeenTxn {}. Triggering recovery.", txId, lastSeenTxn);
+ LOG.info("txID {} is already processed, lastSeenTxn {}. Triggering recovery.", txId,
+ lastSeenTxn);
long start = System.currentTimeMillis();
options.recover(lastSeenTxn.dataFilePath, lastSeenTxn.offset);
LOG.info("Recovery took {} ms.", System.currentTimeMillis() - start);
@@ -184,7 +188,7 @@ public void updateState(List tuples, TridentCollector tridentColle
}
/**
- * for unit tests.
+ * For unit tests.
*/
void close() throws IOException {
this.options.closeOutputFile();
@@ -208,7 +212,8 @@ public abstract static class Options implements Serializable {
abstract void execute(List tuples) throws IOException;
- abstract void doPrepare(Map conf, int partitionIndex, int numPartitions) throws IOException;
+ abstract void doPrepare(Map conf, int partitionIndex,
+ int numPartitions) throws IOException;
abstract long getCurrentOffset() throws IOException;
@@ -238,19 +243,23 @@ protected void rotateOutputFile() throws IOException {
rotateOutputFile(true);
}
-
void prepare(Map conf, int partitionIndex, int numPartitions) {
if (this.rotationPolicy == null) {
throw new IllegalStateException("RotationPolicy must be specified.");
} else if (this.rotationPolicy instanceof FileSizeRotationPolicy) {
long rotationBytes = ((FileSizeRotationPolicy) rotationPolicy).getMaxBytes();
LOG.warn("FileSizeRotationPolicy specified with {} bytes.", rotationBytes);
- LOG.warn("Recovery will fail if data files cannot be copied within topology.message.timeout.secs.");
- LOG.warn("Ensure that the data files does not grow too big with the FileSizeRotationPolicy.");
+ LOG.warn("Recovery will fail if data files cannot be copied within "
+ + "topology.message.timeout.secs.");
+ LOG.warn("Ensure that the data files does not grow too big with the "
+ + "FileSizeRotationPolicy.");
} else if (this.rotationPolicy instanceof TimedRotationPolicy) {
- LOG.warn("TimedRotationPolicy specified with interval {} ms.", ((TimedRotationPolicy) rotationPolicy).getInterval());
- LOG.warn("Recovery will fail if data files cannot be copied within topology.message.timeout.secs.");
- LOG.warn("Ensure that the data files does not grow too big with the TimedRotationPolicy.");
+ LOG.warn("TimedRotationPolicy specified with interval {} ms.",
+ ((TimedRotationPolicy) rotationPolicy).getInterval());
+ LOG.warn("Recovery will fail if data files cannot be copied within "
+ + "topology.message.timeout.secs.");
+ LOG.warn("Ensure that the data files does not grow too big with the "
+ + "TimedRotationPolicy.");
}
if (this.fsUrl == null) {
throw new IllegalStateException("File system URL must be specified.");
@@ -276,7 +285,8 @@ void prepare(Map conf, int partitionIndex, int numPartitions) {
}
/**
- * Recovers nBytes from srcFile to the new file created by calling rotateOutputFile and then deletes the srcFile.
+ * Recovers nBytes from srcFile to the new file created by calling rotateOutputFile and then
+ * deletes the srcFile.
*/
private void recover(String srcFile, long numberOfBytes) {
try {
@@ -285,7 +295,8 @@ private void recover(String srcFile, long numberOfBytes) {
this.rotationPolicy.reset();
if (numberOfBytes > 0) {
doRecover(srcPath, numberOfBytes);
- LOG.info("Recovered {} bytes from {} to {}", numberOfBytes, srcFile, currentFile);
+ LOG.info("Recovered {} bytes from {} to {}", numberOfBytes, srcFile,
+ currentFile);
} else {
LOG.info("Nothing to recover from {}", srcFile);
}
@@ -335,7 +346,7 @@ public HdfsFileOptions withRotationPolicy(FileRotationPolicy rotationPolicy) {
*
Set the size of the buffer used for hdfs file copy in case of recovery. The default
* value is 131072.
*
- *
Note: The lower limit for the parameter is 4096, below which the
+ *
Note: The lower limit for the parameter is 4096, below which the
* option is ignored.
*
* @param sizeInBytes the buffer size in bytes
@@ -353,7 +364,8 @@ public HdfsFileOptions addRotationAction(RotationAction action) {
}
@Override
- void doPrepare(Map conf, int partitionIndex, int numPartitions) throws IOException {
+ void doPrepare(Map conf, int partitionIndex,
+ int numPartitions) throws IOException {
LOG.info("Preparing HDFS File state...");
this.fs = FileSystem.get(URI.create(this.fsUrl), hdfsConfig);
}
@@ -371,7 +383,8 @@ public void doCommit(Long txId) throws IOException {
this.rotationPolicy.reset();
} else {
if (this.out instanceof HdfsDataOutputStream) {
- ((HdfsDataOutputStream) this.out).hsync(EnumSet.of(HdfsDataOutputStream.SyncFlag.UPDATE_LENGTH));
+ ((HdfsDataOutputStream) this.out).hsync(EnumSet
+ .of(HdfsDataOutputStream.SyncFlag.UPDATE_LENGTH));
} else {
this.out.hsync();
}
@@ -386,7 +399,8 @@ void doRecover(Path srcPath, long numberOfBytes) throws IOException {
this.offset = numberOfBytes;
}
- private void copyBytes(FSDataInputStream is, FSDataOutputStream out, long bytesToCopy) throws IOException {
+ private void copyBytes(FSDataInputStream is, FSDataOutputStream out,
+ long bytesToCopy) throws IOException {
byte[] buf = new byte[bufferSize];
int n;
while ((n = is.read(buf)) != -1 && bytesToCopy > 0) {
@@ -402,7 +416,8 @@ void closeOutputFile() throws IOException {
@Override
Path createOutputFile() throws IOException {
- Path path = new Path(this.fileNameFormat.getPath(), this.fileNameFormat.getName(this.rotation, System.currentTimeMillis()));
+ Path path = new Path(this.fileNameFormat.getPath(), this.fileNameFormat
+ .getName(this.rotation, System.currentTimeMillis()));
this.out = this.fs.create(path);
return path;
}
@@ -465,7 +480,8 @@ public SequenceFileOptions addRotationAction(RotationAction action) {
}
@Override
- void doPrepare(Map conf, int partitionIndex, int numPartitions) throws IOException {
+ void doPrepare(Map conf, int partitionIndex,
+ int numPartitions) throws IOException {
LOG.info("Preparing Sequence File State...");
if (this.format == null) {
throw new IllegalStateException("SequenceFormat must be specified.");
@@ -490,7 +506,6 @@ public void doCommit(Long txId) throws IOException {
}
}
-
@Override
void doRecover(Path srcPath, long numberOfBytes) throws Exception {
SequenceFile.Reader reader = new SequenceFile.Reader(this.hdfsConfig,
@@ -507,13 +522,15 @@ void doRecover(Path srcPath, long numberOfBytes) throws Exception {
@Override
Path createOutputFile() throws IOException {
Path p = new Path(this.fsUrl + this.fileNameFormat.getPath(),
- this.fileNameFormat.getName(this.rotation, System.currentTimeMillis()));
+ this.fileNameFormat.getName(this.rotation, System
+ .currentTimeMillis()));
this.writer = SequenceFile.createWriter(
this.hdfsConfig,
SequenceFile.Writer.file(p),
SequenceFile.Writer.keyClass(this.format.keyClass()),
SequenceFile.Writer.valueClass(this.format.valueClass()),
- SequenceFile.Writer.compression(this.compressionType, this.codecFactory.getCodecByName(this.compressionCodec))
+ SequenceFile.Writer.compression(this.compressionType, this.codecFactory
+ .getCodecByName(this.compressionCodec))
);
return p;
}
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/HdfsStateFactory.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/HdfsStateFactory.java
index 568f8bc9310..2cb1e2bffa9 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/HdfsStateFactory.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/HdfsStateFactory.java
@@ -37,7 +37,8 @@ public HdfsStateFactory withOptions(HdfsState.Options options) {
}
@Override
- public State makeState(Map conf, IMetricsContext metrics, int partitionIndex, int numPartitions) {
+ public State makeState(Map conf, IMetricsContext metrics, int partitionIndex,
+ int numPartitions) {
LOG.info("makeState(partitonIndex={}, numpartitions={}", partitionIndex, numPartitions);
HdfsState state = new HdfsState(this.options);
state.prepare(conf, metrics, partitionIndex, numPartitions);
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/HdfsUpdater.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/HdfsUpdater.java
index a63bb40c0e8..3a5faa91740 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/HdfsUpdater.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/HdfsUpdater.java
@@ -25,7 +25,8 @@
public class HdfsUpdater extends BaseStateUpdater {
@Override
- public void updateState(HdfsState state, List tuples, TridentCollector collector) {
+ public void updateState(HdfsState state, List tuples,
+ TridentCollector collector) {
state.updateState(tuples, collector);
}
}
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/DefaultFileNameFormat.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/DefaultFileNameFormat.java
index e48e1986700..a79f0d586e0 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/DefaultFileNameFormat.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/DefaultFileNameFormat.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *
http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License
+ * is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions
* and limitations under the License.
*/
@@ -14,7 +20,6 @@
import java.util.Map;
-
/**
* Creates file names with the following format:
*
@@ -63,7 +68,8 @@ public void prepare(Map conf, int partitionIndex, int numPartiti
@Override
public String getName(long rotation, long timeStamp) {
- return this.prefix + "-" + this.partitionIndex + "-" + rotation + "-" + timeStamp + this.extension;
+ return this.prefix + "-" + this.partitionIndex + "-" + rotation + "-" + timeStamp
+ + this.extension;
}
@Override
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/DefaultSequenceFormat.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/DefaultSequenceFormat.java
index f33c03083cb..afa6a76736a 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/DefaultSequenceFormat.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/DefaultSequenceFormat.java
@@ -39,7 +39,6 @@ public DefaultSequenceFormat(String keyField, String valueField) {
this.valueField = valueField;
}
-
@Override
public Class keyClass() {
return LongWritable.class;
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/DelimitedRecordFormat.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/DelimitedRecordFormat.java
index c12b478f964..d81aeba8fd3 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/DelimitedRecordFormat.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/DelimitedRecordFormat.java
@@ -1,12 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to
+ * you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ *