Details
-
Improvement
-
Status: Closed
-
Minor
-
Resolution: Duplicate
-
None
-
None
-
None
-
None
Description
In HiveInterpreter.java, maximum query parallelism is hardcoded to 10.
public Scheduler getScheduler()
I tried adding an interpreter property but getScheduler is called before properties are read.
tw-172-25-131-152 incubator-zeppelin (presto) $ git diff diff --git a/hive/src/main/java/org/apache/zeppelin/hive/HiveInterpreter.java b/hive/src/main/java/org/apache/zeppelin/hive/HiveInterpreter.java index 47b97bf..9c3bcc3 100644 --- a/hive/src/main/java/org/apache/zeppelin/hive/HiveInterpreter.java +++ b/hive/src/main/java/org/apache/zeppelin/hive/HiveInterpreter.java @@ -52,6 +52,8 @@ public class HiveInterpreter extends Interpreter { static final String COMMON_KEY = "common"; static final String MAX_LINE_KEY = "max_count"; static final String MAX_LINE_DEFAULT = "1000"; + static final String MAX_PARALLELISM_KEY = "max_parallelism"; + static final String MAX_PARALLELISM_DEFAULT = "10"; static final String DEFAULT_KEY = "default"; static final String DRIVER_KEY = "driver"; @@ -67,6 +69,7 @@ public class HiveInterpreter extends Interpreter { static final String UPDATE_COUNT_HEADER = "Update Count"; static final String COMMON_MAX_LINE = COMMON_KEY + DOT + MAX_LINE_KEY; + static final String COMMON_MAX_PARALLELISM = COMMON_KEY + DOT + MAX_PARALLELISM_KEY; static final String DEFAULT_DRIVER = DEFAULT_KEY + DOT + DRIVER_KEY; static final String DEFAULT_URL = DEFAULT_KEY + DOT + URL_KEY; @@ -86,6 +89,7 @@ public class HiveInterpreter extends Interpreter { HiveInterpreter.class.getName(), new InterpreterPropertyBuilder() .add(COMMON_MAX_LINE, MAX_LINE_DEFAULT, "Maximum line of results") + .add(COMMON_MAX_PARALLELISM, MAX_PARALLELISM_DEFAULT, "Maximum queries in parallel") .add(DEFAULT_DRIVER, "org.apache.hive.jdbc.HiveDriver", "Hive JDBC driver") .add(DEFAULT_URL, "jdbc:hive2://localhost:10000", "The URL for HiveServer2.") .add(DEFAULT_USER, "hive", "The hive user") @@ -340,6 +344,14 @@ public class HiveInterpreter extends Interpreter { propertiesMap.get(COMMON_KEY).getProperty(MAX_LINE_KEY, MAX_LINE_DEFAULT)); } + private int getMaxParallelism() { + logger.debug("property: {}", propertiesMap.get(COMMON_KEY)); + logger.debug( + propertiesMap.get(COMMON_KEY).getProperty(MAX_PARALLELISM_KEY, MAX_PARALLELISM_DEFAULT)); + return Integer.valueOf( + propertiesMap.get(COMMON_KEY).getProperty(MAX_PARALLELISM_KEY, MAX_PARALLELISM_DEFAULT)); + } + public String getPropertyKey(String cmd) { int firstLineIndex = cmd.indexOf("\n"); if (-1 == firstLineIndex) { @@ -377,7 +389,7 @@ public class HiveInterpreter extends Interpreter { @Override public Scheduler getScheduler() { return SchedulerFactory.singleton().createOrGetParallelScheduler( - HiveInterpreter.class.getName() + this.hashCode(), 50); + HiveInterpreter.class.getName() + this.hashCode(), getMaxParallelism()); } @Override
Attachments
Issue Links
- is duplicated by
-
ZEPPELIN-995 Change scheduler for JDBC interpreter to use concurrent execution
- Resolved